Lines Matching refs:sem
107 # define DEBUG_RWSEMS_WARN_ON(c, sem) do { \ argument
110 #c, atomic_long_read(&(sem)->count), \
111 (unsigned long) sem->magic, \
112 atomic_long_read(&(sem)->owner), (long)current, \
113 list_empty(&(sem)->wait_list) ? "" : "not ")) \
117 # define DEBUG_RWSEMS_WARN_ON(c, sem) argument
176 static inline void rwsem_set_owner(struct rw_semaphore *sem) in rwsem_set_owner() argument
178 atomic_long_set(&sem->owner, (long)current); in rwsem_set_owner()
179 trace_android_vh_rwsem_set_owner(sem); in rwsem_set_owner()
182 static inline void rwsem_clear_owner(struct rw_semaphore *sem) in rwsem_clear_owner() argument
184 atomic_long_set(&sem->owner, 0); in rwsem_clear_owner()
190 static inline bool rwsem_test_oflags(struct rw_semaphore *sem, long flags) in rwsem_test_oflags() argument
192 return atomic_long_read(&sem->owner) & flags; in rwsem_test_oflags()
205 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, in __rwsem_set_reader_owned() argument
209 (atomic_long_read(&sem->owner) & RWSEM_RD_NONSPINNABLE); in __rwsem_set_reader_owned()
211 atomic_long_set(&sem->owner, val); in __rwsem_set_reader_owned()
214 static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) in rwsem_set_reader_owned() argument
216 __rwsem_set_reader_owned(sem, current); in rwsem_set_reader_owned()
217 trace_android_vh_rwsem_set_reader_owned(sem); in rwsem_set_reader_owned()
223 static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem) in is_rwsem_reader_owned() argument
229 long count = atomic_long_read(&sem->count); in is_rwsem_reader_owned()
234 return rwsem_test_oflags(sem, RWSEM_READER_OWNED); in is_rwsem_reader_owned()
244 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) in rwsem_clear_reader_owned() argument
246 unsigned long val = atomic_long_read(&sem->owner); in rwsem_clear_reader_owned()
249 if (atomic_long_try_cmpxchg(&sem->owner, &val, in rwsem_clear_reader_owned()
255 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) in rwsem_clear_reader_owned() argument
264 static inline void rwsem_set_nonspinnable(struct rw_semaphore *sem) in rwsem_set_nonspinnable() argument
266 unsigned long owner = atomic_long_read(&sem->owner); in rwsem_set_nonspinnable()
273 } while (!atomic_long_try_cmpxchg(&sem->owner, &owner, in rwsem_set_nonspinnable()
277 static inline bool rwsem_read_trylock(struct rw_semaphore *sem) in rwsem_read_trylock() argument
279 long cnt = atomic_long_add_return_acquire(RWSEM_READER_BIAS, &sem->count); in rwsem_read_trylock()
281 rwsem_set_nonspinnable(sem); in rwsem_read_trylock()
292 static inline struct task_struct *rwsem_owner(struct rw_semaphore *sem) in rwsem_owner() argument
295 (atomic_long_read(&sem->owner) & ~RWSEM_OWNER_FLAGS_MASK); in rwsem_owner()
303 rwsem_owner_flags(struct rw_semaphore *sem, unsigned long *pflags) in rwsem_owner_flags() argument
305 unsigned long owner = atomic_long_read(&sem->owner); in rwsem_owner_flags()
331 void __init_rwsem(struct rw_semaphore *sem, const char *name, in __init_rwsem() argument
338 debug_check_no_locks_freed((void *)sem, sizeof(*sem)); in __init_rwsem()
339 lockdep_init_map_wait(&sem->dep_map, name, key, 0, LD_WAIT_SLEEP); in __init_rwsem()
342 sem->magic = sem; in __init_rwsem()
344 atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE); in __init_rwsem()
345 raw_spin_lock_init(&sem->wait_lock); in __init_rwsem()
346 INIT_LIST_HEAD(&sem->wait_list); in __init_rwsem()
347 atomic_long_set(&sem->owner, 0L); in __init_rwsem()
349 osq_lock_init(&sem->osq); in __init_rwsem()
351 trace_android_vh_rwsem_init(sem); in __init_rwsem()
355 #define rwsem_first_waiter(sem) \ argument
356 list_first_entry(&sem->wait_list, struct rwsem_waiter, list)
397 static void rwsem_mark_wake(struct rw_semaphore *sem, in rwsem_mark_wake() argument
405 lockdep_assert_held(&sem->wait_lock); in rwsem_mark_wake()
411 waiter = rwsem_first_waiter(sem); in rwsem_mark_wake()
432 if (unlikely(atomic_long_read(&sem->count) < 0)) in rwsem_mark_wake()
444 oldcount = atomic_long_fetch_add(adjustment, &sem->count); in rwsem_mark_wake()
457 atomic_long_add(-adjustment, &sem->count); in rwsem_mark_wake()
471 __rwsem_set_reader_owned(sem, owner); in rwsem_mark_wake()
498 list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) { in rwsem_mark_wake()
505 trace_android_vh_rwsem_mark_wake_readers(sem, waiter); in rwsem_mark_wake()
515 if (list_empty(&sem->wait_list)) { in rwsem_mark_wake()
524 if (woken && (atomic_long_read(&sem->count) & RWSEM_FLAG_HANDOFF)) in rwsem_mark_wake()
528 atomic_long_add(adjustment, &sem->count); in rwsem_mark_wake()
560 static inline bool rwsem_try_write_lock(struct rw_semaphore *sem, in rwsem_try_write_lock() argument
565 lockdep_assert_held(&sem->wait_lock); in rwsem_try_write_lock()
567 count = atomic_long_read(&sem->count); in rwsem_try_write_lock()
585 if (list_is_singular(&sem->wait_list)) in rwsem_try_write_lock()
588 } while (!atomic_long_try_cmpxchg_acquire(&sem->count, &count, new)); in rwsem_try_write_lock()
597 rwsem_set_owner(sem); in rwsem_try_write_lock()
607 static inline bool rwsem_try_read_lock_unqueued(struct rw_semaphore *sem) in rwsem_try_read_lock_unqueued() argument
609 long count = atomic_long_read(&sem->count); in rwsem_try_read_lock_unqueued()
614 count = atomic_long_fetch_add_acquire(RWSEM_READER_BIAS, &sem->count); in rwsem_try_read_lock_unqueued()
616 rwsem_set_reader_owned(sem); in rwsem_try_read_lock_unqueued()
622 atomic_long_add(-RWSEM_READER_BIAS, &sem->count); in rwsem_try_read_lock_unqueued()
629 static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) in rwsem_try_write_lock_unqueued() argument
631 long count = atomic_long_read(&sem->count); in rwsem_try_write_lock_unqueued()
634 if (atomic_long_try_cmpxchg_acquire(&sem->count, &count, in rwsem_try_write_lock_unqueued()
636 rwsem_set_owner(sem); in rwsem_try_write_lock_unqueued()
653 static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem, in rwsem_can_spin_on_owner() argument
667 owner = rwsem_owner_flags(sem, &flags); in rwsem_can_spin_on_owner()
676 trace_android_vh_rwsem_can_spin_on_owner(sem, &ret, nonspinnable == RWSEM_WR_NONSPINNABLE); in rwsem_can_spin_on_owner()
714 rwsem_spin_on_owner(struct rw_semaphore *sem, unsigned long nonspinnable) in rwsem_spin_on_owner() argument
722 owner = rwsem_owner_flags(sem, &flags); in rwsem_spin_on_owner()
729 trace_android_vh_rwsem_opt_spin_start(sem, &time_out, &cnt, true); in rwsem_spin_on_owner()
738 new = rwsem_owner_flags(sem, &new_flags); in rwsem_spin_on_owner()
776 static inline u64 rwsem_rspin_threshold(struct rw_semaphore *sem) in rwsem_rspin_threshold() argument
778 long count = atomic_long_read(&sem->count); in rwsem_rspin_threshold()
789 static bool rwsem_optimistic_spin(struct rw_semaphore *sem, bool wlock) in rwsem_optimistic_spin() argument
803 if (!osq_lock(&sem->osq)) in rwsem_optimistic_spin()
815 trace_android_vh_rwsem_opt_spin_start(sem, &time_out, &cnt, false); in rwsem_optimistic_spin()
819 owner_state = rwsem_spin_on_owner(sem, nonspinnable); in rwsem_optimistic_spin()
826 taken = wlock ? rwsem_try_write_lock_unqueued(sem) in rwsem_optimistic_spin()
827 : rwsem_try_read_lock_unqueued(sem); in rwsem_optimistic_spin()
844 if (rwsem_test_oflags(sem, nonspinnable)) in rwsem_optimistic_spin()
846 rspin_threshold = rwsem_rspin_threshold(sem); in rwsem_optimistic_spin()
858 rwsem_set_nonspinnable(sem); in rwsem_optimistic_spin()
912 osq_unlock(&sem->osq); in rwsem_optimistic_spin()
913 trace_android_vh_rwsem_opt_spin_finish(sem, taken, wlock); in rwsem_optimistic_spin()
930 static inline void clear_wr_nonspinnable(struct rw_semaphore *sem) in clear_wr_nonspinnable() argument
932 if (rwsem_test_oflags(sem, RWSEM_WR_NONSPINNABLE)) in clear_wr_nonspinnable()
933 atomic_long_andnot(RWSEM_WR_NONSPINNABLE, &sem->owner); in clear_wr_nonspinnable()
955 static inline bool rwsem_reader_phase_trylock(struct rw_semaphore *sem, in rwsem_reader_phase_trylock() argument
958 unsigned long owner = atomic_long_read(&sem->owner); in rwsem_reader_phase_trylock()
964 rwsem_try_read_lock_unqueued(sem)) { in rwsem_reader_phase_trylock()
972 static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem, in rwsem_can_spin_on_owner() argument
978 static inline bool rwsem_optimistic_spin(struct rw_semaphore *sem, bool wlock) in rwsem_optimistic_spin() argument
983 static inline void clear_wr_nonspinnable(struct rw_semaphore *sem) { } in clear_wr_nonspinnable() argument
985 static inline bool rwsem_reader_phase_trylock(struct rw_semaphore *sem, in rwsem_reader_phase_trylock() argument
992 rwsem_spin_on_owner(struct rw_semaphore *sem, unsigned long nonspinnable) in rwsem_spin_on_owner() argument
1003 rwsem_down_read_slowpath(struct rw_semaphore *sem, int state) in rwsem_down_read_slowpath() argument
1015 waiter.last_rowner = atomic_long_read(&sem->owner); in rwsem_down_read_slowpath()
1019 if (!rwsem_can_spin_on_owner(sem, RWSEM_RD_NONSPINNABLE)) in rwsem_down_read_slowpath()
1025 atomic_long_add(-RWSEM_READER_BIAS, &sem->count); in rwsem_down_read_slowpath()
1027 if (rwsem_optimistic_spin(sem, false)) { in rwsem_down_read_slowpath()
1033 if ((atomic_long_read(&sem->count) & RWSEM_FLAG_WAITERS)) { in rwsem_down_read_slowpath()
1034 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1035 if (!list_empty(&sem->wait_list)) in rwsem_down_read_slowpath()
1036 rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, in rwsem_down_read_slowpath()
1038 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1042 return sem; in rwsem_down_read_slowpath()
1043 } else if (rwsem_reader_phase_trylock(sem, waiter.last_rowner)) { in rwsem_down_read_slowpath()
1046 return sem; in rwsem_down_read_slowpath()
1054 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1055 if (list_empty(&sem->wait_list)) { in rwsem_down_read_slowpath()
1062 if (adjustment && !(atomic_long_read(&sem->count) & in rwsem_down_read_slowpath()
1066 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1067 rwsem_set_reader_owned(sem); in rwsem_down_read_slowpath()
1071 return sem; in rwsem_down_read_slowpath()
1077 sem, &already_on_list); in rwsem_down_read_slowpath()
1079 list_add_tail(&waiter.list, &sem->wait_list); in rwsem_down_read_slowpath()
1083 count = atomic_long_add_return(adjustment, &sem->count); in rwsem_down_read_slowpath()
1085 count = atomic_long_read(&sem->count); in rwsem_down_read_slowpath()
1094 clear_wr_nonspinnable(sem); in rwsem_down_read_slowpath()
1099 rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); in rwsem_down_read_slowpath()
1101 trace_android_vh_rwsem_wake(sem); in rwsem_down_read_slowpath()
1102 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1106 trace_android_vh_rwsem_read_wait_start(sem); in rwsem_down_read_slowpath()
1114 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1117 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1126 trace_android_vh_rwsem_read_wait_finish(sem); in rwsem_down_read_slowpath()
1129 return sem; in rwsem_down_read_slowpath()
1133 if (list_empty(&sem->wait_list)) { in rwsem_down_read_slowpath()
1135 &sem->count); in rwsem_down_read_slowpath()
1137 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1139 trace_android_vh_rwsem_read_wait_finish(sem); in rwsem_down_read_slowpath()
1148 static inline void rwsem_disable_reader_optspin(struct rw_semaphore *sem, in rwsem_disable_reader_optspin() argument
1152 atomic_long_or(RWSEM_RD_NONSPINNABLE, &sem->owner); in rwsem_disable_reader_optspin()
1161 rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) in rwsem_down_write_slowpath() argument
1167 struct rw_semaphore *ret = sem; in rwsem_down_write_slowpath()
1172 if (rwsem_can_spin_on_owner(sem, RWSEM_WR_NONSPINNABLE) && in rwsem_down_write_slowpath()
1173 rwsem_optimistic_spin(sem, true)) { in rwsem_down_write_slowpath()
1176 return sem; in rwsem_down_write_slowpath()
1184 disable_rspin = atomic_long_read(&sem->owner) & RWSEM_NONSPINNABLE; in rwsem_down_write_slowpath()
1194 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1197 wstate = list_empty(&sem->wait_list) ? WRITER_FIRST : WRITER_NOT_FIRST; in rwsem_down_write_slowpath()
1201 sem, &already_on_list); in rwsem_down_write_slowpath()
1203 list_add_tail(&waiter.list, &sem->wait_list); in rwsem_down_write_slowpath()
1207 count = atomic_long_read(&sem->count); in rwsem_down_write_slowpath()
1220 rwsem_mark_wake(sem, (count & RWSEM_READER_MASK) in rwsem_down_write_slowpath()
1229 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1232 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1235 atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count); in rwsem_down_write_slowpath()
1239 trace_android_vh_rwsem_wake(sem); in rwsem_down_write_slowpath()
1241 trace_android_vh_rwsem_write_wait_start(sem); in rwsem_down_write_slowpath()
1244 if (rwsem_try_write_lock(sem, wstate)) { in rwsem_down_write_slowpath()
1249 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1260 rwsem_spin_on_owner(sem, RWSEM_NONSPINNABLE) == OWNER_NULL) in rwsem_down_write_slowpath()
1279 (rwsem_first_waiter(sem) == &waiter)) in rwsem_down_write_slowpath()
1282 count = atomic_long_read(&sem->count); in rwsem_down_write_slowpath()
1298 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1301 trace_android_vh_rwsem_write_wait_finish(sem); in rwsem_down_write_slowpath()
1303 rwsem_disable_reader_optspin(sem, disable_rspin); in rwsem_down_write_slowpath()
1304 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1311 trace_android_vh_rwsem_write_wait_finish(sem); in rwsem_down_write_slowpath()
1312 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1316 atomic_long_andnot(RWSEM_FLAG_HANDOFF, &sem->count); in rwsem_down_write_slowpath()
1318 if (list_empty(&sem->wait_list)) in rwsem_down_write_slowpath()
1319 atomic_long_andnot(RWSEM_FLAG_WAITERS, &sem->count); in rwsem_down_write_slowpath()
1321 rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); in rwsem_down_write_slowpath()
1322 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1333 static struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem, long count) in rwsem_wake() argument
1338 raw_spin_lock_irqsave(&sem->wait_lock, flags); in rwsem_wake()
1340 if (!list_empty(&sem->wait_list)) in rwsem_wake()
1341 rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); in rwsem_wake()
1342 trace_android_vh_rwsem_wake_finish(sem); in rwsem_wake()
1344 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); in rwsem_wake()
1347 return sem; in rwsem_wake()
1355 static struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem) in rwsem_downgrade_wake() argument
1360 raw_spin_lock_irqsave(&sem->wait_lock, flags); in rwsem_downgrade_wake()
1362 if (!list_empty(&sem->wait_list)) in rwsem_downgrade_wake()
1363 rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q); in rwsem_downgrade_wake()
1365 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); in rwsem_downgrade_wake()
1368 return sem; in rwsem_downgrade_wake()
1374 static inline void __down_read(struct rw_semaphore *sem) in __down_read() argument
1376 if (!rwsem_read_trylock(sem)) { in __down_read()
1377 rwsem_down_read_slowpath(sem, TASK_UNINTERRUPTIBLE); in __down_read()
1378 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); in __down_read()
1380 rwsem_set_reader_owned(sem); in __down_read()
1384 static inline int __down_read_interruptible(struct rw_semaphore *sem) in __down_read_interruptible() argument
1386 if (!rwsem_read_trylock(sem)) { in __down_read_interruptible()
1387 if (IS_ERR(rwsem_down_read_slowpath(sem, TASK_INTERRUPTIBLE))) in __down_read_interruptible()
1389 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); in __down_read_interruptible()
1391 rwsem_set_reader_owned(sem); in __down_read_interruptible()
1396 static inline int __down_read_killable(struct rw_semaphore *sem) in __down_read_killable() argument
1398 if (!rwsem_read_trylock(sem)) { in __down_read_killable()
1399 if (IS_ERR(rwsem_down_read_slowpath(sem, TASK_KILLABLE))) in __down_read_killable()
1401 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); in __down_read_killable()
1403 rwsem_set_reader_owned(sem); in __down_read_killable()
1408 static inline int __down_read_trylock(struct rw_semaphore *sem) in __down_read_trylock() argument
1412 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); in __down_read_trylock()
1419 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, in __down_read_trylock()
1421 rwsem_set_reader_owned(sem); in __down_read_trylock()
1432 static inline void __down_write(struct rw_semaphore *sem) in __down_write() argument
1436 if (unlikely(!atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, in __down_write()
1438 rwsem_down_write_slowpath(sem, TASK_UNINTERRUPTIBLE); in __down_write()
1441 rwsem_set_owner(sem); in __down_write()
1445 static inline int __down_write_killable(struct rw_semaphore *sem) in __down_write_killable() argument
1449 if (unlikely(!atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, in __down_write_killable()
1451 if (IS_ERR(rwsem_down_write_slowpath(sem, TASK_KILLABLE))) in __down_write_killable()
1455 rwsem_set_owner(sem); in __down_write_killable()
1460 static inline int __down_write_trylock(struct rw_semaphore *sem) in __down_write_trylock() argument
1464 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); in __down_write_trylock()
1467 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, in __down_write_trylock()
1469 rwsem_set_owner(sem); in __down_write_trylock()
1479 static inline void __up_read(struct rw_semaphore *sem) in __up_read() argument
1483 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); in __up_read()
1484 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); in __up_read()
1487 rwsem_clear_reader_owned(sem); in __up_read()
1488 tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count); in __up_read()
1489 DEBUG_RWSEMS_WARN_ON(tmp < 0, sem); in __up_read()
1492 clear_wr_nonspinnable(sem); in __up_read()
1493 rwsem_wake(sem, tmp); in __up_read()
1495 trace_android_vh_rwsem_up_read_end(sem); in __up_read()
1501 static inline void __up_write(struct rw_semaphore *sem) in __up_write() argument
1505 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); in __up_write()
1510 DEBUG_RWSEMS_WARN_ON((rwsem_owner(sem) != current) && in __up_write()
1511 !rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem); in __up_write()
1514 rwsem_clear_owner(sem); in __up_write()
1515 tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count); in __up_write()
1517 rwsem_wake(sem, tmp); in __up_write()
1518 trace_android_vh_rwsem_up_write_end(sem); in __up_write()
1524 static inline void __downgrade_write(struct rw_semaphore *sem) in __downgrade_write() argument
1535 DEBUG_RWSEMS_WARN_ON(rwsem_owner(sem) != current, sem); in __downgrade_write()
1537 -RWSEM_WRITER_LOCKED+RWSEM_READER_BIAS, &sem->count); in __downgrade_write()
1538 rwsem_set_reader_owned(sem); in __downgrade_write()
1540 rwsem_downgrade_wake(sem); in __downgrade_write()
1546 void __sched down_read(struct rw_semaphore *sem) in down_read() argument
1549 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); in down_read()
1551 LOCK_CONTENDED(sem, __down_read_trylock, __down_read); in down_read()
1555 int __sched down_read_interruptible(struct rw_semaphore *sem) in down_read_interruptible() argument
1558 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); in down_read_interruptible()
1560 if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_interruptible)) { in down_read_interruptible()
1561 rwsem_release(&sem->dep_map, _RET_IP_); in down_read_interruptible()
1569 int __sched down_read_killable(struct rw_semaphore *sem) in down_read_killable() argument
1572 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); in down_read_killable()
1574 if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) { in down_read_killable()
1575 rwsem_release(&sem->dep_map, _RET_IP_); in down_read_killable()
1586 int down_read_trylock(struct rw_semaphore *sem) in down_read_trylock() argument
1588 int ret = __down_read_trylock(sem); in down_read_trylock()
1591 rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); in down_read_trylock()
1599 void __sched down_write(struct rw_semaphore *sem) in down_write() argument
1602 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); in down_write()
1603 LOCK_CONTENDED(sem, __down_write_trylock, __down_write); in down_write()
1610 int __sched down_write_killable(struct rw_semaphore *sem) in down_write_killable() argument
1613 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); in down_write_killable()
1615 if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, in down_write_killable()
1617 rwsem_release(&sem->dep_map, _RET_IP_); in down_write_killable()
1628 int down_write_trylock(struct rw_semaphore *sem) in down_write_trylock() argument
1630 int ret = __down_write_trylock(sem); in down_write_trylock()
1633 rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_); in down_write_trylock()
1642 void up_read(struct rw_semaphore *sem) in up_read() argument
1644 rwsem_release(&sem->dep_map, _RET_IP_); in up_read()
1645 __up_read(sem); in up_read()
1652 void up_write(struct rw_semaphore *sem) in up_write() argument
1654 rwsem_release(&sem->dep_map, _RET_IP_); in up_write()
1655 trace_android_vh_rwsem_write_finished(sem); in up_write()
1656 __up_write(sem); in up_write()
1663 void downgrade_write(struct rw_semaphore *sem) in downgrade_write() argument
1665 lock_downgrade(&sem->dep_map, _RET_IP_); in downgrade_write()
1666 trace_android_vh_rwsem_write_finished(sem); in downgrade_write()
1667 __downgrade_write(sem); in downgrade_write()
1673 void down_read_nested(struct rw_semaphore *sem, int subclass) in down_read_nested() argument
1676 rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); in down_read_nested()
1677 LOCK_CONTENDED(sem, __down_read_trylock, __down_read); in down_read_nested()
1681 int down_read_killable_nested(struct rw_semaphore *sem, int subclass) in down_read_killable_nested() argument
1684 rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); in down_read_killable_nested()
1686 if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) { in down_read_killable_nested()
1687 rwsem_release(&sem->dep_map, _RET_IP_); in down_read_killable_nested()
1695 void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest) in _down_write_nest_lock() argument
1698 rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_); in _down_write_nest_lock()
1699 LOCK_CONTENDED(sem, __down_write_trylock, __down_write); in _down_write_nest_lock()
1703 void down_read_non_owner(struct rw_semaphore *sem) in down_read_non_owner() argument
1706 __down_read(sem); in down_read_non_owner()
1707 __rwsem_set_reader_owned(sem, NULL); in down_read_non_owner()
1711 void down_write_nested(struct rw_semaphore *sem, int subclass) in down_write_nested() argument
1714 rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); in down_write_nested()
1715 LOCK_CONTENDED(sem, __down_write_trylock, __down_write); in down_write_nested()
1719 int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass) in down_write_killable_nested() argument
1722 rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); in down_write_killable_nested()
1724 if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, in down_write_killable_nested()
1726 rwsem_release(&sem->dep_map, _RET_IP_); in down_write_killable_nested()
1734 void up_read_non_owner(struct rw_semaphore *sem) in up_read_non_owner() argument
1736 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); in up_read_non_owner()
1737 __up_read(sem); in up_read_non_owner()