110 		#c, atomic_long_read(&(sem)->count),		\
229 	long count = atomic_long_read(&sem->count);  in is_rwsem_reader_owned()  local
231 	if (count & RWSEM_WRITER_MASK)  in is_rwsem_reader_owned()
279 	long cnt = atomic_long_add_return_acquire(RWSEM_READER_BIAS, &sem->count);  in rwsem_read_trylock()
344 	atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE);  in __init_rwsem()
432 	if (unlikely(atomic_long_read(&sem->count) < 0))  in rwsem_mark_wake()
444 		oldcount = atomic_long_fetch_add(adjustment, &sem->count);  in rwsem_mark_wake()
457 			atomic_long_add(-adjustment, &sem->count);  in rwsem_mark_wake()
524 	if (woken && (atomic_long_read(&sem->count) & RWSEM_FLAG_HANDOFF))  in rwsem_mark_wake()
528 		atomic_long_add(adjustment, &sem->count);  in rwsem_mark_wake()
563 	long count, new;  in rwsem_try_write_lock()  local
567 	count = atomic_long_read(&sem->count);  in rwsem_try_write_lock()
569 		bool has_handoff = !!(count & RWSEM_FLAG_HANDOFF);  in rwsem_try_write_lock()
574 		new = count;  in rwsem_try_write_lock()
576 		if (count & RWSEM_LOCK_MASK) {  in rwsem_try_write_lock()
588 	} while (!atomic_long_try_cmpxchg_acquire(&sem->count, &count, new));  in rwsem_try_write_lock()
609 	long count = atomic_long_read(&sem->count);  in rwsem_try_read_lock_unqueued()  local
611 	if (count & (RWSEM_WRITER_MASK | RWSEM_FLAG_HANDOFF))  in rwsem_try_read_lock_unqueued()
614 	count = atomic_long_fetch_add_acquire(RWSEM_READER_BIAS, &sem->count);  in rwsem_try_read_lock_unqueued()
615 	if (!(count & (RWSEM_WRITER_MASK | RWSEM_FLAG_HANDOFF))) {  in rwsem_try_read_lock_unqueued()
622 	atomic_long_add(-RWSEM_READER_BIAS, &sem->count);  in rwsem_try_read_lock_unqueued()
631 	long count = atomic_long_read(&sem->count);  in rwsem_try_write_lock_unqueued()  local
633 	while (!(count & (RWSEM_LOCK_MASK|RWSEM_FLAG_HANDOFF))) {  in rwsem_try_write_lock_unqueued()
634 		if (atomic_long_try_cmpxchg_acquire(&sem->count, &count,  in rwsem_try_write_lock_unqueued()
635 					count | RWSEM_WRITER_LOCKED)) {  in rwsem_try_write_lock_unqueued()
778 	long count = atomic_long_read(&sem->count);  in rwsem_rspin_threshold()  local
779 	int readers = count >> RWSEM_READER_SHIFT;  in rwsem_rspin_threshold()
1005 	long count, adjustment = -RWSEM_READER_BIAS;  in rwsem_down_read_slowpath()  local
1025 	atomic_long_add(-RWSEM_READER_BIAS, &sem->count);  in rwsem_down_read_slowpath()
1033 		if ((atomic_long_read(&sem->count) & RWSEM_FLAG_WAITERS)) {  in rwsem_down_read_slowpath()
1062 		if (adjustment && !(atomic_long_read(&sem->count) &  in rwsem_down_read_slowpath()
1083 		count = atomic_long_add_return(adjustment, &sem->count);  in rwsem_down_read_slowpath()
1085 		count = atomic_long_read(&sem->count);  in rwsem_down_read_slowpath()
1093 	if (!(count & RWSEM_LOCK_MASK)) {  in rwsem_down_read_slowpath()
1097 	if (wake || (!(count & RWSEM_WRITER_MASK) &&  in rwsem_down_read_slowpath()
1135 				   &sem->count);  in rwsem_down_read_slowpath()
1163 	long count;  in rwsem_down_write_slowpath()  local
1207 		count = atomic_long_read(&sem->count);  in rwsem_down_write_slowpath()
1217 		if (count & RWSEM_WRITER_MASK)  in rwsem_down_write_slowpath()
1220 		rwsem_mark_wake(sem, (count & RWSEM_READER_MASK)  in rwsem_down_write_slowpath()
1235 		atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count);  in rwsem_down_write_slowpath()
1282 			count = atomic_long_read(&sem->count);  in rwsem_down_write_slowpath()
1283 			if (!(count & RWSEM_LOCK_MASK))  in rwsem_down_write_slowpath()
1316 		atomic_long_andnot(RWSEM_FLAG_HANDOFF,  &sem->count);  in rwsem_down_write_slowpath()
1319 		atomic_long_andnot(RWSEM_FLAG_WAITERS, &sem->count);  in rwsem_down_write_slowpath()
1333 static struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem, long count)  in rwsem_wake()  argument
1419 		if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,  in __down_read_trylock()
1436 	if (unlikely(!atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,  in __down_write()
1449 	if (unlikely(!atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,  in __down_write_killable()
1467 	if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,  in __down_write_trylock()
1488 	tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count);  in __up_read()
1515 	tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count);  in __up_write()
1537 		-RWSEM_WRITER_LOCKED+RWSEM_READER_BIAS, &sem->count);  in __downgrade_write()