37 		unsigned long settime, struct percpu_rw_semaphore *sem)  in _trace_android_vh_record_pcpu_rwsem_time_early()  argument
39 	trace_android_vh_record_pcpu_rwsem_time_early(settime, sem);  in _trace_android_vh_record_pcpu_rwsem_time_early()
43 int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,  in __percpu_init_rwsem()  argument
46 	sem->read_count = alloc_percpu(int);  in __percpu_init_rwsem()
47 	if (unlikely(!sem->read_count))  in __percpu_init_rwsem()
50 	rcu_sync_init(&sem->rss);  in __percpu_init_rwsem()
51 	rcuwait_init(&sem->writer);  in __percpu_init_rwsem()
52 	init_waitqueue_head(&sem->waiters);  in __percpu_init_rwsem()
53 	atomic_set(&sem->block, 0);  in __percpu_init_rwsem()
55 	debug_check_no_locks_freed((void *)sem, sizeof(*sem));  in __percpu_init_rwsem()
56 	lockdep_init_map(&sem->dep_map, name, key, 0);  in __percpu_init_rwsem()
62 void percpu_free_rwsem(struct percpu_rw_semaphore *sem)  in percpu_free_rwsem()  argument
68 	if (!sem->read_count)  in percpu_free_rwsem()
71 	rcu_sync_dtor(&sem->rss);  in percpu_free_rwsem()
72 	free_percpu(sem->read_count);  in percpu_free_rwsem()
73 	sem->read_count = NULL; /* catch use after free bugs */  in percpu_free_rwsem()
77 static bool __percpu_down_read_trylock(struct percpu_rw_semaphore *sem)  in __percpu_down_read_trylock()  argument
79 	this_cpu_inc(*sem->read_count);  in __percpu_down_read_trylock()
102 	if (likely(!atomic_read_acquire(&sem->block)))  in __percpu_down_read_trylock()
105 	this_cpu_dec(*sem->read_count);  in __percpu_down_read_trylock()
108 	rcuwait_wake_up(&sem->writer);  in __percpu_down_read_trylock()
113 static inline bool __percpu_down_write_trylock(struct percpu_rw_semaphore *sem)  in __percpu_down_write_trylock()  argument
115 	if (atomic_read(&sem->block))  in __percpu_down_write_trylock()
118 	return atomic_xchg(&sem->block, 1) == 0;  in __percpu_down_write_trylock()
121 static bool __percpu_rwsem_trylock(struct percpu_rw_semaphore *sem, bool reader)  in __percpu_rwsem_trylock()  argument
127 		ret = __percpu_down_read_trylock(sem);  in __percpu_rwsem_trylock()
132 	return __percpu_down_write_trylock(sem);  in __percpu_rwsem_trylock()
153 	struct percpu_rw_semaphore *sem = key;  in percpu_rwsem_wake_function()  local
157 	if (!__percpu_rwsem_trylock(sem, reader))  in percpu_rwsem_wake_function()
170 static void percpu_rwsem_wait(struct percpu_rw_semaphore *sem, bool reader)  in percpu_rwsem_wait()  argument
175 	spin_lock_irq(&sem->waiters.lock);  in percpu_rwsem_wait()
180 	wait = !__percpu_rwsem_trylock(sem, reader);  in percpu_rwsem_wait()
183 		__add_wait_queue_entry_tail(&sem->waiters, &wq_entry);  in percpu_rwsem_wait()
184 		trace_android_vh_percpu_rwsem_wq_add(sem, reader);  in percpu_rwsem_wait()
186 	spin_unlock_irq(&sem->waiters.lock);  in percpu_rwsem_wait()
197 bool __sched __percpu_down_read(struct percpu_rw_semaphore *sem, bool try)  in __percpu_down_read()  argument
199 	if (__percpu_down_read_trylock(sem))  in __percpu_down_read()
205 	trace_contention_begin(sem, LCB_F_PERCPU | LCB_F_READ);  in __percpu_down_read()
207 	percpu_rwsem_wait(sem, /* .reader = */ true);  in __percpu_down_read()
209 	trace_contention_end(sem, 0);  in __percpu_down_read()
225 bool percpu_is_read_locked(struct percpu_rw_semaphore *sem)  in percpu_is_read_locked()  argument
227 	return per_cpu_sum(*sem->read_count) != 0 && !atomic_read(&sem->block);  in percpu_is_read_locked()
239 static bool readers_active_check(struct percpu_rw_semaphore *sem)  in readers_active_check()  argument
241 	if (per_cpu_sum(*sem->read_count) != 0)  in readers_active_check()
254 void __sched percpu_down_write(struct percpu_rw_semaphore *sem)  in percpu_down_write()  argument
257 	rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);  in percpu_down_write()
258 	trace_contention_begin(sem, LCB_F_PERCPU | LCB_F_WRITE);  in percpu_down_write()
260 	trace_android_vh_record_pcpu_rwsem_time_early(jiffies, sem);  in percpu_down_write()
263 	rcu_sync_enter(&sem->rss);  in percpu_down_write()
269 	if (!__percpu_down_write_trylock(sem))  in percpu_down_write()
270 		percpu_rwsem_wait(sem, /* .reader = */ false);  in percpu_down_write()
281 	rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE);  in percpu_down_write()
282 	trace_contention_end(sem, 0);  in percpu_down_write()
287 void percpu_up_write(struct percpu_rw_semaphore *sem)  in percpu_up_write()  argument
289 	rwsem_release(&sem->dep_map, _RET_IP_);  in percpu_up_write()
301 	atomic_set_release(&sem->block, 0);  in percpu_up_write()
306 	__wake_up(&sem->waiters, TASK_NORMAL, 1, sem);  in percpu_up_write()
313 	rcu_sync_exit(&sem->rss);  in percpu_up_write()
314 	trace_android_vh_record_pcpu_rwsem_time_early(0, sem);  in percpu_up_write()