21 static void posix_cpu_timer_rearm(struct k_itimer *timer);
106 static inline struct task_struct *cpu_timer_task_rcu(struct k_itimer *timer)  in cpu_timer_task_rcu()  argument
108 	return pid_task(timer->it.cpu.pid, clock_pid_type(timer->it_clock));  in cpu_timer_task_rcu()
115 static u64 bump_cpu_timer(struct k_itimer *timer, u64 now)  in bump_cpu_timer()  argument
117 	u64 delta, incr, expires = timer->it.cpu.node.expires;  in bump_cpu_timer()
120 	if (!timer->it_interval)  in bump_cpu_timer()
126 	incr = timer->it_interval;  in bump_cpu_timer()
137 		timer->it.cpu.node.expires += incr;  in bump_cpu_timer()
138 		timer->it_overrun += 1LL << i;  in bump_cpu_timer()
141 	return timer->it.cpu.node.expires;  in bump_cpu_timer()
414 static int posix_cpu_timer_del(struct k_itimer *timer)  in posix_cpu_timer_del()  argument
416 	struct cpu_timer *ctmr = &timer->it.cpu;  in posix_cpu_timer_del()
423 	p = cpu_timer_task_rcu(timer);  in posix_cpu_timer_del()
439 		if (timer->it.cpu.firing)  in posix_cpu_timer_del()
499 static void arm_timer(struct k_itimer *timer, struct task_struct *p)  in arm_timer()  argument
501 	int clkidx = CPUCLOCK_WHICH(timer->it_clock);  in arm_timer()
502 	struct cpu_timer *ctmr = &timer->it.cpu;  in arm_timer()
506 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in arm_timer()
523 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in arm_timer()
532 static void cpu_timer_fire(struct k_itimer *timer)  in cpu_timer_fire()  argument
534 	struct cpu_timer *ctmr = &timer->it.cpu;  in cpu_timer_fire()
536 	if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) {  in cpu_timer_fire()
541 	} else if (unlikely(timer->sigq == NULL)) {  in cpu_timer_fire()
546 		wake_up_process(timer->it_process);  in cpu_timer_fire()
548 	} else if (!timer->it_interval) {  in cpu_timer_fire()
552 		posix_timer_event(timer, 0);  in cpu_timer_fire()
554 	} else if (posix_timer_event(timer, ++timer->it_requeue_pending)) {  in cpu_timer_fire()
561 		posix_cpu_timer_rearm(timer);  in cpu_timer_fire()
562 		++timer->it_requeue_pending;  in cpu_timer_fire()
572 static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,  in posix_cpu_timer_set()  argument
575 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);  in posix_cpu_timer_set()
577 	struct cpu_timer *ctmr = &timer->it.cpu;  in posix_cpu_timer_set()
584 	p = cpu_timer_task_rcu(timer);  in posix_cpu_timer_set()
617 	old_incr = timer->it_interval;  in posix_cpu_timer_set()
620 	if (unlikely(timer->it.cpu.firing)) {  in posix_cpu_timer_set()
621 		timer->it.cpu.firing = -1;  in posix_cpu_timer_set()
635 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in posix_cpu_timer_set()
652 			u64 exp = bump_cpu_timer(timer, val);  in posix_cpu_timer_set()
686 		arm_timer(timer, p);  in posix_cpu_timer_set()
694 	timer->it_interval = timespec64_to_ktime(new->it_interval);  in posix_cpu_timer_set()
701 	timer->it_requeue_pending = (timer->it_requeue_pending + 2) &  in posix_cpu_timer_set()
703 	timer->it_overrun_last = 0;  in posix_cpu_timer_set()
704 	timer->it_overrun = -1;  in posix_cpu_timer_set()
712 		cpu_timer_fire(timer);  in posix_cpu_timer_set()
724 static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp)  in posix_cpu_timer_get()  argument
726 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);  in posix_cpu_timer_get()
727 	struct cpu_timer *ctmr = &timer->it.cpu;  in posix_cpu_timer_get()
732 	p = cpu_timer_task_rcu(timer);  in posix_cpu_timer_get()
739 	itp->it_interval = ktime_to_timespec64(timer->it_interval);  in posix_cpu_timer_get()
747 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in posix_cpu_timer_get()
983 static void posix_cpu_timer_rearm(struct k_itimer *timer)  in posix_cpu_timer_rearm()  argument
985 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);  in posix_cpu_timer_rearm()
992 	p = cpu_timer_task_rcu(timer);  in posix_cpu_timer_rearm()
1004 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in posix_cpu_timer_rearm()
1009 	bump_cpu_timer(timer, now);  in posix_cpu_timer_rearm()
1014 	arm_timer(timer, p);  in posix_cpu_timer_rearm()
1265 	struct k_itimer *timer, *next;  in handle_posix_cpu_timers()  local
1338 	list_for_each_entry_safe(timer, next, &firing, it.cpu.elist) {  in handle_posix_cpu_timers()
1348 		spin_lock(&timer->it_lock);  in handle_posix_cpu_timers()
1349 		list_del_init(&timer->it.cpu.elist);  in handle_posix_cpu_timers()
1350 		cpu_firing = timer->it.cpu.firing;  in handle_posix_cpu_timers()
1351 		timer->it.cpu.firing = 0;  in handle_posix_cpu_timers()
1358 			cpu_timer_fire(timer);  in handle_posix_cpu_timers()
1360 		rcu_assign_pointer(timer->it.cpu.handling, NULL);  in handle_posix_cpu_timers()
1361 		spin_unlock(&timer->it_lock);  in handle_posix_cpu_timers()
1442 	struct k_itimer timer;  in do_cpu_nanosleep()  local
1449 	memset(&timer, 0, sizeof timer);  in do_cpu_nanosleep()
1450 	spin_lock_init(&timer.it_lock);  in do_cpu_nanosleep()
1451 	timer.it_clock = which_clock;  in do_cpu_nanosleep()
1452 	timer.it_overrun = -1;  in do_cpu_nanosleep()
1453 	error = posix_cpu_timer_create(&timer);  in do_cpu_nanosleep()
1454 	timer.it_process = current;  in do_cpu_nanosleep()
1463 		spin_lock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1464 		error = posix_cpu_timer_set(&timer, flags, &it, NULL);  in do_cpu_nanosleep()
1466 			spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1471 			if (!cpu_timer_getexpires(&timer.it.cpu)) {  in do_cpu_nanosleep()
1476 				posix_cpu_timer_del(&timer);  in do_cpu_nanosleep()
1477 				spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1485 			spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1487 			spin_lock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1493 		expires = cpu_timer_getexpires(&timer.it.cpu);  in do_cpu_nanosleep()
1494 		error = posix_cpu_timer_set(&timer, 0, &zero_it, &it);  in do_cpu_nanosleep()
1497 			posix_cpu_timer_del(&timer);  in do_cpu_nanosleep()
1500 				posix_cpu_timer_wait_running_nsleep(&timer);  in do_cpu_nanosleep()
1501 				error = posix_cpu_timer_del(&timer);  in do_cpu_nanosleep()
1505 		spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1579 static int process_cpu_timer_create(struct k_itimer *timer)  in process_cpu_timer_create()  argument
1581 	timer->it_clock = PROCESS_CLOCK;  in process_cpu_timer_create()
1582 	return posix_cpu_timer_create(timer);  in process_cpu_timer_create()
1599 static int thread_cpu_timer_create(struct k_itimer *timer)  in thread_cpu_timer_create()  argument
1601 	timer->it_clock = THREAD_CLOCK;  in thread_cpu_timer_create()
1602 	return posix_cpu_timer_create(timer);  in thread_cpu_timer_create()