174 struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,  in lock_hrtimer_base()  argument
180 		base = timer->base;  in lock_hrtimer_base()
183 			if (likely(base == timer->base))  in lock_hrtimer_base()
196 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base)  in switch_hrtimer_base()  argument
214 		if (unlikely(hrtimer_callback_running(timer)))  in switch_hrtimer_base()
218 		timer->base = NULL;  in switch_hrtimer_base()
221 		timer->base = new_base;  in switch_hrtimer_base()
229 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)  in lock_hrtimer_base()  argument
231 	struct hrtimer_clock_base *base = timer->base;  in lock_hrtimer_base()
345 	struct hrtimer *timer = addr;  in hrtimer_fixup_init()  local
349 		hrtimer_cancel(timer);  in hrtimer_fixup_init()
350 		debug_object_init(timer, &hrtimer_debug_descr);  in hrtimer_fixup_init()
384 	struct hrtimer *timer = addr;  in hrtimer_fixup_free()  local
388 		hrtimer_cancel(timer);  in hrtimer_fixup_free()
389 		debug_object_free(timer, &hrtimer_debug_descr);  in hrtimer_fixup_free()
403 static inline void debug_hrtimer_init(struct hrtimer *timer)  in debug_hrtimer_init()  argument
405 	debug_object_init(timer, &hrtimer_debug_descr);  in debug_hrtimer_init()
408 static inline void debug_hrtimer_activate(struct hrtimer *timer)  in debug_hrtimer_activate()  argument
410 	debug_object_activate(timer, &hrtimer_debug_descr);  in debug_hrtimer_activate()
413 static inline void debug_hrtimer_deactivate(struct hrtimer *timer)  in debug_hrtimer_deactivate()  argument
415 	debug_object_deactivate(timer, &hrtimer_debug_descr);  in debug_hrtimer_deactivate()
418 static inline void debug_hrtimer_free(struct hrtimer *timer)  in debug_hrtimer_free()  argument
420 	debug_object_free(timer, &hrtimer_debug_descr);  in debug_hrtimer_free()
423 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
426 void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,  in hrtimer_init_on_stack()  argument
429 	debug_object_init_on_stack(timer, &hrtimer_debug_descr);  in hrtimer_init_on_stack()
430 	__hrtimer_init(timer, clock_id, mode);  in hrtimer_init_on_stack()
433 void destroy_hrtimer_on_stack(struct hrtimer *timer)  in destroy_hrtimer_on_stack()  argument
435 	debug_object_free(timer, &hrtimer_debug_descr);  in destroy_hrtimer_on_stack()
439 static inline void debug_hrtimer_init(struct hrtimer *timer) { }  in debug_hrtimer_init()  argument
440 static inline void debug_hrtimer_activate(struct hrtimer *timer) { }  in debug_hrtimer_activate()  argument
441 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }  in debug_hrtimer_deactivate()  argument
498 		struct hrtimer *timer;  in hrtimer_force_reprogram()  local
502 		timer = rb_entry(base->first, struct hrtimer, node);  in hrtimer_force_reprogram()
503 		expires = ktime_sub(hrtimer_get_expires(timer), base->offset);  in hrtimer_force_reprogram()
528 static int hrtimer_reprogram(struct hrtimer *timer,  in hrtimer_reprogram()  argument
532 	ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);  in hrtimer_reprogram()
535 	WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);  in hrtimer_reprogram()
544 	if (hrtimer_callback_running(timer))  in hrtimer_reprogram()
642 static inline void hrtimer_init_timer_hres(struct hrtimer *timer)  in hrtimer_init_timer_hres()  argument
653 static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,  in hrtimer_enqueue_reprogram()  argument
656 	if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) {  in hrtimer_enqueue_reprogram()
705 static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,  in hrtimer_enqueue_reprogram()  argument
711 static inline void hrtimer_init_timer_hres(struct hrtimer *timer) { }  in hrtimer_init_timer_hres()  argument
716 void __timer_stats_hrtimer_set_start_info(struct hrtimer *timer, void *addr)  in __timer_stats_hrtimer_set_start_info()  argument
718 	if (timer->start_site)  in __timer_stats_hrtimer_set_start_info()
721 	timer->start_site = addr;  in __timer_stats_hrtimer_set_start_info()
722 	memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);  in __timer_stats_hrtimer_set_start_info()
723 	timer->start_pid = current->pid;  in __timer_stats_hrtimer_set_start_info()
731 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)  in unlock_hrtimer_base()  argument
733 	spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
745 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)  in hrtimer_forward()  argument
750 	delta = ktime_sub(now, hrtimer_get_expires(timer));  in hrtimer_forward()
755 	if (interval.tv64 < timer->base->resolution.tv64)  in hrtimer_forward()
756 		interval.tv64 = timer->base->resolution.tv64;  in hrtimer_forward()
762 		hrtimer_add_expires_ns(timer, incr * orun);  in hrtimer_forward()
763 		if (hrtimer_get_expires_tv64(timer) > now.tv64)  in hrtimer_forward()
771 	hrtimer_add_expires(timer, interval);  in hrtimer_forward()
785 static int enqueue_hrtimer(struct hrtimer *timer,  in enqueue_hrtimer()  argument
793 	debug_hrtimer_activate(timer);  in enqueue_hrtimer()
805 		if (hrtimer_get_expires_tv64(timer) <  in enqueue_hrtimer()
819 		base->first = &timer->node;  in enqueue_hrtimer()
821 	rb_link_node(&timer->node, parent, link);  in enqueue_hrtimer()
822 	rb_insert_color(&timer->node, &base->active);  in enqueue_hrtimer()
827 	timer->state |= HRTIMER_STATE_ENQUEUED;  in enqueue_hrtimer()
842 static void __remove_hrtimer(struct hrtimer *timer,  in __remove_hrtimer()  argument
846 	if (timer->state & HRTIMER_STATE_ENQUEUED) {  in __remove_hrtimer()
851 		if (base->first == &timer->node) {  in __remove_hrtimer()
852 			base->first = rb_next(&timer->node);  in __remove_hrtimer()
857 		rb_erase(&timer->node, &base->active);  in __remove_hrtimer()
859 	timer->state = newstate;  in __remove_hrtimer()
866 remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base)  in remove_hrtimer()  argument
868 	if (hrtimer_is_queued(timer)) {  in remove_hrtimer()
879 		debug_hrtimer_deactivate(timer);  in remove_hrtimer()
880 		timer_stats_hrtimer_clear_start_info(timer);  in remove_hrtimer()
882 		__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE,  in remove_hrtimer()
901 hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_ns,  in hrtimer_start_range_ns()  argument
908 	base = lock_hrtimer_base(timer, &flags);  in hrtimer_start_range_ns()
911 	ret = remove_hrtimer(timer, base);  in hrtimer_start_range_ns()
914 	new_base = switch_hrtimer_base(timer, base);  in hrtimer_start_range_ns()
930 	hrtimer_set_expires_range_ns(timer, tim, delta_ns);  in hrtimer_start_range_ns()
932 	timer_stats_hrtimer_set_start_info(timer);  in hrtimer_start_range_ns()
934 	leftmost = enqueue_hrtimer(timer, new_base);  in hrtimer_start_range_ns()
943 		hrtimer_enqueue_reprogram(timer, new_base);  in hrtimer_start_range_ns()
945 	unlock_hrtimer_base(timer, &flags);  in hrtimer_start_range_ns()
962 hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)  in hrtimer_start()  argument
964 	return hrtimer_start_range_ns(timer, tim, 0, mode);  in hrtimer_start()
979 int hrtimer_try_to_cancel(struct hrtimer *timer)  in hrtimer_try_to_cancel()  argument
985 	base = lock_hrtimer_base(timer, &flags);  in hrtimer_try_to_cancel()
987 	if (!hrtimer_callback_running(timer))  in hrtimer_try_to_cancel()
988 		ret = remove_hrtimer(timer, base);  in hrtimer_try_to_cancel()
990 	unlock_hrtimer_base(timer, &flags);  in hrtimer_try_to_cancel()
1005 int hrtimer_cancel(struct hrtimer *timer)  in hrtimer_cancel()  argument
1008 		int ret = hrtimer_try_to_cancel(timer);  in hrtimer_cancel()
1021 ktime_t hrtimer_get_remaining(const struct hrtimer *timer)  in hrtimer_get_remaining()  argument
1027 	base = lock_hrtimer_base(timer, &flags);  in hrtimer_get_remaining()
1028 	rem = hrtimer_expires_remaining(timer);  in hrtimer_get_remaining()
1029 	unlock_hrtimer_base(timer, &flags);  in hrtimer_get_remaining()
1054 			struct hrtimer *timer;  in hrtimer_get_next_event()  local
1059 			timer = rb_entry(base->first, struct hrtimer, node);  in hrtimer_get_next_event()
1060 			delta.tv64 = hrtimer_get_expires_tv64(timer);  in hrtimer_get_next_event()
1075 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,  in __hrtimer_init()  argument
1080 	memset(timer, 0, sizeof(struct hrtimer));  in __hrtimer_init()
1087 	timer->base = &cpu_base->clock_base[clock_id];  in __hrtimer_init()
1088 	INIT_LIST_HEAD(&timer->cb_entry);  in __hrtimer_init()
1089 	hrtimer_init_timer_hres(timer);  in __hrtimer_init()
1092 	timer->start_site = NULL;  in __hrtimer_init()
1093 	timer->start_pid = -1;  in __hrtimer_init()
1094 	memset(timer->start_comm, 0, TASK_COMM_LEN);  in __hrtimer_init()
1104 void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,  in hrtimer_init()  argument
1107 	debug_hrtimer_init(timer);  in hrtimer_init()
1108 	__hrtimer_init(timer, clock_id, mode);  in hrtimer_init()
1131 static void __run_hrtimer(struct hrtimer *timer)  in __run_hrtimer()  argument
1133 	struct hrtimer_clock_base *base = timer->base;  in __run_hrtimer()
1140 	debug_hrtimer_deactivate(timer);  in __run_hrtimer()
1141 	__remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);  in __run_hrtimer()
1142 	timer_stats_account_hrtimer(timer);  in __run_hrtimer()
1143 	fn = timer->function;  in __run_hrtimer()
1151 	restart = fn(timer);  in __run_hrtimer()
1160 		BUG_ON(timer->state != HRTIMER_STATE_CALLBACK);  in __run_hrtimer()
1161 		enqueue_hrtimer(timer, base);  in __run_hrtimer()
1163 	timer->state &= ~HRTIMER_STATE_CALLBACK;  in __run_hrtimer()
1227 			struct hrtimer *timer;  in hrtimer_interrupt()  local
1229 			timer = rb_entry(node, struct hrtimer, node);  in hrtimer_interrupt()
1244 			if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer)) {  in hrtimer_interrupt()
1247 				expires = ktime_sub(hrtimer_get_expires(timer),  in hrtimer_interrupt()
1254 			__run_hrtimer(timer);  in hrtimer_interrupt()
1365 			struct hrtimer *timer;  in hrtimer_run_queues()  local
1367 			timer = rb_entry(node, struct hrtimer, node);  in hrtimer_run_queues()
1369 					hrtimer_get_expires_tv64(timer))  in hrtimer_run_queues()
1372 			__run_hrtimer(timer);  in hrtimer_run_queues()
1381 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)  in hrtimer_wakeup()  argument
1384 		container_of(timer, struct hrtimer_sleeper, timer);  in hrtimer_wakeup()
1396 	sl->timer.function = hrtimer_wakeup;  in hrtimer_init_sleeper()
1406 		hrtimer_start_expires(&t->timer, mode);  in do_nanosleep()
1407 		if (!hrtimer_active(&t->timer))  in do_nanosleep()
1413 		hrtimer_cancel(&t->timer);  in do_nanosleep()
1423 static int update_rmtp(struct hrtimer *timer, struct timespec __user *rmtp)  in update_rmtp()  argument
1428 	rem = hrtimer_expires_remaining(timer);  in update_rmtp()
1445 	hrtimer_init_on_stack(&t.timer, restart->nanosleep.index,  in hrtimer_nanosleep_restart()
1447 	hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);  in hrtimer_nanosleep_restart()
1454 		ret = update_rmtp(&t.timer, rmtp);  in hrtimer_nanosleep_restart()
1462 	destroy_hrtimer_on_stack(&t.timer);  in hrtimer_nanosleep_restart()
1478 	hrtimer_init_on_stack(&t.timer, clockid, mode);  in hrtimer_nanosleep()
1479 	hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack);  in hrtimer_nanosleep()
1490 		ret = update_rmtp(&t.timer, rmtp);  in hrtimer_nanosleep()
1497 	restart->nanosleep.index = t.timer.base->index;  in hrtimer_nanosleep()
1499 	restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);  in hrtimer_nanosleep()
1503 	destroy_hrtimer_on_stack(&t.timer);  in hrtimer_nanosleep()
1542 	struct hrtimer *timer;  in migrate_hrtimer_list()  local
1546 		timer = rb_entry(node, struct hrtimer, node);  in migrate_hrtimer_list()
1547 		BUG_ON(hrtimer_callback_running(timer));  in migrate_hrtimer_list()
1548 		debug_hrtimer_deactivate(timer);  in migrate_hrtimer_list()
1555 		__remove_hrtimer(timer, old_base, HRTIMER_STATE_MIGRATE, 0);  in migrate_hrtimer_list()
1556 		timer->base = new_base;  in migrate_hrtimer_list()
1565 		enqueue_hrtimer(timer, new_base);  in migrate_hrtimer_list()
1568 		timer->state &= ~HRTIMER_STATE_MIGRATE;  in migrate_hrtimer_list()
1703 	hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, mode);  in schedule_hrtimeout_range()
1704 	hrtimer_set_expires_range_ns(&t.timer, *expires, delta);  in schedule_hrtimeout_range()
1708 	hrtimer_start_expires(&t.timer, mode);  in schedule_hrtimeout_range()
1709 	if (!hrtimer_active(&t.timer))  in schedule_hrtimeout_range()
1715 	hrtimer_cancel(&t.timer);  in schedule_hrtimeout_range()
1716 	destroy_hrtimer_on_stack(&t.timer);  in schedule_hrtimeout_range()