476 static inline unsigned int timer_get_idx(struct timer_list *timer)  in timer_get_idx()  argument
478 	return (timer->flags & TIMER_ARRAYMASK) >> TIMER_ARRAYSHIFT;  in timer_get_idx()
481 static inline void timer_set_idx(struct timer_list *timer, unsigned int idx)  in timer_set_idx()  argument
483 	timer->flags = (timer->flags & ~TIMER_ARRAYMASK) |  in timer_set_idx()
537 static void enqueue_timer(struct timer_base *base, struct timer_list *timer,  in enqueue_timer()  argument
540 	hlist_add_head(&timer->entry, base->vectors + idx);  in enqueue_timer()
542 	timer_set_idx(timer, idx);  in enqueue_timer()
544 	trace_timer_start(timer, timer->expires, timer->flags);  in enqueue_timer()
548 __internal_add_timer(struct timer_base *base, struct timer_list *timer)  in __internal_add_timer()  argument
552 	idx = calc_wheel_index(timer->expires, base->clk);  in __internal_add_timer()
553 	enqueue_timer(base, timer, idx);  in __internal_add_timer()
557 trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer)  in trigger_dyntick_cpu()  argument
566 	if (timer->flags & TIMER_DEFERRABLE) {  in trigger_dyntick_cpu()
581 	if (time_after_eq(timer->expires, base->next_expiry))  in trigger_dyntick_cpu()
588 	base->next_expiry = timer->expires;  in trigger_dyntick_cpu()
593 internal_add_timer(struct timer_base *base, struct timer_list *timer)  in internal_add_timer()  argument
595 	__internal_add_timer(base, timer);  in internal_add_timer()
596 	trigger_dyntick_cpu(base, timer);  in internal_add_timer()
610 	struct timer_list *timer = addr;  in timer_is_static_object()  local
612 	return (timer->entry.pprev == NULL &&  in timer_is_static_object()
613 		timer->entry.next == TIMER_ENTRY_STATIC);  in timer_is_static_object()
622 	struct timer_list *timer = addr;  in timer_fixup_init()  local
626 		del_timer_sync(timer);  in timer_fixup_init()
627 		debug_object_init(timer, &timer_debug_descr);  in timer_fixup_init()
647 	struct timer_list *timer = addr;  in timer_fixup_activate()  local
651 		timer_setup(timer, stub_timer, 0);  in timer_fixup_activate()
668 	struct timer_list *timer = addr;  in timer_fixup_free()  local
672 		del_timer_sync(timer);  in timer_fixup_free()
673 		debug_object_free(timer, &timer_debug_descr);  in timer_fixup_free()
686 	struct timer_list *timer = addr;  in timer_fixup_assert_init()  local
690 		timer_setup(timer, stub_timer, 0);  in timer_fixup_assert_init()
707 static inline void debug_timer_init(struct timer_list *timer)  in debug_timer_init()  argument
709 	debug_object_init(timer, &timer_debug_descr);  in debug_timer_init()
712 static inline void debug_timer_activate(struct timer_list *timer)  in debug_timer_activate()  argument
714 	debug_object_activate(timer, &timer_debug_descr);  in debug_timer_activate()
717 static inline void debug_timer_deactivate(struct timer_list *timer)  in debug_timer_deactivate()  argument
719 	debug_object_deactivate(timer, &timer_debug_descr);  in debug_timer_deactivate()
722 static inline void debug_timer_free(struct timer_list *timer)  in debug_timer_free()  argument
724 	debug_object_free(timer, &timer_debug_descr);  in debug_timer_free()
727 static inline void debug_timer_assert_init(struct timer_list *timer)  in debug_timer_assert_init()  argument
729 	debug_object_assert_init(timer, &timer_debug_descr);  in debug_timer_assert_init()
732 static void do_init_timer(struct timer_list *timer,
737 void init_timer_on_stack_key(struct timer_list *timer,  in init_timer_on_stack_key()  argument
742 	debug_object_init_on_stack(timer, &timer_debug_descr);  in init_timer_on_stack_key()
743 	do_init_timer(timer, func, flags, name, key);  in init_timer_on_stack_key()
747 void destroy_timer_on_stack(struct timer_list *timer)  in destroy_timer_on_stack()  argument
749 	debug_object_free(timer, &timer_debug_descr);  in destroy_timer_on_stack()
754 static inline void debug_timer_init(struct timer_list *timer) { }  in debug_timer_init()  argument
755 static inline void debug_timer_activate(struct timer_list *timer) { }  in debug_timer_activate()  argument
756 static inline void debug_timer_deactivate(struct timer_list *timer) { }  in debug_timer_deactivate()  argument
757 static inline void debug_timer_assert_init(struct timer_list *timer) { }  in debug_timer_assert_init()  argument
760 static inline void debug_init(struct timer_list *timer)  in debug_init()  argument
762 	debug_timer_init(timer);  in debug_init()
763 	trace_timer_init(timer);  in debug_init()
766 static inline void debug_deactivate(struct timer_list *timer)  in debug_deactivate()  argument
768 	debug_timer_deactivate(timer);  in debug_deactivate()
769 	trace_timer_cancel(timer);  in debug_deactivate()
772 static inline void debug_assert_init(struct timer_list *timer)  in debug_assert_init()  argument
774 	debug_timer_assert_init(timer);  in debug_assert_init()
777 static void do_init_timer(struct timer_list *timer,  in do_init_timer()  argument
782 	timer->entry.pprev = NULL;  in do_init_timer()
783 	timer->function = func;  in do_init_timer()
784 	timer->flags = flags | raw_smp_processor_id();  in do_init_timer()
785 	lockdep_init_map(&timer->lockdep_map, name, key, 0);  in do_init_timer()
800 void init_timer_key(struct timer_list *timer,  in init_timer_key()  argument
804 	debug_init(timer);  in init_timer_key()
805 	do_init_timer(timer, func, flags, name, key);  in init_timer_key()
809 static inline void detach_timer(struct timer_list *timer, bool clear_pending)  in detach_timer()  argument
811 	struct hlist_node *entry = &timer->entry;  in detach_timer()
813 	debug_deactivate(timer);  in detach_timer()
821 static int detach_if_pending(struct timer_list *timer, struct timer_base *base,  in detach_if_pending()  argument
824 	unsigned idx = timer_get_idx(timer);  in detach_if_pending()
826 	if (!timer_pending(timer))  in detach_if_pending()
829 	if (hlist_is_singular_node(&timer->entry, base->vectors + idx))  in detach_if_pending()
832 	detach_timer(timer, clear_pending);  in detach_if_pending()
919 static struct timer_base *lock_timer_base(struct timer_list *timer,  in lock_timer_base()  argument
921 	__acquires(timer->base->lock)  in lock_timer_base()
932 		tf = READ_ONCE(timer->flags);  in lock_timer_base()
937 			if (timer->flags == tf)  in lock_timer_base()
949 __mod_timer(struct timer_list *timer, unsigned long expires, unsigned int options)  in __mod_timer()  argument
956 	BUG_ON(!timer->function);  in __mod_timer()
963 	if (timer_pending(timer)) {  in __mod_timer()
969 		long diff = timer->expires - expires;  in __mod_timer()
982 		base = lock_timer_base(timer, &flags);  in __mod_timer()
985 		if (timer_pending(timer) && (options & MOD_TIMER_REDUCE) &&  in __mod_timer()
986 		    time_before_eq(timer->expires, expires)) {  in __mod_timer()
999 		if (idx == timer_get_idx(timer)) {  in __mod_timer()
1001 				timer->expires = expires;  in __mod_timer()
1002 			else if (time_after(timer->expires, expires))  in __mod_timer()
1003 				timer->expires = expires;  in __mod_timer()
1008 		base = lock_timer_base(timer, &flags);  in __mod_timer()
1012 	ret = detach_if_pending(timer, base, false);  in __mod_timer()
1016 	new_base = get_target_base(base, timer->flags);  in __mod_timer()
1026 		if (likely(base->running_timer != timer)) {  in __mod_timer()
1028 			timer->flags |= TIMER_MIGRATING;  in __mod_timer()
1033 			WRITE_ONCE(timer->flags,  in __mod_timer()
1034 				   (timer->flags & ~TIMER_BASEMASK) | base->cpu);  in __mod_timer()
1039 	debug_timer_activate(timer);  in __mod_timer()
1041 	timer->expires = expires;  in __mod_timer()
1050 		enqueue_timer(base, timer, idx);  in __mod_timer()
1051 		trigger_dyntick_cpu(base, timer);  in __mod_timer()
1053 		internal_add_timer(base, timer);  in __mod_timer()
1072 int mod_timer_pending(struct timer_list *timer, unsigned long expires)  in mod_timer_pending()  argument
1074 	return __mod_timer(timer, expires, MOD_TIMER_PENDING_ONLY);  in mod_timer_pending()
1098 int mod_timer(struct timer_list *timer, unsigned long expires)  in mod_timer()  argument
1100 	return __mod_timer(timer, expires, 0);  in mod_timer()
1113 int timer_reduce(struct timer_list *timer, unsigned long expires)  in timer_reduce()  argument
1115 	return __mod_timer(timer, expires, MOD_TIMER_REDUCE);  in timer_reduce()
1133 void add_timer(struct timer_list *timer)  in add_timer()  argument
1135 	BUG_ON(timer_pending(timer));  in add_timer()
1136 	mod_timer(timer, timer->expires);  in add_timer()
1147 void add_timer_on(struct timer_list *timer, int cpu)  in add_timer_on()  argument
1152 	BUG_ON(timer_pending(timer) || !timer->function);  in add_timer_on()
1154 	new_base = get_timer_cpu_base(timer->flags, cpu);  in add_timer_on()
1161 	base = lock_timer_base(timer, &flags);  in add_timer_on()
1163 		timer->flags |= TIMER_MIGRATING;  in add_timer_on()
1168 		WRITE_ONCE(timer->flags,  in add_timer_on()
1169 			   (timer->flags & ~TIMER_BASEMASK) | cpu);  in add_timer_on()
1173 	debug_timer_activate(timer);  in add_timer_on()
1174 	internal_add_timer(base, timer);  in add_timer_on()
1190 int del_timer(struct timer_list *timer)  in del_timer()  argument
1196 	debug_assert_init(timer);  in del_timer()
1198 	if (timer_pending(timer)) {  in del_timer()
1199 		base = lock_timer_base(timer, &flags);  in del_timer()
1200 		ret = detach_if_pending(timer, base, true);  in del_timer()
1215 int try_to_del_timer_sync(struct timer_list *timer)  in try_to_del_timer_sync()  argument
1221 	debug_assert_init(timer);  in try_to_del_timer_sync()
1223 	base = lock_timer_base(timer, &flags);  in try_to_del_timer_sync()
1225 	if (base->running_timer != timer)  in try_to_del_timer_sync()
1226 		ret = detach_if_pending(timer, base, true);  in try_to_del_timer_sync()
1275 static void del_timer_wait_running(struct timer_list *timer)  in del_timer_wait_running()  argument
1279 	tf = READ_ONCE(timer->flags);  in del_timer_wait_running()
1302 static inline void del_timer_wait_running(struct timer_list *timer) { }  in del_timer_wait_running()  argument
1342 int del_timer_sync(struct timer_list *timer)  in del_timer_sync()  argument
1354 	lock_map_acquire(&timer->lockdep_map);  in del_timer_sync()
1355 	lock_map_release(&timer->lockdep_map);  in del_timer_sync()
1362 	WARN_ON(in_irq() && !(timer->flags & TIMER_IRQSAFE));  in del_timer_sync()
1365 		ret = try_to_del_timer_sync(timer);  in del_timer_sync()
1368 			del_timer_wait_running(timer);  in del_timer_sync()
1378 static void call_timer_fn(struct timer_list *timer,  in call_timer_fn()  argument
1394 	lockdep_copy_map(&lockdep_map, &timer->lockdep_map);  in call_timer_fn()
1403 	trace_timer_expire_entry(timer, baseclk);  in call_timer_fn()
1404 	fn(timer);  in call_timer_fn()
1405 	trace_timer_expire_exit(timer);  in call_timer_fn()
1432 		struct timer_list *timer;  in expire_timers()  local
1435 		timer = hlist_entry(head->first, struct timer_list, entry);  in expire_timers()
1437 		base->running_timer = timer;  in expire_timers()
1438 		detach_timer(timer, true);  in expire_timers()
1440 		fn = timer->function;  in expire_timers()
1442 		if (timer->flags & TIMER_IRQSAFE) {  in expire_timers()
1444 			call_timer_fn(timer, fn, baseclk);  in expire_timers()
1449 			call_timer_fn(timer, fn, baseclk);  in expire_timers()
1816 	struct timer_list timer;  member
1822 	struct process_timer *timeout = from_timer(timeout, t, timer);  in process_timeout()
1858 	struct process_timer timer;  in schedule_timeout()  local
1892 	timer.task = current;  in schedule_timeout()
1893 	timer_setup_on_stack(&timer.timer, process_timeout, 0);  in schedule_timeout()
1894 	__mod_timer(&timer.timer, expire, 0);  in schedule_timeout()
1896 	del_singleshot_timer_sync(&timer.timer);  in schedule_timeout()
1899 	destroy_timer_on_stack(&timer.timer);  in schedule_timeout()
1947 	struct timer_list *timer;  in migrate_timer_list()  local
1951 		timer = hlist_entry(head->first, struct timer_list, entry);  in migrate_timer_list()
1952 		detach_timer(timer, false);  in migrate_timer_list()
1953 		timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu;  in migrate_timer_list()
1954 		internal_add_timer(new_base, timer);  in migrate_timer_list()