140 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in tick_nohz_update_jiffies()  local
144 	if (!ts->tick_stopped)  in tick_nohz_update_jiffies()
149 	ts->idle_waketime = now;  in tick_nohz_update_jiffies()
160 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in tick_nohz_stop_idle()  local
162 	if (ts->idle_active) {  in tick_nohz_stop_idle()
165 		delta = ktime_sub(now, ts->idle_entrytime);  in tick_nohz_stop_idle()
166 		ts->idle_lastupdate = now;  in tick_nohz_stop_idle()
167 		ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);  in tick_nohz_stop_idle()
168 		ts->idle_active = 0;  in tick_nohz_stop_idle()
174 static ktime_t tick_nohz_start_idle(struct tick_sched *ts)  in tick_nohz_start_idle()  argument
179 	if (ts->idle_active) {  in tick_nohz_start_idle()
180 		delta = ktime_sub(now, ts->idle_entrytime);  in tick_nohz_start_idle()
181 		ts->idle_lastupdate = now;  in tick_nohz_start_idle()
182 		ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);  in tick_nohz_start_idle()
184 	ts->idle_entrytime = now;  in tick_nohz_start_idle()
185 	ts->idle_active = 1;  in tick_nohz_start_idle()
192 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in get_cpu_idle_time_us()  local
197 	if (ts->idle_active)  in get_cpu_idle_time_us()
198 		*last_update_time = ktime_to_us(ts->idle_lastupdate);  in get_cpu_idle_time_us()
202 	return ktime_to_us(ts->idle_sleeptime);  in get_cpu_idle_time_us()
216 	struct tick_sched *ts;  in tick_nohz_stop_sched_tick()  local
224 	ts = &per_cpu(tick_cpu_sched, cpu);  in tick_nohz_stop_sched_tick()
238 	if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))  in tick_nohz_stop_sched_tick()
241 	if (!inidle && !ts->inidle)  in tick_nohz_stop_sched_tick()
244 	now = tick_nohz_start_idle(ts);  in tick_nohz_stop_sched_tick()
245 	ts->inidle = 1;  in tick_nohz_stop_sched_tick()
261 	ts->idle_calls++;  in tick_nohz_stop_sched_tick()
279 	if (!ts->tick_stopped && delta_jiffies == 1)  in tick_nohz_stop_sched_tick()
307 		if (ts->tick_stopped && ktime_equal(expires, dev->next_event))  in tick_nohz_stop_sched_tick()
317 		if (!ts->tick_stopped) {  in tick_nohz_stop_sched_tick()
326 			ts->idle_tick = hrtimer_get_expires(&ts->sched_timer);  in tick_nohz_stop_sched_tick()
327 			ts->tick_stopped = 1;  in tick_nohz_stop_sched_tick()
328 			ts->idle_jiffies = last_jiffies;  in tick_nohz_stop_sched_tick()
332 		ts->idle_sleeps++;  in tick_nohz_stop_sched_tick()
341 			ts->idle_expires.tv64 = KTIME_MAX;  in tick_nohz_stop_sched_tick()
342 			if (ts->nohz_mode == NOHZ_MODE_HIGHRES)  in tick_nohz_stop_sched_tick()
343 				hrtimer_cancel(&ts->sched_timer);  in tick_nohz_stop_sched_tick()
348 		ts->idle_expires = expires;  in tick_nohz_stop_sched_tick()
350 		if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {  in tick_nohz_stop_sched_tick()
351 			hrtimer_start(&ts->sched_timer, expires,  in tick_nohz_stop_sched_tick()
354 			if (hrtimer_active(&ts->sched_timer))  in tick_nohz_stop_sched_tick()
368 	ts->next_jiffies = next_jiffies;  in tick_nohz_stop_sched_tick()
369 	ts->last_jiffies = last_jiffies;  in tick_nohz_stop_sched_tick()
370 	ts->sleep_length = ktime_sub(dev->next_event, now);  in tick_nohz_stop_sched_tick()
382 	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);  in tick_nohz_get_sleep_length()  local
384 	return ts->sleep_length;  in tick_nohz_get_sleep_length()
387 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart()  argument
389 	hrtimer_cancel(&ts->sched_timer);  in tick_nohz_restart()
390 	hrtimer_set_expires(&ts->sched_timer, ts->idle_tick);  in tick_nohz_restart()
394 		hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_nohz_restart()
396 		if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {  in tick_nohz_restart()
397 			hrtimer_start_expires(&ts->sched_timer,  in tick_nohz_restart()
400 			if (hrtimer_active(&ts->sched_timer))  in tick_nohz_restart()
404 				hrtimer_get_expires(&ts->sched_timer), 0))  in tick_nohz_restart()
421 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in tick_nohz_restart_sched_tick()  local
430 	if (!ts->inidle || !ts->tick_stopped) {  in tick_nohz_restart_sched_tick()
431 		ts->inidle = 0;  in tick_nohz_restart_sched_tick()
436 	ts->inidle = 0;  in tick_nohz_restart_sched_tick()
452 	ticks = jiffies - ts->idle_jiffies;  in tick_nohz_restart_sched_tick()
464 	ts->tick_stopped  = 0;  in tick_nohz_restart_sched_tick()
465 	ts->idle_exittime = now;  in tick_nohz_restart_sched_tick()
467 	tick_nohz_restart(ts, now);  in tick_nohz_restart_sched_tick()
472 static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)  in tick_nohz_reprogram()  argument
474 	hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_nohz_reprogram()
475 	return tick_program_event(hrtimer_get_expires(&ts->sched_timer), 0);  in tick_nohz_reprogram()
483 	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);  in tick_nohz_handler()  local
512 	if (ts->tick_stopped) {  in tick_nohz_handler()
514 		ts->idle_jiffies++;  in tick_nohz_handler()
520 	while (tick_nohz_reprogram(ts, now)) {  in tick_nohz_handler()
531 	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);  in tick_nohz_switch_to_nohz()  local
543 	ts->nohz_mode = NOHZ_MODE_LOWRES;  in tick_nohz_switch_to_nohz()
549 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);  in tick_nohz_switch_to_nohz()
554 		hrtimer_set_expires(&ts->sched_timer, next);  in tick_nohz_switch_to_nohz()
581 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in tick_nohz_kick_tick()
584 	if (!ts->tick_stopped)  in tick_nohz_kick_tick()
592 	delta =	ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);  in tick_nohz_kick_tick()
596 	tick_nohz_restart(ts, now);  in tick_nohz_kick_tick()
629 	struct tick_sched *ts =  in tick_sched_timer()  local
664 		if (ts->tick_stopped) {  in tick_sched_timer()
666 			ts->idle_jiffies++;  in tick_sched_timer()
682 	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);  in tick_setup_sched_timer()  local
689 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);  in tick_setup_sched_timer()
690 	ts->sched_timer.function = tick_sched_timer;  in tick_setup_sched_timer()
693 	hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());  in tick_setup_sched_timer()
697 	hrtimer_add_expires_ns(&ts->sched_timer, offset);  in tick_setup_sched_timer()
700 		hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_setup_sched_timer()
701 		hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS);  in tick_setup_sched_timer()
703 		if (hrtimer_active(&ts->sched_timer))  in tick_setup_sched_timer()
710 		ts->nohz_mode = NOHZ_MODE_HIGHRES;  in tick_setup_sched_timer()
718 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in tick_cancel_sched_timer()  local
721 	if (ts->sched_timer.base)  in tick_cancel_sched_timer()
722 		hrtimer_cancel(&ts->sched_timer);  in tick_cancel_sched_timer()
725 	ts->nohz_mode = NOHZ_MODE_INACTIVE;  in tick_cancel_sched_timer()
745 	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);  in tick_oneshot_notify()  local
747 	set_bit(0, &ts->check_clocks);  in tick_oneshot_notify()
760 	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);  in tick_check_oneshot_change()  local
762 	if (!test_and_clear_bit(0, &ts->check_clocks))  in tick_check_oneshot_change()
765 	if (ts->nohz_mode != NOHZ_MODE_INACTIVE)  in tick_check_oneshot_change()