Lines Matching refs:ts
129 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) in tick_sched_handle() argument
140 if (ts->tick_stopped) { in tick_sched_handle()
143 ts->idle_jiffies++; in tick_sched_handle()
188 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now);
196 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); in tick_nohz_full_check() local
199 if (ts->tick_stopped && !is_idle_task(current)) { in tick_nohz_full_check()
201 tick_nohz_restart_sched_tick(ts, ktime_get()); in tick_nohz_full_check()
404 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in tick_nohz_update_jiffies() local
407 ts->idle_waketime = now; in tick_nohz_update_jiffies()
420 update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time) in update_ts_time_stats() argument
424 if (ts->idle_active) { in update_ts_time_stats()
425 delta = ktime_sub(now, ts->idle_entrytime); in update_ts_time_stats()
427 ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta); in update_ts_time_stats()
429 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); in update_ts_time_stats()
430 ts->idle_entrytime = now; in update_ts_time_stats()
440 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in tick_nohz_stop_idle() local
442 update_ts_time_stats(cpu, ts, now, NULL); in tick_nohz_stop_idle()
443 ts->idle_active = 0; in tick_nohz_stop_idle()
448 static ktime_t tick_nohz_start_idle(int cpu, struct tick_sched *ts) in tick_nohz_start_idle() argument
452 ts->idle_entrytime = now; in tick_nohz_start_idle()
453 ts->idle_active = 1; in tick_nohz_start_idle()
474 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in get_cpu_idle_time_us() local
482 update_ts_time_stats(cpu, ts, now, last_update_time); in get_cpu_idle_time_us()
483 idle = ts->idle_sleeptime; in get_cpu_idle_time_us()
485 if (ts->idle_active && !nr_iowait_cpu(cpu)) { in get_cpu_idle_time_us()
486 ktime_t delta = ktime_sub(now, ts->idle_entrytime); in get_cpu_idle_time_us()
488 idle = ktime_add(ts->idle_sleeptime, delta); in get_cpu_idle_time_us()
490 idle = ts->idle_sleeptime; in get_cpu_idle_time_us()
515 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in get_cpu_iowait_time_us() local
523 update_ts_time_stats(cpu, ts, now, last_update_time); in get_cpu_iowait_time_us()
524 iowait = ts->iowait_sleeptime; in get_cpu_iowait_time_us()
526 if (ts->idle_active && nr_iowait_cpu(cpu) > 0) { in get_cpu_iowait_time_us()
527 ktime_t delta = ktime_sub(now, ts->idle_entrytime); in get_cpu_iowait_time_us()
529 iowait = ktime_add(ts->iowait_sleeptime, delta); in get_cpu_iowait_time_us()
531 iowait = ts->iowait_sleeptime; in get_cpu_iowait_time_us()
539 static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, in tick_nohz_stop_sched_tick() argument
574 if (!ts->tick_stopped && delta_jiffies <= 1) in tick_nohz_stop_sched_tick()
595 ts->do_timer_last = 1; in tick_nohz_stop_sched_tick()
598 ts->do_timer_last = 0; in tick_nohz_stop_sched_tick()
599 } else if (!ts->do_timer_last) { in tick_nohz_stop_sched_tick()
604 if (!ts->inidle) { in tick_nohz_stop_sched_tick()
635 if (ts->tick_stopped && ktime_equal(expires, dev->next_event)) in tick_nohz_stop_sched_tick()
647 if (!ts->tick_stopped) { in tick_nohz_stop_sched_tick()
651 ts->last_tick = hrtimer_get_expires(&ts->sched_timer); in tick_nohz_stop_sched_tick()
652 ts->tick_stopped = 1; in tick_nohz_stop_sched_tick()
661 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) in tick_nohz_stop_sched_tick()
662 hrtimer_cancel(&ts->sched_timer); in tick_nohz_stop_sched_tick()
666 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { in tick_nohz_stop_sched_tick()
667 hrtimer_start(&ts->sched_timer, expires, in tick_nohz_stop_sched_tick()
670 if (hrtimer_active(&ts->sched_timer)) in tick_nohz_stop_sched_tick()
683 ts->next_jiffies = next_jiffies; in tick_nohz_stop_sched_tick()
684 ts->last_jiffies = last_jiffies; in tick_nohz_stop_sched_tick()
685 ts->sleep_length = ktime_sub(dev->next_event, now); in tick_nohz_stop_sched_tick()
690 static void tick_nohz_full_stop_tick(struct tick_sched *ts) in tick_nohz_full_stop_tick() argument
698 if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE) in tick_nohz_full_stop_tick()
704 tick_nohz_stop_sched_tick(ts, ktime_get(), cpu); in tick_nohz_full_stop_tick()
708 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) in can_stop_idle_tick() argument
723 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) in can_stop_idle_tick()
759 static void __tick_nohz_idle_enter(struct tick_sched *ts) in __tick_nohz_idle_enter() argument
764 now = tick_nohz_start_idle(cpu, ts); in __tick_nohz_idle_enter()
766 if (can_stop_idle_tick(cpu, ts)) { in __tick_nohz_idle_enter()
767 int was_stopped = ts->tick_stopped; in __tick_nohz_idle_enter()
769 ts->idle_calls++; in __tick_nohz_idle_enter()
771 expires = tick_nohz_stop_sched_tick(ts, now, cpu); in __tick_nohz_idle_enter()
773 ts->idle_sleeps++; in __tick_nohz_idle_enter()
774 ts->idle_expires = expires; in __tick_nohz_idle_enter()
777 if (!was_stopped && ts->tick_stopped) in __tick_nohz_idle_enter()
778 ts->idle_jiffies = ts->last_jiffies; in __tick_nohz_idle_enter()
796 struct tick_sched *ts; in tick_nohz_idle_enter() local
810 ts = &__get_cpu_var(tick_cpu_sched); in tick_nohz_idle_enter()
816 ts->inidle = 1; in tick_nohz_idle_enter()
817 __tick_nohz_idle_enter(ts); in tick_nohz_idle_enter()
833 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); in tick_nohz_irq_exit() local
835 if (ts->inidle) { in tick_nohz_irq_exit()
838 __tick_nohz_idle_enter(ts); in tick_nohz_irq_exit()
840 tick_nohz_full_stop_tick(ts); in tick_nohz_irq_exit()
851 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); in tick_nohz_get_sleep_length() local
853 return ts->sleep_length; in tick_nohz_get_sleep_length()
856 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) in tick_nohz_restart() argument
858 hrtimer_cancel(&ts->sched_timer); in tick_nohz_restart()
859 hrtimer_set_expires(&ts->sched_timer, ts->last_tick); in tick_nohz_restart()
863 hrtimer_forward(&ts->sched_timer, now, tick_period); in tick_nohz_restart()
865 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { in tick_nohz_restart()
866 hrtimer_start_expires(&ts->sched_timer, in tick_nohz_restart()
869 if (hrtimer_active(&ts->sched_timer)) in tick_nohz_restart()
873 hrtimer_get_expires(&ts->sched_timer), 0)) in tick_nohz_restart()
882 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now) in tick_nohz_restart_sched_tick() argument
893 ts->tick_stopped = 0; in tick_nohz_restart_sched_tick()
894 ts->idle_exittime = now; in tick_nohz_restart_sched_tick()
896 tick_nohz_restart(ts, now); in tick_nohz_restart_sched_tick()
899 static void tick_nohz_account_idle_ticks(struct tick_sched *ts) in tick_nohz_account_idle_ticks() argument
911 ticks = jiffies - ts->idle_jiffies; in tick_nohz_account_idle_ticks()
930 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in tick_nohz_idle_exit() local
935 WARN_ON_ONCE(!ts->inidle); in tick_nohz_idle_exit()
937 ts->inidle = 0; in tick_nohz_idle_exit()
941 if (ts->idle_active || ts->tick_stopped) in tick_nohz_idle_exit()
944 if (ts->idle_active) in tick_nohz_idle_exit()
947 if (ts->tick_stopped) { in tick_nohz_idle_exit()
948 tick_nohz_restart_sched_tick(ts, now); in tick_nohz_idle_exit()
949 tick_nohz_account_idle_ticks(ts); in tick_nohz_idle_exit()
956 static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now) in tick_nohz_reprogram() argument
958 hrtimer_forward(&ts->sched_timer, now, tick_period); in tick_nohz_reprogram()
959 return tick_program_event(hrtimer_get_expires(&ts->sched_timer), 0); in tick_nohz_reprogram()
967 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); in tick_nohz_handler() local
974 tick_sched_handle(ts, regs); in tick_nohz_handler()
976 while (tick_nohz_reprogram(ts, now)) { in tick_nohz_handler()
987 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); in tick_nohz_switch_to_nohz() local
999 ts->nohz_mode = NOHZ_MODE_LOWRES; in tick_nohz_switch_to_nohz()
1005 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); in tick_nohz_switch_to_nohz()
1010 hrtimer_set_expires(&ts->sched_timer, next); in tick_nohz_switch_to_nohz()
1034 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in tick_nohz_kick_tick()
1041 delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now); in tick_nohz_kick_tick()
1045 tick_nohz_restart(ts, now); in tick_nohz_kick_tick()
1051 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in tick_check_nohz() local
1054 if (!ts->idle_active && !ts->tick_stopped) in tick_check_nohz()
1057 if (ts->idle_active) in tick_check_nohz()
1059 if (ts->tick_stopped) { in tick_check_nohz()
1091 struct tick_sched *ts = in tick_sched_timer() local
1103 tick_sched_handle(ts, regs); in tick_sched_timer()
1125 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); in tick_setup_sched_timer() local
1131 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); in tick_setup_sched_timer()
1132 ts->sched_timer.function = tick_sched_timer; in tick_setup_sched_timer()
1135 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); in tick_setup_sched_timer()
1142 hrtimer_add_expires_ns(&ts->sched_timer, offset); in tick_setup_sched_timer()
1146 hrtimer_forward(&ts->sched_timer, now, tick_period); in tick_setup_sched_timer()
1147 hrtimer_start_expires(&ts->sched_timer, in tick_setup_sched_timer()
1150 if (hrtimer_active(&ts->sched_timer)) in tick_setup_sched_timer()
1157 ts->nohz_mode = NOHZ_MODE_HIGHRES; in tick_setup_sched_timer()
1165 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in tick_cancel_sched_timer() local
1168 if (ts->sched_timer.base) in tick_cancel_sched_timer()
1169 hrtimer_cancel(&ts->sched_timer); in tick_cancel_sched_timer()
1172 memset(ts, 0, sizeof(*ts)); in tick_cancel_sched_timer()
1192 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); in tick_oneshot_notify() local
1194 set_bit(0, &ts->check_clocks); in tick_oneshot_notify()
1207 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); in tick_check_oneshot_change() local
1209 if (!test_and_clear_bit(0, &ts->check_clocks)) in tick_check_oneshot_change()
1212 if (ts->nohz_mode != NOHZ_MODE_INACTIVE) in tick_check_oneshot_change()