Lines Matching refs:ts
186 static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now) in tick_sched_do_timer() argument
219 if (ts->last_tick_jiffies != jiffies) { in tick_sched_do_timer()
220 ts->stalled_jiffies = 0; in tick_sched_do_timer()
221 ts->last_tick_jiffies = READ_ONCE(jiffies); in tick_sched_do_timer()
223 if (++ts->stalled_jiffies == MAX_STALLED_JIFFIES) { in tick_sched_do_timer()
225 ts->stalled_jiffies = 0; in tick_sched_do_timer()
226 ts->last_tick_jiffies = READ_ONCE(jiffies); in tick_sched_do_timer()
230 if (ts->inidle) in tick_sched_do_timer()
231 ts->got_idle_tick = 1; in tick_sched_do_timer()
234 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) in tick_sched_handle() argument
245 if (ts->tick_stopped) { in tick_sched_handle()
248 ts->idle_jiffies++; in tick_sched_handle()
254 ts->next_tick = 0; in tick_sched_handle()
306 static bool can_stop_full_tick(int cpu, struct tick_sched *ts) in can_stop_full_tick() argument
316 if (check_tick_dependency(&ts->tick_dep_mask)) in can_stop_full_tick()
450 struct tick_sched *ts; in tick_nohz_dep_set_cpu() local
452 ts = per_cpu_ptr(&tick_cpu_sched, cpu); in tick_nohz_dep_set_cpu()
454 prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask); in tick_nohz_dep_set_cpu()
472 struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu); in tick_nohz_dep_clear_cpu() local
474 atomic_andnot(BIT(bit), &ts->tick_dep_mask); in tick_nohz_dep_clear_cpu()
527 struct tick_sched *ts; in __tick_nohz_task_switch() local
532 ts = this_cpu_ptr(&tick_cpu_sched); in __tick_nohz_task_switch()
534 if (ts->tick_stopped) { in __tick_nohz_task_switch()
629 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_tick_stopped() local
631 return ts->tick_stopped; in tick_nohz_tick_stopped()
636 struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu); in tick_nohz_tick_stopped_cpu() local
638 return ts->tick_stopped; in tick_nohz_tick_stopped_cpu()
668 update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time) in update_ts_time_stats() argument
672 if (ts->idle_active) { in update_ts_time_stats()
673 delta = ktime_sub(now, ts->idle_entrytime); in update_ts_time_stats()
675 ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta); in update_ts_time_stats()
677 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); in update_ts_time_stats()
678 ts->idle_entrytime = now; in update_ts_time_stats()
686 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now) in tick_nohz_stop_idle() argument
688 update_ts_time_stats(smp_processor_id(), ts, now, NULL); in tick_nohz_stop_idle()
689 ts->idle_active = 0; in tick_nohz_stop_idle()
694 static void tick_nohz_start_idle(struct tick_sched *ts) in tick_nohz_start_idle() argument
696 ts->idle_entrytime = ktime_get(); in tick_nohz_start_idle()
697 ts->idle_active = 1; in tick_nohz_start_idle()
717 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in get_cpu_idle_time_us() local
725 update_ts_time_stats(cpu, ts, now, last_update_time); in get_cpu_idle_time_us()
726 idle = ts->idle_sleeptime; in get_cpu_idle_time_us()
728 if (ts->idle_active && !nr_iowait_cpu(cpu)) { in get_cpu_idle_time_us()
729 ktime_t delta = ktime_sub(now, ts->idle_entrytime); in get_cpu_idle_time_us()
731 idle = ktime_add(ts->idle_sleeptime, delta); in get_cpu_idle_time_us()
733 idle = ts->idle_sleeptime; in get_cpu_idle_time_us()
758 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in get_cpu_iowait_time_us() local
766 update_ts_time_stats(cpu, ts, now, last_update_time); in get_cpu_iowait_time_us()
767 iowait = ts->iowait_sleeptime; in get_cpu_iowait_time_us()
769 if (ts->idle_active && nr_iowait_cpu(cpu) > 0) { in get_cpu_iowait_time_us()
770 ktime_t delta = ktime_sub(now, ts->idle_entrytime); in get_cpu_iowait_time_us()
772 iowait = ktime_add(ts->iowait_sleeptime, delta); in get_cpu_iowait_time_us()
774 iowait = ts->iowait_sleeptime; in get_cpu_iowait_time_us()
782 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) in tick_nohz_restart() argument
784 hrtimer_cancel(&ts->sched_timer); in tick_nohz_restart()
785 hrtimer_set_expires(&ts->sched_timer, ts->last_tick); in tick_nohz_restart()
788 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); in tick_nohz_restart()
790 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { in tick_nohz_restart()
791 hrtimer_start_expires(&ts->sched_timer, in tick_nohz_restart()
794 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_restart()
801 ts->next_tick = 0; in tick_nohz_restart()
809 static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu) in tick_nohz_next_event() argument
821 ts->last_jiffies = basejiff; in tick_nohz_next_event()
822 ts->timer_expires_base = basemono; in tick_nohz_next_event()
846 ts->next_timer = next_tick; in tick_nohz_next_event()
864 if (!ts->tick_stopped) { in tick_nohz_next_event()
865 ts->timer_expires = 0; in tick_nohz_next_event()
877 (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last)) in tick_nohz_next_event()
886 ts->timer_expires = min_t(u64, expires, next_tick); in tick_nohz_next_event()
889 return ts->timer_expires; in tick_nohz_next_event()
892 static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu) in tick_nohz_stop_tick() argument
895 u64 basemono = ts->timer_expires_base; in tick_nohz_stop_tick()
896 u64 expires = ts->timer_expires; in tick_nohz_stop_tick()
900 ts->timer_expires_base = 0; in tick_nohz_stop_tick()
912 ts->do_timer_last = 1; in tick_nohz_stop_tick()
914 ts->do_timer_last = 0; in tick_nohz_stop_tick()
918 if (ts->tick_stopped && (expires == ts->next_tick)) { in tick_nohz_stop_tick()
920 if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer)) in tick_nohz_stop_tick()
925 basemono, ts->next_tick, dev->next_event, in tick_nohz_stop_tick()
926 hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer)); in tick_nohz_stop_tick()
936 if (!ts->tick_stopped) { in tick_nohz_stop_tick()
940 ts->last_tick = hrtimer_get_expires(&ts->sched_timer); in tick_nohz_stop_tick()
941 ts->tick_stopped = 1; in tick_nohz_stop_tick()
945 ts->next_tick = tick; in tick_nohz_stop_tick()
952 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) in tick_nohz_stop_tick()
953 hrtimer_cancel(&ts->sched_timer); in tick_nohz_stop_tick()
959 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { in tick_nohz_stop_tick()
960 hrtimer_start(&ts->sched_timer, tick, in tick_nohz_stop_tick()
963 hrtimer_set_expires(&ts->sched_timer, tick); in tick_nohz_stop_tick()
968 static void tick_nohz_retain_tick(struct tick_sched *ts) in tick_nohz_retain_tick() argument
970 ts->timer_expires_base = 0; in tick_nohz_retain_tick()
974 static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu) in tick_nohz_stop_sched_tick() argument
976 if (tick_nohz_next_event(ts, cpu)) in tick_nohz_stop_sched_tick()
977 tick_nohz_stop_tick(ts, cpu); in tick_nohz_stop_sched_tick()
979 tick_nohz_retain_tick(ts); in tick_nohz_stop_sched_tick()
983 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now) in tick_nohz_restart_sched_tick() argument
999 ts->tick_stopped = 0; in tick_nohz_restart_sched_tick()
1000 tick_nohz_restart(ts, now); in tick_nohz_restart_sched_tick()
1003 static void __tick_nohz_full_update_tick(struct tick_sched *ts, in __tick_nohz_full_update_tick() argument
1009 if (can_stop_full_tick(cpu, ts)) in __tick_nohz_full_update_tick()
1010 tick_nohz_stop_sched_tick(ts, cpu); in __tick_nohz_full_update_tick()
1011 else if (ts->tick_stopped) in __tick_nohz_full_update_tick()
1012 tick_nohz_restart_sched_tick(ts, now); in __tick_nohz_full_update_tick()
1016 static void tick_nohz_full_update_tick(struct tick_sched *ts) in tick_nohz_full_update_tick() argument
1021 if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE) in tick_nohz_full_update_tick()
1024 __tick_nohz_full_update_tick(ts, ktime_get()); in tick_nohz_full_update_tick()
1066 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) in can_stop_idle_tick() argument
1082 ts->next_tick = 0; in can_stop_idle_tick()
1086 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) in can_stop_idle_tick()
1111 static void __tick_nohz_idle_stop_tick(struct tick_sched *ts) in __tick_nohz_idle_stop_tick() argument
1120 if (ts->timer_expires_base) in __tick_nohz_idle_stop_tick()
1121 expires = ts->timer_expires; in __tick_nohz_idle_stop_tick()
1122 else if (can_stop_idle_tick(cpu, ts)) in __tick_nohz_idle_stop_tick()
1123 expires = tick_nohz_next_event(ts, cpu); in __tick_nohz_idle_stop_tick()
1127 ts->idle_calls++; in __tick_nohz_idle_stop_tick()
1130 int was_stopped = ts->tick_stopped; in __tick_nohz_idle_stop_tick()
1132 tick_nohz_stop_tick(ts, cpu); in __tick_nohz_idle_stop_tick()
1134 ts->idle_sleeps++; in __tick_nohz_idle_stop_tick()
1135 ts->idle_expires = expires; in __tick_nohz_idle_stop_tick()
1137 if (!was_stopped && ts->tick_stopped) { in __tick_nohz_idle_stop_tick()
1138 ts->idle_jiffies = ts->last_jiffies; in __tick_nohz_idle_stop_tick()
1142 tick_nohz_retain_tick(ts); in __tick_nohz_idle_stop_tick()
1173 struct tick_sched *ts; in tick_nohz_idle_enter() local
1179 ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_enter()
1181 WARN_ON_ONCE(ts->timer_expires_base); in tick_nohz_idle_enter()
1183 ts->inidle = 1; in tick_nohz_idle_enter()
1184 tick_nohz_start_idle(ts); in tick_nohz_idle_enter()
1199 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_irq_exit() local
1201 if (ts->inidle) in tick_nohz_irq_exit()
1202 tick_nohz_start_idle(ts); in tick_nohz_irq_exit()
1204 tick_nohz_full_update_tick(ts); in tick_nohz_irq_exit()
1212 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_got_tick() local
1214 if (ts->got_idle_tick) { in tick_nohz_idle_got_tick()
1215 ts->got_idle_tick = 0; in tick_nohz_idle_got_tick()
1246 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_get_sleep_length() local
1252 ktime_t now = ts->idle_entrytime; in tick_nohz_get_sleep_length()
1255 WARN_ON_ONCE(!ts->inidle); in tick_nohz_get_sleep_length()
1259 if (!can_stop_idle_tick(cpu, ts)) in tick_nohz_get_sleep_length()
1262 next_event = tick_nohz_next_event(ts, cpu); in tick_nohz_get_sleep_length()
1271 hrtimer_next_event_without(&ts->sched_timer)); in tick_nohz_get_sleep_length()
1285 struct tick_sched *ts = tick_get_tick_sched(cpu); in tick_nohz_get_idle_calls_cpu() local
1287 return ts->idle_calls; in tick_nohz_get_idle_calls_cpu()
1298 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_get_idle_calls() local
1300 return ts->idle_calls; in tick_nohz_get_idle_calls()
1303 static void tick_nohz_account_idle_time(struct tick_sched *ts, in tick_nohz_account_idle_time() argument
1308 ts->idle_exittime = now; in tick_nohz_account_idle_time()
1317 ticks = jiffies - ts->idle_jiffies; in tick_nohz_account_idle_time()
1327 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_restart_tick() local
1329 if (ts->tick_stopped) { in tick_nohz_idle_restart_tick()
1331 tick_nohz_restart_sched_tick(ts, now); in tick_nohz_idle_restart_tick()
1332 tick_nohz_account_idle_time(ts, now); in tick_nohz_idle_restart_tick()
1336 static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now) in tick_nohz_idle_update_tick() argument
1339 __tick_nohz_full_update_tick(ts, now); in tick_nohz_idle_update_tick()
1341 tick_nohz_restart_sched_tick(ts, now); in tick_nohz_idle_update_tick()
1343 tick_nohz_account_idle_time(ts, now); in tick_nohz_idle_update_tick()
1355 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_exit() local
1361 WARN_ON_ONCE(!ts->inidle); in tick_nohz_idle_exit()
1362 WARN_ON_ONCE(ts->timer_expires_base); in tick_nohz_idle_exit()
1364 ts->inidle = 0; in tick_nohz_idle_exit()
1365 idle_active = ts->idle_active; in tick_nohz_idle_exit()
1366 tick_stopped = ts->tick_stopped; in tick_nohz_idle_exit()
1372 tick_nohz_stop_idle(ts, now); in tick_nohz_idle_exit()
1375 tick_nohz_idle_update_tick(ts, now); in tick_nohz_idle_exit()
1385 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_handler() local
1391 tick_sched_do_timer(ts, now); in tick_nohz_handler()
1392 tick_sched_handle(ts, regs); in tick_nohz_handler()
1394 if (unlikely(ts->tick_stopped)) { in tick_nohz_handler()
1404 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); in tick_nohz_handler()
1405 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_handler()
1408 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) in tick_nohz_activate() argument
1412 ts->nohz_mode = mode; in tick_nohz_activate()
1423 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_switch_to_nohz() local
1436 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); in tick_nohz_switch_to_nohz()
1440 hrtimer_set_expires(&ts->sched_timer, next); in tick_nohz_switch_to_nohz()
1441 hrtimer_forward_now(&ts->sched_timer, TICK_NSEC); in tick_nohz_switch_to_nohz()
1442 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_switch_to_nohz()
1443 tick_nohz_activate(ts, NOHZ_MODE_LOWRES); in tick_nohz_switch_to_nohz()
1448 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_irq_enter() local
1451 if (!ts->idle_active && !ts->tick_stopped) in tick_nohz_irq_enter()
1454 if (ts->idle_active) in tick_nohz_irq_enter()
1455 tick_nohz_stop_idle(ts, now); in tick_nohz_irq_enter()
1463 if (ts->tick_stopped) in tick_nohz_irq_enter()
1471 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { } in tick_nohz_activate() argument
1494 struct tick_sched *ts = in tick_sched_timer() local
1499 tick_sched_do_timer(ts, now); in tick_sched_timer()
1506 tick_sched_handle(ts, regs); in tick_sched_timer()
1508 ts->next_tick = 0; in tick_sched_timer()
1511 if (unlikely(ts->tick_stopped)) in tick_sched_timer()
1534 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_setup_sched_timer() local
1540 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); in tick_setup_sched_timer()
1541 ts->sched_timer.function = tick_sched_timer; in tick_setup_sched_timer()
1544 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); in tick_setup_sched_timer()
1551 hrtimer_add_expires_ns(&ts->sched_timer, offset); in tick_setup_sched_timer()
1554 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); in tick_setup_sched_timer()
1555 hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD); in tick_setup_sched_timer()
1556 tick_nohz_activate(ts, NOHZ_MODE_HIGHRES); in tick_setup_sched_timer()
1563 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in tick_cancel_sched_timer() local
1568 if (ts->sched_timer.base) in tick_cancel_sched_timer()
1569 hrtimer_cancel(&ts->sched_timer); in tick_cancel_sched_timer()
1572 idle_sleeptime = ts->idle_sleeptime; in tick_cancel_sched_timer()
1573 iowait_sleeptime = ts->iowait_sleeptime; in tick_cancel_sched_timer()
1574 idle_calls = ts->idle_calls; in tick_cancel_sched_timer()
1575 idle_sleeps = ts->idle_sleeps; in tick_cancel_sched_timer()
1576 memset(ts, 0, sizeof(*ts)); in tick_cancel_sched_timer()
1577 ts->idle_sleeptime = idle_sleeptime; in tick_cancel_sched_timer()
1578 ts->iowait_sleeptime = iowait_sleeptime; in tick_cancel_sched_timer()
1579 ts->idle_calls = idle_calls; in tick_cancel_sched_timer()
1580 ts->idle_sleeps = idle_sleeps; in tick_cancel_sched_timer()
1600 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_oneshot_notify() local
1602 set_bit(0, &ts->check_clocks); in tick_oneshot_notify()
1615 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_check_oneshot_change() local
1617 if (!test_and_clear_bit(0, &ts->check_clocks)) in tick_check_oneshot_change()
1620 if (ts->nohz_mode != NOHZ_MODE_INACTIVE) in tick_check_oneshot_change()