Lines Matching refs:ts
151 static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now) in tick_sched_do_timer() argument
178 if (ts->inidle) in tick_sched_do_timer()
179 ts->got_idle_tick = 1; in tick_sched_do_timer()
182 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) in tick_sched_handle() argument
193 if (ts->tick_stopped) { in tick_sched_handle()
196 ts->idle_jiffies++; in tick_sched_handle()
202 ts->next_tick = 0; in tick_sched_handle()
247 static bool can_stop_full_tick(int cpu, struct tick_sched *ts) in can_stop_full_tick() argument
257 if (check_tick_dependency(&ts->tick_dep_mask)) in can_stop_full_tick()
352 struct tick_sched *ts; in tick_nohz_dep_set_cpu() local
354 ts = per_cpu_ptr(&tick_cpu_sched, cpu); in tick_nohz_dep_set_cpu()
356 prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask); in tick_nohz_dep_set_cpu()
374 struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu); in tick_nohz_dep_clear_cpu() local
376 atomic_andnot(BIT(bit), &ts->tick_dep_mask); in tick_nohz_dep_clear_cpu()
420 struct tick_sched *ts; in __tick_nohz_task_switch() local
427 ts = this_cpu_ptr(&tick_cpu_sched); in __tick_nohz_task_switch()
429 if (ts->tick_stopped) { in __tick_nohz_task_switch()
526 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_tick_stopped() local
528 return ts->tick_stopped; in tick_nohz_tick_stopped()
533 struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu); in tick_nohz_tick_stopped_cpu() local
535 return ts->tick_stopped; in tick_nohz_tick_stopped_cpu()
565 update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time) in update_ts_time_stats() argument
569 if (ts->idle_active) { in update_ts_time_stats()
570 delta = ktime_sub(now, ts->idle_entrytime); in update_ts_time_stats()
572 ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta); in update_ts_time_stats()
574 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); in update_ts_time_stats()
575 ts->idle_entrytime = now; in update_ts_time_stats()
583 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now) in tick_nohz_stop_idle() argument
585 update_ts_time_stats(smp_processor_id(), ts, now, NULL); in tick_nohz_stop_idle()
586 ts->idle_active = 0; in tick_nohz_stop_idle()
591 static void tick_nohz_start_idle(struct tick_sched *ts) in tick_nohz_start_idle() argument
593 ts->idle_entrytime = ktime_get(); in tick_nohz_start_idle()
594 ts->idle_active = 1; in tick_nohz_start_idle()
614 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in get_cpu_idle_time_us() local
622 update_ts_time_stats(cpu, ts, now, last_update_time); in get_cpu_idle_time_us()
623 idle = ts->idle_sleeptime; in get_cpu_idle_time_us()
625 if (ts->idle_active && !nr_iowait_cpu(cpu)) { in get_cpu_idle_time_us()
626 ktime_t delta = ktime_sub(now, ts->idle_entrytime); in get_cpu_idle_time_us()
628 idle = ktime_add(ts->idle_sleeptime, delta); in get_cpu_idle_time_us()
630 idle = ts->idle_sleeptime; in get_cpu_idle_time_us()
655 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in get_cpu_iowait_time_us() local
663 update_ts_time_stats(cpu, ts, now, last_update_time); in get_cpu_iowait_time_us()
664 iowait = ts->iowait_sleeptime; in get_cpu_iowait_time_us()
666 if (ts->idle_active && nr_iowait_cpu(cpu) > 0) { in get_cpu_iowait_time_us()
667 ktime_t delta = ktime_sub(now, ts->idle_entrytime); in get_cpu_iowait_time_us()
669 iowait = ktime_add(ts->iowait_sleeptime, delta); in get_cpu_iowait_time_us()
671 iowait = ts->iowait_sleeptime; in get_cpu_iowait_time_us()
679 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) in tick_nohz_restart() argument
681 hrtimer_cancel(&ts->sched_timer); in tick_nohz_restart()
682 hrtimer_set_expires(&ts->sched_timer, ts->last_tick); in tick_nohz_restart()
685 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); in tick_nohz_restart()
687 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { in tick_nohz_restart()
688 hrtimer_start_expires(&ts->sched_timer, in tick_nohz_restart()
691 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_restart()
698 ts->next_tick = 0; in tick_nohz_restart()
706 static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu) in tick_nohz_next_event() argument
718 ts->last_jiffies = basejiff; in tick_nohz_next_event()
719 ts->timer_expires_base = basemono; in tick_nohz_next_event()
743 ts->next_timer = next_tmr; in tick_nohz_next_event()
763 if (!ts->tick_stopped) { in tick_nohz_next_event()
764 ts->timer_expires = 0; in tick_nohz_next_event()
776 (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last)) in tick_nohz_next_event()
785 ts->timer_expires = min_t(u64, expires, next_tick); in tick_nohz_next_event()
788 return ts->timer_expires; in tick_nohz_next_event()
791 static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu) in tick_nohz_stop_tick() argument
794 u64 basemono = ts->timer_expires_base; in tick_nohz_stop_tick()
795 u64 expires = ts->timer_expires; in tick_nohz_stop_tick()
799 ts->timer_expires_base = 0; in tick_nohz_stop_tick()
811 ts->do_timer_last = 1; in tick_nohz_stop_tick()
813 ts->do_timer_last = 0; in tick_nohz_stop_tick()
817 if (ts->tick_stopped && (expires == ts->next_tick)) { in tick_nohz_stop_tick()
819 if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer)) in tick_nohz_stop_tick()
824 basemono, ts->next_tick, dev->next_event, in tick_nohz_stop_tick()
825 hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer)); in tick_nohz_stop_tick()
835 if (!ts->tick_stopped) { in tick_nohz_stop_tick()
839 ts->last_tick = hrtimer_get_expires(&ts->sched_timer); in tick_nohz_stop_tick()
840 ts->tick_stopped = 1; in tick_nohz_stop_tick()
844 ts->next_tick = tick; in tick_nohz_stop_tick()
851 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) in tick_nohz_stop_tick()
852 hrtimer_cancel(&ts->sched_timer); in tick_nohz_stop_tick()
856 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { in tick_nohz_stop_tick()
857 hrtimer_start(&ts->sched_timer, tick, in tick_nohz_stop_tick()
860 hrtimer_set_expires(&ts->sched_timer, tick); in tick_nohz_stop_tick()
865 static void tick_nohz_retain_tick(struct tick_sched *ts) in tick_nohz_retain_tick() argument
867 ts->timer_expires_base = 0; in tick_nohz_retain_tick()
871 static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu) in tick_nohz_stop_sched_tick() argument
873 if (tick_nohz_next_event(ts, cpu)) in tick_nohz_stop_sched_tick()
874 tick_nohz_stop_tick(ts, cpu); in tick_nohz_stop_sched_tick()
876 tick_nohz_retain_tick(ts); in tick_nohz_stop_sched_tick()
880 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now) in tick_nohz_restart_sched_tick() argument
896 ts->tick_stopped = 0; in tick_nohz_restart_sched_tick()
897 ts->idle_exittime = now; in tick_nohz_restart_sched_tick()
899 tick_nohz_restart(ts, now); in tick_nohz_restart_sched_tick()
902 static void tick_nohz_full_update_tick(struct tick_sched *ts) in tick_nohz_full_update_tick() argument
910 if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE) in tick_nohz_full_update_tick()
913 if (can_stop_full_tick(cpu, ts)) in tick_nohz_full_update_tick()
914 tick_nohz_stop_sched_tick(ts, cpu); in tick_nohz_full_update_tick()
915 else if (ts->tick_stopped) in tick_nohz_full_update_tick()
916 tick_nohz_restart_sched_tick(ts, ktime_get()); in tick_nohz_full_update_tick()
920 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) in can_stop_idle_tick() argument
936 ts->next_tick = 0; in can_stop_idle_tick()
940 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) in can_stop_idle_tick()
974 static void __tick_nohz_idle_stop_tick(struct tick_sched *ts) in __tick_nohz_idle_stop_tick() argument
983 if (ts->timer_expires_base) in __tick_nohz_idle_stop_tick()
984 expires = ts->timer_expires; in __tick_nohz_idle_stop_tick()
985 else if (can_stop_idle_tick(cpu, ts)) in __tick_nohz_idle_stop_tick()
986 expires = tick_nohz_next_event(ts, cpu); in __tick_nohz_idle_stop_tick()
990 ts->idle_calls++; in __tick_nohz_idle_stop_tick()
993 int was_stopped = ts->tick_stopped; in __tick_nohz_idle_stop_tick()
995 tick_nohz_stop_tick(ts, cpu); in __tick_nohz_idle_stop_tick()
997 ts->idle_sleeps++; in __tick_nohz_idle_stop_tick()
998 ts->idle_expires = expires; in __tick_nohz_idle_stop_tick()
1000 if (!was_stopped && ts->tick_stopped) { in __tick_nohz_idle_stop_tick()
1001 ts->idle_jiffies = ts->last_jiffies; in __tick_nohz_idle_stop_tick()
1005 tick_nohz_retain_tick(ts); in __tick_nohz_idle_stop_tick()
1036 struct tick_sched *ts; in tick_nohz_idle_enter() local
1042 ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_enter()
1044 WARN_ON_ONCE(ts->timer_expires_base); in tick_nohz_idle_enter()
1046 ts->inidle = 1; in tick_nohz_idle_enter()
1047 tick_nohz_start_idle(ts); in tick_nohz_idle_enter()
1062 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_irq_exit() local
1064 if (ts->inidle) in tick_nohz_irq_exit()
1065 tick_nohz_start_idle(ts); in tick_nohz_irq_exit()
1067 tick_nohz_full_update_tick(ts); in tick_nohz_irq_exit()
1075 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_got_tick() local
1077 if (ts->got_idle_tick) { in tick_nohz_idle_got_tick()
1078 ts->got_idle_tick = 0; in tick_nohz_idle_got_tick()
1105 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_get_sleep_length() local
1111 ktime_t now = ts->idle_entrytime; in tick_nohz_get_sleep_length()
1114 WARN_ON_ONCE(!ts->inidle); in tick_nohz_get_sleep_length()
1118 if (!can_stop_idle_tick(cpu, ts)) in tick_nohz_get_sleep_length()
1121 next_event = tick_nohz_next_event(ts, cpu); in tick_nohz_get_sleep_length()
1130 hrtimer_next_event_without(&ts->sched_timer)); in tick_nohz_get_sleep_length()
1143 struct tick_sched *ts = tick_get_tick_sched(cpu); in tick_nohz_get_idle_calls_cpu() local
1145 return ts->idle_calls; in tick_nohz_get_idle_calls_cpu()
1155 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_get_idle_calls() local
1157 return ts->idle_calls; in tick_nohz_get_idle_calls()
1160 static void tick_nohz_account_idle_ticks(struct tick_sched *ts) in tick_nohz_account_idle_ticks() argument
1172 ticks = jiffies - ts->idle_jiffies; in tick_nohz_account_idle_ticks()
1181 static void __tick_nohz_idle_restart_tick(struct tick_sched *ts, ktime_t now) in __tick_nohz_idle_restart_tick() argument
1183 tick_nohz_restart_sched_tick(ts, now); in __tick_nohz_idle_restart_tick()
1184 tick_nohz_account_idle_ticks(ts); in __tick_nohz_idle_restart_tick()
1189 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_restart_tick() local
1191 if (ts->tick_stopped) in tick_nohz_idle_restart_tick()
1192 __tick_nohz_idle_restart_tick(ts, ktime_get()); in tick_nohz_idle_restart_tick()
1204 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_exit() local
1210 WARN_ON_ONCE(!ts->inidle); in tick_nohz_idle_exit()
1211 WARN_ON_ONCE(ts->timer_expires_base); in tick_nohz_idle_exit()
1213 ts->inidle = 0; in tick_nohz_idle_exit()
1214 idle_active = ts->idle_active; in tick_nohz_idle_exit()
1215 tick_stopped = ts->tick_stopped; in tick_nohz_idle_exit()
1221 tick_nohz_stop_idle(ts, now); in tick_nohz_idle_exit()
1224 __tick_nohz_idle_restart_tick(ts, now); in tick_nohz_idle_exit()
1234 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_handler() local
1240 tick_sched_do_timer(ts, now); in tick_nohz_handler()
1241 tick_sched_handle(ts, regs); in tick_nohz_handler()
1244 if (unlikely(ts->tick_stopped)) in tick_nohz_handler()
1247 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); in tick_nohz_handler()
1248 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_handler()
1251 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) in tick_nohz_activate() argument
1255 ts->nohz_mode = mode; in tick_nohz_activate()
1266 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_switch_to_nohz() local
1279 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); in tick_nohz_switch_to_nohz()
1283 hrtimer_set_expires(&ts->sched_timer, next); in tick_nohz_switch_to_nohz()
1284 hrtimer_forward_now(&ts->sched_timer, TICK_NSEC); in tick_nohz_switch_to_nohz()
1285 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_switch_to_nohz()
1286 tick_nohz_activate(ts, NOHZ_MODE_LOWRES); in tick_nohz_switch_to_nohz()
1291 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_irq_enter() local
1294 if (!ts->idle_active && !ts->tick_stopped) in tick_nohz_irq_enter()
1297 if (ts->idle_active) in tick_nohz_irq_enter()
1298 tick_nohz_stop_idle(ts, now); in tick_nohz_irq_enter()
1299 if (ts->tick_stopped) in tick_nohz_irq_enter()
1307 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { } in tick_nohz_activate() argument
1330 struct tick_sched *ts = in tick_sched_timer() local
1335 tick_sched_do_timer(ts, now); in tick_sched_timer()
1342 tick_sched_handle(ts, regs); in tick_sched_timer()
1344 ts->next_tick = 0; in tick_sched_timer()
1347 if (unlikely(ts->tick_stopped)) in tick_sched_timer()
1370 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_setup_sched_timer() local
1376 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); in tick_setup_sched_timer()
1377 ts->sched_timer.function = tick_sched_timer; in tick_setup_sched_timer()
1380 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); in tick_setup_sched_timer()
1387 hrtimer_add_expires_ns(&ts->sched_timer, offset); in tick_setup_sched_timer()
1390 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); in tick_setup_sched_timer()
1391 hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD); in tick_setup_sched_timer()
1392 tick_nohz_activate(ts, NOHZ_MODE_HIGHRES); in tick_setup_sched_timer()
1399 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in tick_cancel_sched_timer() local
1404 if (ts->sched_timer.base) in tick_cancel_sched_timer()
1405 hrtimer_cancel(&ts->sched_timer); in tick_cancel_sched_timer()
1408 idle_sleeptime = ts->idle_sleeptime; in tick_cancel_sched_timer()
1409 iowait_sleeptime = ts->iowait_sleeptime; in tick_cancel_sched_timer()
1410 idle_calls = ts->idle_calls; in tick_cancel_sched_timer()
1411 idle_sleeps = ts->idle_sleeps; in tick_cancel_sched_timer()
1412 memset(ts, 0, sizeof(*ts)); in tick_cancel_sched_timer()
1413 ts->idle_sleeptime = idle_sleeptime; in tick_cancel_sched_timer()
1414 ts->iowait_sleeptime = iowait_sleeptime; in tick_cancel_sched_timer()
1415 ts->idle_calls = idle_calls; in tick_cancel_sched_timer()
1416 ts->idle_sleeps = idle_sleeps; in tick_cancel_sched_timer()
1436 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_oneshot_notify() local
1438 set_bit(0, &ts->check_clocks); in tick_oneshot_notify()
1451 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_check_oneshot_change() local
1453 if (!test_and_clear_bit(0, &ts->check_clocks)) in tick_check_oneshot_change()
1456 if (ts->nohz_mode != NOHZ_MODE_INACTIVE) in tick_check_oneshot_change()