Lines Matching refs:tk
117 static inline void tk_normalize_xtime(struct timekeeper *tk) in tk_normalize_xtime() argument
119 while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) { in tk_normalize_xtime()
120 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift; in tk_normalize_xtime()
121 tk->xtime_sec++; in tk_normalize_xtime()
123 while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) { in tk_normalize_xtime()
124 tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift; in tk_normalize_xtime()
125 tk->raw_sec++; in tk_normalize_xtime()
129 static inline struct timespec64 tk_xtime(const struct timekeeper *tk) in tk_xtime() argument
133 ts.tv_sec = tk->xtime_sec; in tk_xtime()
134 ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); in tk_xtime()
138 static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts) in tk_set_xtime() argument
140 tk->xtime_sec = ts->tv_sec; in tk_set_xtime()
141 tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift; in tk_set_xtime()
144 static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts) in tk_xtime_add() argument
146 tk->xtime_sec += ts->tv_sec; in tk_xtime_add()
147 tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift; in tk_xtime_add()
148 tk_normalize_xtime(tk); in tk_xtime_add()
151 static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm) in tk_set_wall_to_mono() argument
159 set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec, in tk_set_wall_to_mono()
160 -tk->wall_to_monotonic.tv_nsec); in tk_set_wall_to_mono()
161 WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp)); in tk_set_wall_to_mono()
162 tk->wall_to_monotonic = wtm; in tk_set_wall_to_mono()
164 tk->offs_real = timespec64_to_ktime(tmp); in tk_set_wall_to_mono()
165 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)); in tk_set_wall_to_mono()
168 static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta) in tk_update_sleep_time() argument
170 tk->offs_boot = ktime_add(tk->offs_boot, delta); in tk_update_sleep_time()
175 tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot); in tk_update_sleep_time()
201 static void timekeeping_check_update(struct timekeeper *tk, u64 offset) in timekeeping_check_update() argument
204 u64 max_cycles = tk->tkr_mono.clock->max_cycles; in timekeeping_check_update()
205 const char *name = tk->tkr_mono.clock->name; in timekeeping_check_update()
219 if (tk->underflow_seen) { in timekeeping_check_update()
220 if (jiffies - tk->last_warning > WARNING_FREQ) { in timekeeping_check_update()
224 tk->last_warning = jiffies; in timekeeping_check_update()
226 tk->underflow_seen = 0; in timekeeping_check_update()
229 if (tk->overflow_seen) { in timekeeping_check_update()
230 if (jiffies - tk->last_warning > WARNING_FREQ) { in timekeeping_check_update()
234 tk->last_warning = jiffies; in timekeeping_check_update()
236 tk->overflow_seen = 0; in timekeeping_check_update()
242 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_get_delta() local
268 tk->underflow_seen = 1; in timekeeping_get_delta()
274 tk->overflow_seen = 1; in timekeeping_get_delta()
281 static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset) in timekeeping_check_update() argument
309 static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) in tk_setup_internals() argument
315 ++tk->cs_was_changed_seq; in tk_setup_internals()
316 old_clock = tk->tkr_mono.clock; in tk_setup_internals()
317 tk->tkr_mono.clock = clock; in tk_setup_internals()
318 tk->tkr_mono.mask = clock->mask; in tk_setup_internals()
319 tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono); in tk_setup_internals()
321 tk->tkr_raw.clock = clock; in tk_setup_internals()
322 tk->tkr_raw.mask = clock->mask; in tk_setup_internals()
323 tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last; in tk_setup_internals()
335 tk->cycle_interval = interval; in tk_setup_internals()
338 tk->xtime_interval = interval * clock->mult; in tk_setup_internals()
339 tk->xtime_remainder = ntpinterval - tk->xtime_interval; in tk_setup_internals()
340 tk->raw_interval = interval * clock->mult; in tk_setup_internals()
346 tk->tkr_mono.xtime_nsec >>= -shift_change; in tk_setup_internals()
347 tk->tkr_raw.xtime_nsec >>= -shift_change; in tk_setup_internals()
349 tk->tkr_mono.xtime_nsec <<= shift_change; in tk_setup_internals()
350 tk->tkr_raw.xtime_nsec <<= shift_change; in tk_setup_internals()
354 tk->tkr_mono.shift = clock->shift; in tk_setup_internals()
355 tk->tkr_raw.shift = clock->shift; in tk_setup_internals()
357 tk->ntp_error = 0; in tk_setup_internals()
358 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; in tk_setup_internals()
359 tk->ntp_tick = ntpinterval << tk->ntp_error_shift; in tk_setup_internals()
366 tk->tkr_mono.mult = clock->mult; in tk_setup_internals()
367 tk->tkr_raw.mult = clock->mult; in tk_setup_internals()
368 tk->ntp_err_mult = 0; in tk_setup_internals()
369 tk->skip_second_overflow = 0; in tk_setup_internals()
534 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_boot_fast_ns() local
536 return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_boot))); in ktime_get_boot_fast_ns()
551 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_tai_fast_ns() local
553 return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_tai))); in ktime_get_tai_fast_ns()
635 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_fast_timestamps() local
638 snapshot->boot = snapshot->mono + ktime_to_ns(data_race(tk->offs_boot)); in ktime_get_fast_timestamps()
651 static void halt_fast_timekeeper(const struct timekeeper *tk) in halt_fast_timekeeper() argument
654 const struct tk_read_base *tkr = &tk->tkr_mono; in halt_fast_timekeeper()
659 tkr_dummy.base_real = tkr->base + tk->offs_real; in halt_fast_timekeeper()
662 tkr = &tk->tkr_raw; in halt_fast_timekeeper()
670 static void update_pvclock_gtod(struct timekeeper *tk, bool was_set) in update_pvclock_gtod() argument
672 raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk); in update_pvclock_gtod()
681 struct timekeeper *tk = &tk_core.timekeeper; in pvclock_gtod_register_notifier() local
687 update_pvclock_gtod(tk, true); in pvclock_gtod_register_notifier()
715 static inline void tk_update_leap_state(struct timekeeper *tk) in tk_update_leap_state() argument
717 tk->next_leap_ktime = ntp_get_next_leap(); in tk_update_leap_state()
718 if (tk->next_leap_ktime != KTIME_MAX) in tk_update_leap_state()
720 tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real); in tk_update_leap_state()
726 static inline void tk_update_ktime_data(struct timekeeper *tk) in tk_update_ktime_data() argument
738 seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); in tk_update_ktime_data()
739 nsec = (u32) tk->wall_to_monotonic.tv_nsec; in tk_update_ktime_data()
740 tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); in tk_update_ktime_data()
747 nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); in tk_update_ktime_data()
750 tk->ktime_sec = seconds; in tk_update_ktime_data()
753 tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC); in tk_update_ktime_data()
757 static void timekeeping_update(struct timekeeper *tk, unsigned int action) in timekeeping_update() argument
760 tk->ntp_error = 0; in timekeeping_update()
764 tk_update_leap_state(tk); in timekeeping_update()
765 tk_update_ktime_data(tk); in timekeeping_update()
767 update_vsyscall(tk); in timekeeping_update()
768 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); in timekeeping_update()
770 tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real; in timekeeping_update()
771 update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono); in timekeeping_update()
772 update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw); in timekeeping_update()
775 tk->clock_was_set_seq++; in timekeeping_update()
794 static void timekeeping_forward_now(struct timekeeper *tk) in timekeeping_forward_now() argument
798 cycle_now = tk_clock_read(&tk->tkr_mono); in timekeeping_forward_now()
799 delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask); in timekeeping_forward_now()
800 tk->tkr_mono.cycle_last = cycle_now; in timekeeping_forward_now()
801 tk->tkr_raw.cycle_last = cycle_now; in timekeeping_forward_now()
803 tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult; in timekeeping_forward_now()
804 tk->tkr_raw.xtime_nsec += delta * tk->tkr_raw.mult; in timekeeping_forward_now()
806 tk_normalize_xtime(tk); in timekeeping_forward_now()
817 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_real_ts64() local
826 ts->tv_sec = tk->xtime_sec; in ktime_get_real_ts64()
827 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_real_ts64()
838 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get() local
847 base = tk->tkr_mono.base; in ktime_get()
848 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get()
858 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_resolution_ns() local
866 nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift; in ktime_get_resolution_ns()
881 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_with_offset() local
890 base = ktime_add(tk->tkr_mono.base, *offset); in ktime_get_with_offset()
891 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_with_offset()
902 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_coarse_with_offset() local
911 base = ktime_add(tk->tkr_mono.base, *offset); in ktime_get_coarse_with_offset()
912 nsecs = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift; in ktime_get_coarse_with_offset()
945 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_raw() local
952 base = tk->tkr_raw.base; in ktime_get_raw()
953 nsecs = timekeeping_get_ns(&tk->tkr_raw); in ktime_get_raw()
971 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_ts64() local
980 ts->tv_sec = tk->xtime_sec; in ktime_get_ts64()
981 nsec = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_ts64()
982 tomono = tk->wall_to_monotonic; in ktime_get_ts64()
1003 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_seconds() local
1006 return tk->ktime_sec; in ktime_get_seconds()
1022 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_real_seconds() local
1027 return tk->xtime_sec; in ktime_get_real_seconds()
1031 seconds = tk->xtime_sec; in ktime_get_real_seconds()
1046 struct timekeeper *tk = &tk_core.timekeeper; in __ktime_get_real_seconds() local
1048 return tk->xtime_sec; in __ktime_get_real_seconds()
1057 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_snapshot() local
1071 now = tk_clock_read(&tk->tkr_mono); in ktime_get_snapshot()
1072 systime_snapshot->cs_id = tk->tkr_mono.clock->id; in ktime_get_snapshot()
1073 systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq; in ktime_get_snapshot()
1074 systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq; in ktime_get_snapshot()
1075 base_real = ktime_add(tk->tkr_mono.base, in ktime_get_snapshot()
1077 base_boot = ktime_add(tk->tkr_mono.base, in ktime_get_snapshot()
1079 base_raw = tk->tkr_raw.base; in ktime_get_snapshot()
1080 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now); in ktime_get_snapshot()
1081 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now); in ktime_get_snapshot()
1082 mono_mult = tk->tkr_mono.mult; in ktime_get_snapshot()
1083 mono_shift = tk->tkr_mono.shift; in ktime_get_snapshot()
1135 struct timekeeper *tk = &tk_core.timekeeper; in adjust_historical_crosststamp() local
1169 (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult); in adjust_historical_crosststamp()
1223 struct timekeeper *tk = &tk_core.timekeeper; in get_device_system_crosststamp() local
1248 if (tk->tkr_mono.clock != system_counterval.cs) in get_device_system_crosststamp()
1256 now = tk_clock_read(&tk->tkr_mono); in get_device_system_crosststamp()
1257 interval_start = tk->tkr_mono.cycle_last; in get_device_system_crosststamp()
1259 clock_was_set_seq = tk->clock_was_set_seq; in get_device_system_crosststamp()
1260 cs_was_changed_seq = tk->cs_was_changed_seq; in get_device_system_crosststamp()
1267 base_real = ktime_add(tk->tkr_mono.base, in get_device_system_crosststamp()
1269 base_raw = tk->tkr_raw.base; in get_device_system_crosststamp()
1271 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, in get_device_system_crosststamp()
1273 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, in get_device_system_crosststamp()
1323 struct timekeeper *tk = &tk_core.timekeeper; in do_settimeofday64() local
1334 timekeeping_forward_now(tk); in do_settimeofday64()
1336 xt = tk_xtime(tk); in do_settimeofday64()
1339 if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) { in do_settimeofday64()
1344 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta)); in do_settimeofday64()
1346 tk_set_xtime(tk, ts); in do_settimeofday64()
1348 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); in do_settimeofday64()
1373 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_inject_offset() local
1384 timekeeping_forward_now(tk); in timekeeping_inject_offset()
1387 tmp = timespec64_add(tk_xtime(tk), *ts); in timekeeping_inject_offset()
1388 if (timespec64_compare(&tk->wall_to_monotonic, ts) > 0 || in timekeeping_inject_offset()
1394 tk_xtime_add(tk, ts); in timekeeping_inject_offset()
1395 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *ts)); in timekeeping_inject_offset()
1398 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); in timekeeping_inject_offset()
1446 static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset) in __timekeeping_set_tai_offset() argument
1448 tk->tai_offset = tai_offset; in __timekeeping_set_tai_offset()
1449 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); in __timekeeping_set_tai_offset()
1459 struct timekeeper *tk = &tk_core.timekeeper; in change_clocksource() local
1480 timekeeping_forward_now(tk); in change_clocksource()
1483 old = tk->tkr_mono.clock; in change_clocksource()
1484 tk_setup_internals(tk, new); in change_clocksource()
1487 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); in change_clocksource()
1511 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_notify() local
1513 if (tk->tkr_mono.clock == clock) in timekeeping_notify()
1517 return tk->tkr_mono.clock == clock ? 0 : -1; in timekeeping_notify()
1528 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_raw_ts64() local
1534 ts->tv_sec = tk->raw_sec; in ktime_get_raw_ts64()
1535 nsecs = timekeeping_get_ns(&tk->tkr_raw); in ktime_get_raw_ts64()
1550 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_valid_for_hres() local
1557 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; in timekeeping_valid_for_hres()
1569 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_max_deferment() local
1576 ret = tk->tkr_mono.clock->max_idle_ns; in timekeeping_max_deferment()
1644 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_init() local
1673 tk_setup_internals(tk, clock); in timekeeping_init()
1675 tk_set_xtime(tk, &wall_time); in timekeeping_init()
1676 tk->raw_sec = 0; in timekeeping_init()
1678 tk_set_wall_to_mono(tk, wall_to_mono); in timekeeping_init()
1680 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); in timekeeping_init()
1697 static void __timekeeping_inject_sleeptime(struct timekeeper *tk, in __timekeeping_inject_sleeptime() argument
1706 tk_xtime_add(tk, delta); in __timekeeping_inject_sleeptime()
1707 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta)); in __timekeeping_inject_sleeptime()
1708 tk_update_sleep_time(tk, timespec64_to_ktime(*delta)); in __timekeeping_inject_sleeptime()
1761 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_inject_sleeptime64() local
1769 timekeeping_forward_now(tk); in timekeeping_inject_sleeptime64()
1771 __timekeeping_inject_sleeptime(tk, delta); in timekeeping_inject_sleeptime64()
1773 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); in timekeeping_inject_sleeptime64()
1788 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_resume() local
1789 struct clocksource *clock = tk->tkr_mono.clock; in timekeeping_resume()
1815 cycle_now = tk_clock_read(&tk->tkr_mono); in timekeeping_resume()
1827 __timekeeping_inject_sleeptime(tk, &ts_delta); in timekeeping_resume()
1831 tk->tkr_mono.cycle_last = cycle_now; in timekeeping_resume()
1832 tk->tkr_raw.cycle_last = cycle_now; in timekeeping_resume()
1834 tk->ntp_error = 0; in timekeeping_resume()
1836 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); in timekeeping_resume()
1850 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_suspend() local
1871 timekeeping_forward_now(tk); in timekeeping_suspend()
1879 curr_clock = tk->tkr_mono.clock; in timekeeping_suspend()
1880 cycle_now = tk->tkr_mono.cycle_last; in timekeeping_suspend()
1890 delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time); in timekeeping_suspend()
1905 timekeeping_update(tk, TK_MIRROR); in timekeeping_suspend()
1906 halt_fast_timekeeper(tk); in timekeeping_suspend()
1933 static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, in timekeeping_apply_adjustment() argument
1937 s64 interval = tk->cycle_interval; in timekeeping_apply_adjustment()
1996 if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) { in timekeeping_apply_adjustment()
2002 tk->tkr_mono.mult += mult_adj; in timekeeping_apply_adjustment()
2003 tk->xtime_interval += interval; in timekeeping_apply_adjustment()
2004 tk->tkr_mono.xtime_nsec -= offset; in timekeeping_apply_adjustment()
2011 static void timekeeping_adjust(struct timekeeper *tk, s64 offset) in timekeeping_adjust() argument
2019 if (likely(tk->ntp_tick == ntp_tick_length())) { in timekeeping_adjust()
2020 mult = tk->tkr_mono.mult - tk->ntp_err_mult; in timekeeping_adjust()
2022 tk->ntp_tick = ntp_tick_length(); in timekeeping_adjust()
2023 mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) - in timekeeping_adjust()
2024 tk->xtime_remainder, tk->cycle_interval); in timekeeping_adjust()
2033 tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0; in timekeeping_adjust()
2034 mult += tk->ntp_err_mult; in timekeeping_adjust()
2036 timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult); in timekeeping_adjust()
2038 if (unlikely(tk->tkr_mono.clock->maxadj && in timekeeping_adjust()
2039 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult) in timekeeping_adjust()
2040 > tk->tkr_mono.clock->maxadj))) { in timekeeping_adjust()
2043 tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult, in timekeeping_adjust()
2044 (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj); in timekeeping_adjust()
2057 if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) { in timekeeping_adjust()
2058 tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC << in timekeeping_adjust()
2059 tk->tkr_mono.shift; in timekeeping_adjust()
2060 tk->xtime_sec--; in timekeeping_adjust()
2061 tk->skip_second_overflow = 1; in timekeeping_adjust()
2072 static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk) in accumulate_nsecs_to_secs() argument
2074 u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift; in accumulate_nsecs_to_secs()
2077 while (tk->tkr_mono.xtime_nsec >= nsecps) { in accumulate_nsecs_to_secs()
2080 tk->tkr_mono.xtime_nsec -= nsecps; in accumulate_nsecs_to_secs()
2081 tk->xtime_sec++; in accumulate_nsecs_to_secs()
2087 if (unlikely(tk->skip_second_overflow)) { in accumulate_nsecs_to_secs()
2088 tk->skip_second_overflow = 0; in accumulate_nsecs_to_secs()
2093 leap = second_overflow(tk->xtime_sec); in accumulate_nsecs_to_secs()
2097 tk->xtime_sec += leap; in accumulate_nsecs_to_secs()
2101 tk_set_wall_to_mono(tk, in accumulate_nsecs_to_secs()
2102 timespec64_sub(tk->wall_to_monotonic, ts)); in accumulate_nsecs_to_secs()
2104 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); in accumulate_nsecs_to_secs()
2121 static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset, in logarithmic_accumulation() argument
2124 u64 interval = tk->cycle_interval << shift; in logarithmic_accumulation()
2133 tk->tkr_mono.cycle_last += interval; in logarithmic_accumulation()
2134 tk->tkr_raw.cycle_last += interval; in logarithmic_accumulation()
2136 tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift; in logarithmic_accumulation()
2137 *clock_set |= accumulate_nsecs_to_secs(tk); in logarithmic_accumulation()
2140 tk->tkr_raw.xtime_nsec += tk->raw_interval << shift; in logarithmic_accumulation()
2141 snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift; in logarithmic_accumulation()
2142 while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) { in logarithmic_accumulation()
2143 tk->tkr_raw.xtime_nsec -= snsec_per_sec; in logarithmic_accumulation()
2144 tk->raw_sec++; in logarithmic_accumulation()
2148 tk->ntp_error += tk->ntp_tick << shift; in logarithmic_accumulation()
2149 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << in logarithmic_accumulation()
2150 (tk->ntp_error_shift + shift); in logarithmic_accumulation()
2162 struct timekeeper *tk = &shadow_timekeeper; in timekeeping_advance() local
2174 offset = clocksource_delta(tk_clock_read(&tk->tkr_mono), in timekeeping_advance()
2175 tk->tkr_mono.cycle_last, tk->tkr_mono.mask); in timekeeping_advance()
2182 timekeeping_check_update(tk, offset); in timekeeping_advance()
2192 shift = ilog2(offset) - ilog2(tk->cycle_interval); in timekeeping_advance()
2197 while (offset >= tk->cycle_interval) { in timekeeping_advance()
2198 offset = logarithmic_accumulation(tk, offset, shift, in timekeeping_advance()
2200 if (offset < tk->cycle_interval<<shift) in timekeeping_advance()
2205 timekeeping_adjust(tk, offset); in timekeeping_advance()
2211 clock_set |= accumulate_nsecs_to_secs(tk); in timekeeping_advance()
2224 timekeeping_update(tk, clock_set); in timekeeping_advance()
2225 memcpy(real_tk, tk, sizeof(*tk)); in timekeeping_advance()
2257 struct timekeeper *tk = &tk_core.timekeeper; in getboottime64() local
2258 ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot); in getboottime64()
2266 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_coarse_real_ts64() local
2272 *ts = tk_xtime(tk); in ktime_get_coarse_real_ts64()
2279 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_coarse_ts64() local
2286 now = tk_xtime(tk); in ktime_get_coarse_ts64()
2287 mono = tk->wall_to_monotonic; in ktime_get_coarse_ts64()
2320 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_update_offsets_now() local
2328 base = tk->tkr_mono.base; in ktime_get_update_offsets_now()
2329 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_update_offsets_now()
2332 if (*cwsseq != tk->clock_was_set_seq) { in ktime_get_update_offsets_now()
2333 *cwsseq = tk->clock_was_set_seq; in ktime_get_update_offsets_now()
2334 *offs_real = tk->offs_real; in ktime_get_update_offsets_now()
2335 *offs_boot = tk->offs_boot; in ktime_get_update_offsets_now()
2336 *offs_tai = tk->offs_tai; in ktime_get_update_offsets_now()
2340 if (unlikely(base >= tk->next_leap_ktime)) in ktime_get_update_offsets_now()
2341 *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0)); in ktime_get_update_offsets_now()
2433 struct timekeeper *tk = &tk_core.timekeeper; in do_adjtimex() local
2468 orig_tai = tai = tk->tai_offset; in do_adjtimex()
2472 __timekeeping_set_tai_offset(tk, tai); in do_adjtimex()
2473 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); in do_adjtimex()
2476 tk_update_leap_state(tk); in do_adjtimex()