35 static inline bool vdso_clocksource_ok(const struct vdso_data *vd)  in vdso_clocksource_ok()  argument
37 	return vd->clock_mode != VDSO_CLOCKMODE_NONE;  in vdso_clocksource_ok()
52 	const struct vdso_data *vd = __arch_get_timens_vdso_data();  in do_hres_timens()  local
60 		vd = &vd[CS_HRES_COARSE];  in do_hres_timens()
62 		vd = &vd[CS_RAW];  in do_hres_timens()
63 	vdso_ts = &vd->basetime[clk];  in do_hres_timens()
66 		seq = vdso_read_begin(vd);  in do_hres_timens()
68 		if (unlikely(!vdso_clocksource_ok(vd)))  in do_hres_timens()
71 		cycles = __arch_get_hw_counter(vd->clock_mode, vd);  in do_hres_timens()
75 		last = vd->cycle_last;  in do_hres_timens()
76 		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);  in do_hres_timens()
77 		ns = vdso_shift_ns(ns, vd->shift);  in do_hres_timens()
79 	} while (unlikely(vdso_read_retry(vd, seq)));  in do_hres_timens()
107 static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,  in do_hres()  argument
110 	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];  in do_hres()
130 		while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {  in do_hres()
132 			    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)  in do_hres()
133 				return do_hres_timens(vd, clk, ts);  in do_hres()
138 		if (unlikely(!vdso_clocksource_ok(vd)))  in do_hres()
141 		cycles = __arch_get_hw_counter(vd->clock_mode, vd);  in do_hres()
145 		last = vd->cycle_last;  in do_hres()
146 		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);  in do_hres()
147 		ns = vdso_shift_ns(ns, vd->shift);  in do_hres()
149 	} while (unlikely(vdso_read_retry(vd, seq)));  in do_hres()
165 	const struct vdso_data *vd = __arch_get_timens_vdso_data();  in do_coarse_timens()  local
166 	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];  in do_coarse_timens()
173 		seq = vdso_read_begin(vd);  in do_coarse_timens()
176 	} while (unlikely(vdso_read_retry(vd, seq)));  in do_coarse_timens()
198 static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,  in do_coarse()  argument
201 	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];  in do_coarse()
209 		while ((seq = READ_ONCE(vd->seq)) & 1) {  in do_coarse()
211 			    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)  in do_coarse()
212 				return do_coarse_timens(vd, clk, ts);  in do_coarse()
219 	} while (unlikely(vdso_read_retry(vd, seq)));  in do_coarse()
225 __cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,  in __cvdso_clock_gettime_common()  argument
240 		vd = &vd[CS_HRES_COARSE];  in __cvdso_clock_gettime_common()
242 		return do_coarse(&vd[CS_HRES_COARSE], clock, ts);  in __cvdso_clock_gettime_common()
244 		vd = &vd[CS_RAW];  in __cvdso_clock_gettime_common()
248 	return do_hres(vd, clock, ts);  in __cvdso_clock_gettime_common()
252 __cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,  in __cvdso_clock_gettime_data()  argument
255 	int ret = __cvdso_clock_gettime_common(vd, clock, ts);  in __cvdso_clock_gettime_data()
270 __cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,  in __cvdso_clock_gettime32_data()  argument
276 	ret = __cvdso_clock_gettime_common(vd, clock, &ts);  in __cvdso_clock_gettime32_data()
296 __cvdso_gettimeofday_data(const struct vdso_data *vd,  in __cvdso_gettimeofday_data()  argument
303 		if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))  in __cvdso_gettimeofday_data()
312 		    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)  in __cvdso_gettimeofday_data()
313 			vd = __arch_get_timens_vdso_data();  in __cvdso_gettimeofday_data()
315 		tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;  in __cvdso_gettimeofday_data()
316 		tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;  in __cvdso_gettimeofday_data()
330 __cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time)  in __cvdso_time_data()  argument
335 	    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)  in __cvdso_time_data()
336 		vd = __arch_get_timens_vdso_data();  in __cvdso_time_data()
338 	t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);  in __cvdso_time_data()
354 int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,  in __cvdso_clock_getres_common()  argument
365 	    vd->clock_mode == VDSO_CLOCKMODE_TIMENS)  in __cvdso_clock_getres_common()
366 		vd = __arch_get_timens_vdso_data();  in __cvdso_clock_getres_common()
377 		ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);  in __cvdso_clock_getres_common()
395 int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,  in __cvdso_clock_getres_data()  argument
398 	int ret = __cvdso_clock_getres_common(vd, clock, res);  in __cvdso_clock_getres_data()
413 __cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,  in __cvdso_clock_getres_time32_data()  argument
419 	ret = __cvdso_clock_getres_common(vd, clock, &ts);  in __cvdso_clock_getres_time32_data()