1 /*
2 * Copyright 2006 Andi Kleen, SUSE Labs.
3 * Subject to the GNU Public License, v.2
4 *
5 * Fast user context implementation of clock_gettime, gettimeofday, and time.
6 *
7 * The code should have no internal unresolved relocations.
8 * Check with readelf after changing.
9 * Also alternative() doesn't work.
10 */
11 /*
12 * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
13 */
14
15 /* Disable profiling for userspace code: */
16 #ifndef DISABLE_BRANCH_PROFILING
17 #define DISABLE_BRANCH_PROFILING
18 #endif
19
20 #include <linux/kernel.h>
21 #include <linux/time.h>
22 #include <linux/string.h>
23 #include <asm/io.h>
24 #include <asm/unistd.h>
25 #include <asm/timex.h>
26 #include <asm/clocksource.h>
27 #include <asm/vvar.h>
28
29 #undef TICK_PRIV_BIT
30 #ifdef CONFIG_SPARC64
31 #define TICK_PRIV_BIT (1UL << 63)
32 #else
33 #define TICK_PRIV_BIT (1ULL << 63)
34 #endif
35
36 #ifdef CONFIG_SPARC64
37 #define SYSCALL_STRING \
38 "ta 0x6d;" \
39 "bcs,a 1f;" \
40 " sub %%g0, %%o0, %%o0;" \
41 "1:"
42 #else
43 #define SYSCALL_STRING \
44 "ta 0x10;" \
45 "bcs,a 1f;" \
46 " sub %%g0, %%o0, %%o0;" \
47 "1:"
48 #endif
49
50 #define SYSCALL_CLOBBERS \
51 "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \
52 "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \
53 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \
54 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \
55 "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \
56 "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \
57 "cc", "memory"
58
59 /*
60 * Compute the vvar page's address in the process address space, and return it
61 * as a pointer to the vvar_data.
62 */
63 static notrace noinline struct vvar_data *
get_vvar_data(void)64 get_vvar_data(void)
65 {
66 unsigned long ret;
67
68 /*
69 * vdso data page is the first vDSO page so grab the return address
70 * and move up a page to get to the data page.
71 */
72 ret = (unsigned long)__builtin_return_address(0);
73 ret &= ~(8192 - 1);
74 ret -= 8192;
75
76 return (struct vvar_data *) ret;
77 }
78
79 static notrace long
vdso_fallback_gettime(long clock,struct timespec * ts)80 vdso_fallback_gettime(long clock, struct timespec *ts)
81 {
82 register long num __asm__("g1") = __NR_clock_gettime;
83 register long o0 __asm__("o0") = clock;
84 register long o1 __asm__("o1") = (long) ts;
85
86 __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
87 "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
88 return o0;
89 }
90
91 static notrace __always_inline long
vdso_fallback_gettimeofday(struct timeval * tv,struct timezone * tz)92 vdso_fallback_gettimeofday(struct timeval *tv, struct timezone *tz)
93 {
94 register long num __asm__("g1") = __NR_gettimeofday;
95 register long o0 __asm__("o0") = (long) tv;
96 register long o1 __asm__("o1") = (long) tz;
97
98 __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
99 "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
100 return o0;
101 }
102
103 #ifdef CONFIG_SPARC64
104 static notrace noinline u64
vread_tick(void)105 vread_tick(void) {
106 u64 ret;
107
108 __asm__ __volatile__("rd %%asr24, %0 \n"
109 ".section .vread_tick_patch, \"ax\" \n"
110 "rd %%tick, %0 \n"
111 ".previous \n"
112 : "=&r" (ret));
113 return ret & ~TICK_PRIV_BIT;
114 }
115 #else
116 static notrace noinline u64
vread_tick(void)117 vread_tick(void)
118 {
119 unsigned int lo, hi;
120
121 __asm__ __volatile__("rd %%asr24, %%g1\n\t"
122 "srlx %%g1, 32, %1\n\t"
123 "srl %%g1, 0, %0\n"
124 ".section .vread_tick_patch, \"ax\" \n"
125 "rd %%tick, %%g1\n"
126 ".previous \n"
127 : "=&r" (lo), "=&r" (hi)
128 :
129 : "g1");
130 return lo | ((u64)hi << 32);
131 }
132 #endif
133
134 static notrace inline u64
vgetsns(struct vvar_data * vvar)135 vgetsns(struct vvar_data *vvar)
136 {
137 u64 v;
138 u64 cycles;
139
140 cycles = vread_tick();
141 v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
142 return v * vvar->clock.mult;
143 }
144
145 static notrace noinline int
do_realtime(struct vvar_data * vvar,struct timespec * ts)146 do_realtime(struct vvar_data *vvar, struct timespec *ts)
147 {
148 unsigned long seq;
149 u64 ns;
150
151 ts->tv_nsec = 0;
152 do {
153 seq = vvar_read_begin(vvar);
154 ts->tv_sec = vvar->wall_time_sec;
155 ns = vvar->wall_time_snsec;
156 ns += vgetsns(vvar);
157 ns >>= vvar->clock.shift;
158 } while (unlikely(vvar_read_retry(vvar, seq)));
159
160 timespec_add_ns(ts, ns);
161
162 return 0;
163 }
164
165 static notrace noinline int
do_monotonic(struct vvar_data * vvar,struct timespec * ts)166 do_monotonic(struct vvar_data *vvar, struct timespec *ts)
167 {
168 unsigned long seq;
169 u64 ns;
170
171 ts->tv_nsec = 0;
172 do {
173 seq = vvar_read_begin(vvar);
174 ts->tv_sec = vvar->monotonic_time_sec;
175 ns = vvar->monotonic_time_snsec;
176 ns += vgetsns(vvar);
177 ns >>= vvar->clock.shift;
178 } while (unlikely(vvar_read_retry(vvar, seq)));
179
180 timespec_add_ns(ts, ns);
181
182 return 0;
183 }
184
185 static notrace noinline int
do_realtime_coarse(struct vvar_data * vvar,struct timespec * ts)186 do_realtime_coarse(struct vvar_data *vvar, struct timespec *ts)
187 {
188 unsigned long seq;
189
190 do {
191 seq = vvar_read_begin(vvar);
192 ts->tv_sec = vvar->wall_time_coarse_sec;
193 ts->tv_nsec = vvar->wall_time_coarse_nsec;
194 } while (unlikely(vvar_read_retry(vvar, seq)));
195 return 0;
196 }
197
198 static notrace noinline int
do_monotonic_coarse(struct vvar_data * vvar,struct timespec * ts)199 do_monotonic_coarse(struct vvar_data *vvar, struct timespec *ts)
200 {
201 unsigned long seq;
202
203 do {
204 seq = vvar_read_begin(vvar);
205 ts->tv_sec = vvar->monotonic_time_coarse_sec;
206 ts->tv_nsec = vvar->monotonic_time_coarse_nsec;
207 } while (unlikely(vvar_read_retry(vvar, seq)));
208
209 return 0;
210 }
211
212 notrace int
__vdso_clock_gettime(clockid_t clock,struct timespec * ts)213 __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
214 {
215 struct vvar_data *vvd = get_vvar_data();
216
217 switch (clock) {
218 case CLOCK_REALTIME:
219 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
220 break;
221 return do_realtime(vvd, ts);
222 case CLOCK_MONOTONIC:
223 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
224 break;
225 return do_monotonic(vvd, ts);
226 case CLOCK_REALTIME_COARSE:
227 return do_realtime_coarse(vvd, ts);
228 case CLOCK_MONOTONIC_COARSE:
229 return do_monotonic_coarse(vvd, ts);
230 }
231 /*
232 * Unknown clock ID ? Fall back to the syscall.
233 */
234 return vdso_fallback_gettime(clock, ts);
235 }
236 int
237 clock_gettime(clockid_t, struct timespec *)
238 __attribute__((weak, alias("__vdso_clock_gettime")));
239
240 notrace int
__vdso_gettimeofday(struct timeval * tv,struct timezone * tz)241 __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
242 {
243 struct vvar_data *vvd = get_vvar_data();
244
245 if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
246 if (likely(tv != NULL)) {
247 union tstv_t {
248 struct timespec ts;
249 struct timeval tv;
250 } *tstv = (union tstv_t *) tv;
251 do_realtime(vvd, &tstv->ts);
252 /*
253 * Assign before dividing to ensure that the division is
254 * done in the type of tv_usec, not tv_nsec.
255 *
256 * There cannot be > 1 billion usec in a second:
257 * do_realtime() has already distributed such overflow
258 * into tv_sec. So we can assign it to an int safely.
259 */
260 tstv->tv.tv_usec = tstv->ts.tv_nsec;
261 tstv->tv.tv_usec /= 1000;
262 }
263 if (unlikely(tz != NULL)) {
264 /* Avoid memcpy. Some old compilers fail to inline it */
265 tz->tz_minuteswest = vvd->tz_minuteswest;
266 tz->tz_dsttime = vvd->tz_dsttime;
267 }
268 return 0;
269 }
270 return vdso_fallback_gettimeofday(tv, tz);
271 }
272 int
273 gettimeofday(struct timeval *, struct timezone *)
274 __attribute__((weak, alias("__vdso_gettimeofday")));
275