1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2
3 #define RSEQ_SIG 0x53053053
4
5 #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory")
6 #define rseq_smp_rmb() rseq_smp_mb()
7 #define rseq_smp_wmb() rseq_smp_mb()
8
9 #define rseq_smp_load_acquire(p) \
10 __extension__ ({ \
11 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
12 rseq_barrier(); \
13 ____p1; \
14 })
15
16 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
17
18 #define rseq_smp_store_release(p, v) \
19 do { \
20 rseq_barrier(); \
21 RSEQ_WRITE_ONCE(*p, v); \
22 } while (0)
23
24 #ifdef RSEQ_SKIP_FASTPATH
25 #include "rseq-skip.h"
26 #else /* !RSEQ_SKIP_FASTPATH */
27
28 #ifdef __s390x__
29
30 #define LONG_L "lg"
31 #define LONG_S "stg"
32 #define LONG_LT_R "ltgr"
33 #define LONG_CMP "cg"
34 #define LONG_CMP_R "cgr"
35 #define LONG_ADDI "aghi"
36 #define LONG_ADD_R "agr"
37
38 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
39 start_ip, post_commit_offset, abort_ip) \
40 ".pushsection __rseq_table, \"aw\"\n\t" \
41 ".balign 32\n\t" \
42 __rseq_str(label) ":\n\t" \
43 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
44 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
45 ".popsection\n\t"
46
47 #elif __s390__
48
49 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
50 start_ip, post_commit_offset, abort_ip) \
51 ".pushsection __rseq_table, \"aw\"\n\t" \
52 ".balign 32\n\t" \
53 __rseq_str(label) ":\n\t" \
54 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
55 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
56 ".popsection\n\t"
57
58 #define LONG_L "l"
59 #define LONG_S "st"
60 #define LONG_LT_R "ltr"
61 #define LONG_CMP "c"
62 #define LONG_CMP_R "cr"
63 #define LONG_ADDI "ahi"
64 #define LONG_ADD_R "ar"
65
66 #endif
67
68 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
69 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
70 (post_commit_ip - start_ip), abort_ip)
71
72 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
73 RSEQ_INJECT_ASM(1) \
74 "larl %%r0, " __rseq_str(cs_label) "\n\t" \
75 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
76 __rseq_str(label) ":\n\t"
77
78 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
79 RSEQ_INJECT_ASM(2) \
80 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
81 "jnz " __rseq_str(label) "\n\t"
82
83 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
84 ".pushsection __rseq_failure, \"ax\"\n\t" \
85 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
86 __rseq_str(label) ":\n\t" \
87 teardown \
88 "j %l[" __rseq_str(abort_label) "]\n\t" \
89 ".popsection\n\t"
90
91 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
92 ".pushsection __rseq_failure, \"ax\"\n\t" \
93 __rseq_str(label) ":\n\t" \
94 teardown \
95 "j %l[" __rseq_str(cmpfail_label) "]\n\t" \
96 ".popsection\n\t"
97
98 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)99 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
100 {
101 RSEQ_INJECT_C(9)
102
103 __asm__ __volatile__ goto (
104 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
105 /* Start rseq by storing table entry pointer into rseq_cs. */
106 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
107 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
108 RSEQ_INJECT_ASM(3)
109 LONG_CMP " %[expect], %[v]\n\t"
110 "jnz %l[cmpfail]\n\t"
111 RSEQ_INJECT_ASM(4)
112 #ifdef RSEQ_COMPARE_TWICE
113 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
114 LONG_CMP " %[expect], %[v]\n\t"
115 "jnz %l[error2]\n\t"
116 #endif
117 /* final store */
118 LONG_S " %[newv], %[v]\n\t"
119 "2:\n\t"
120 RSEQ_INJECT_ASM(5)
121 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
122 : /* gcc asm goto does not allow outputs */
123 : [cpu_id] "r" (cpu),
124 [current_cpu_id] "m" (__rseq_abi.cpu_id),
125 [rseq_cs] "m" (__rseq_abi.rseq_cs),
126 [v] "m" (*v),
127 [expect] "r" (expect),
128 [newv] "r" (newv)
129 RSEQ_INJECT_INPUT
130 : "memory", "cc", "r0"
131 RSEQ_INJECT_CLOBBER
132 : abort, cmpfail
133 #ifdef RSEQ_COMPARE_TWICE
134 , error1, error2
135 #endif
136 );
137 return 0;
138 abort:
139 RSEQ_INJECT_FAILED
140 return -1;
141 cmpfail:
142 return 1;
143 #ifdef RSEQ_COMPARE_TWICE
144 error1:
145 rseq_bug("cpu_id comparison failed");
146 error2:
147 rseq_bug("expected value comparison failed");
148 #endif
149 }
150
151 /*
152 * Compare @v against @expectnot. When it does _not_ match, load @v
153 * into @load, and store the content of *@v + voffp into @v.
154 */
155 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)156 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
157 off_t voffp, intptr_t *load, int cpu)
158 {
159 RSEQ_INJECT_C(9)
160
161 __asm__ __volatile__ goto (
162 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
163 /* Start rseq by storing table entry pointer into rseq_cs. */
164 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
165 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
166 RSEQ_INJECT_ASM(3)
167 LONG_L " %%r1, %[v]\n\t"
168 LONG_CMP_R " %%r1, %[expectnot]\n\t"
169 "je %l[cmpfail]\n\t"
170 RSEQ_INJECT_ASM(4)
171 #ifdef RSEQ_COMPARE_TWICE
172 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
173 LONG_L " %%r1, %[v]\n\t"
174 LONG_CMP_R " %%r1, %[expectnot]\n\t"
175 "je %l[error2]\n\t"
176 #endif
177 LONG_S " %%r1, %[load]\n\t"
178 LONG_ADD_R " %%r1, %[voffp]\n\t"
179 LONG_L " %%r1, 0(%%r1)\n\t"
180 /* final store */
181 LONG_S " %%r1, %[v]\n\t"
182 "2:\n\t"
183 RSEQ_INJECT_ASM(5)
184 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
185 : /* gcc asm goto does not allow outputs */
186 : [cpu_id] "r" (cpu),
187 [current_cpu_id] "m" (__rseq_abi.cpu_id),
188 [rseq_cs] "m" (__rseq_abi.rseq_cs),
189 /* final store input */
190 [v] "m" (*v),
191 [expectnot] "r" (expectnot),
192 [voffp] "r" (voffp),
193 [load] "m" (*load)
194 RSEQ_INJECT_INPUT
195 : "memory", "cc", "r0", "r1"
196 RSEQ_INJECT_CLOBBER
197 : abort, cmpfail
198 #ifdef RSEQ_COMPARE_TWICE
199 , error1, error2
200 #endif
201 );
202 return 0;
203 abort:
204 RSEQ_INJECT_FAILED
205 return -1;
206 cmpfail:
207 return 1;
208 #ifdef RSEQ_COMPARE_TWICE
209 error1:
210 rseq_bug("cpu_id comparison failed");
211 error2:
212 rseq_bug("expected value comparison failed");
213 #endif
214 }
215
216 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)217 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
218 {
219 RSEQ_INJECT_C(9)
220
221 __asm__ __volatile__ goto (
222 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
223 /* Start rseq by storing table entry pointer into rseq_cs. */
224 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
225 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
226 RSEQ_INJECT_ASM(3)
227 #ifdef RSEQ_COMPARE_TWICE
228 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
229 #endif
230 LONG_L " %%r0, %[v]\n\t"
231 LONG_ADD_R " %%r0, %[count]\n\t"
232 /* final store */
233 LONG_S " %%r0, %[v]\n\t"
234 "2:\n\t"
235 RSEQ_INJECT_ASM(4)
236 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
237 : /* gcc asm goto does not allow outputs */
238 : [cpu_id] "r" (cpu),
239 [current_cpu_id] "m" (__rseq_abi.cpu_id),
240 [rseq_cs] "m" (__rseq_abi.rseq_cs),
241 /* final store input */
242 [v] "m" (*v),
243 [count] "r" (count)
244 RSEQ_INJECT_INPUT
245 : "memory", "cc", "r0"
246 RSEQ_INJECT_CLOBBER
247 : abort
248 #ifdef RSEQ_COMPARE_TWICE
249 , error1
250 #endif
251 );
252 return 0;
253 abort:
254 RSEQ_INJECT_FAILED
255 return -1;
256 #ifdef RSEQ_COMPARE_TWICE
257 error1:
258 rseq_bug("cpu_id comparison failed");
259 #endif
260 }
261
262 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)263 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
264 intptr_t *v2, intptr_t newv2,
265 intptr_t newv, int cpu)
266 {
267 RSEQ_INJECT_C(9)
268
269 __asm__ __volatile__ goto (
270 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
271 /* Start rseq by storing table entry pointer into rseq_cs. */
272 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
273 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
274 RSEQ_INJECT_ASM(3)
275 LONG_CMP " %[expect], %[v]\n\t"
276 "jnz %l[cmpfail]\n\t"
277 RSEQ_INJECT_ASM(4)
278 #ifdef RSEQ_COMPARE_TWICE
279 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
280 LONG_CMP " %[expect], %[v]\n\t"
281 "jnz %l[error2]\n\t"
282 #endif
283 /* try store */
284 LONG_S " %[newv2], %[v2]\n\t"
285 RSEQ_INJECT_ASM(5)
286 /* final store */
287 LONG_S " %[newv], %[v]\n\t"
288 "2:\n\t"
289 RSEQ_INJECT_ASM(6)
290 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
291 : /* gcc asm goto does not allow outputs */
292 : [cpu_id] "r" (cpu),
293 [current_cpu_id] "m" (__rseq_abi.cpu_id),
294 [rseq_cs] "m" (__rseq_abi.rseq_cs),
295 /* try store input */
296 [v2] "m" (*v2),
297 [newv2] "r" (newv2),
298 /* final store input */
299 [v] "m" (*v),
300 [expect] "r" (expect),
301 [newv] "r" (newv)
302 RSEQ_INJECT_INPUT
303 : "memory", "cc", "r0"
304 RSEQ_INJECT_CLOBBER
305 : abort, cmpfail
306 #ifdef RSEQ_COMPARE_TWICE
307 , error1, error2
308 #endif
309 );
310 return 0;
311 abort:
312 RSEQ_INJECT_FAILED
313 return -1;
314 cmpfail:
315 return 1;
316 #ifdef RSEQ_COMPARE_TWICE
317 error1:
318 rseq_bug("cpu_id comparison failed");
319 error2:
320 rseq_bug("expected value comparison failed");
321 #endif
322 }
323
324 /* s390 is TSO. */
325 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)326 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
327 intptr_t *v2, intptr_t newv2,
328 intptr_t newv, int cpu)
329 {
330 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
331 }
332
333 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)334 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
335 intptr_t *v2, intptr_t expect2,
336 intptr_t newv, int cpu)
337 {
338 RSEQ_INJECT_C(9)
339
340 __asm__ __volatile__ goto (
341 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
342 /* Start rseq by storing table entry pointer into rseq_cs. */
343 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
344 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
345 RSEQ_INJECT_ASM(3)
346 LONG_CMP " %[expect], %[v]\n\t"
347 "jnz %l[cmpfail]\n\t"
348 RSEQ_INJECT_ASM(4)
349 LONG_CMP " %[expect2], %[v2]\n\t"
350 "jnz %l[cmpfail]\n\t"
351 RSEQ_INJECT_ASM(5)
352 #ifdef RSEQ_COMPARE_TWICE
353 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
354 LONG_CMP " %[expect], %[v]\n\t"
355 "jnz %l[error2]\n\t"
356 LONG_CMP " %[expect2], %[v2]\n\t"
357 "jnz %l[error3]\n\t"
358 #endif
359 /* final store */
360 LONG_S " %[newv], %[v]\n\t"
361 "2:\n\t"
362 RSEQ_INJECT_ASM(6)
363 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
364 : /* gcc asm goto does not allow outputs */
365 : [cpu_id] "r" (cpu),
366 [current_cpu_id] "m" (__rseq_abi.cpu_id),
367 [rseq_cs] "m" (__rseq_abi.rseq_cs),
368 /* cmp2 input */
369 [v2] "m" (*v2),
370 [expect2] "r" (expect2),
371 /* final store input */
372 [v] "m" (*v),
373 [expect] "r" (expect),
374 [newv] "r" (newv)
375 RSEQ_INJECT_INPUT
376 : "memory", "cc", "r0"
377 RSEQ_INJECT_CLOBBER
378 : abort, cmpfail
379 #ifdef RSEQ_COMPARE_TWICE
380 , error1, error2, error3
381 #endif
382 );
383 return 0;
384 abort:
385 RSEQ_INJECT_FAILED
386 return -1;
387 cmpfail:
388 return 1;
389 #ifdef RSEQ_COMPARE_TWICE
390 error1:
391 rseq_bug("cpu_id comparison failed");
392 error2:
393 rseq_bug("1st expected value comparison failed");
394 error3:
395 rseq_bug("2nd expected value comparison failed");
396 #endif
397 }
398
399 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)400 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
401 void *dst, void *src, size_t len,
402 intptr_t newv, int cpu)
403 {
404 uint64_t rseq_scratch[3];
405
406 RSEQ_INJECT_C(9)
407
408 __asm__ __volatile__ goto (
409 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
410 LONG_S " %[src], %[rseq_scratch0]\n\t"
411 LONG_S " %[dst], %[rseq_scratch1]\n\t"
412 LONG_S " %[len], %[rseq_scratch2]\n\t"
413 /* Start rseq by storing table entry pointer into rseq_cs. */
414 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
415 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
416 RSEQ_INJECT_ASM(3)
417 LONG_CMP " %[expect], %[v]\n\t"
418 "jnz 5f\n\t"
419 RSEQ_INJECT_ASM(4)
420 #ifdef RSEQ_COMPARE_TWICE
421 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
422 LONG_CMP " %[expect], %[v]\n\t"
423 "jnz 7f\n\t"
424 #endif
425 /* try memcpy */
426 LONG_LT_R " %[len], %[len]\n\t"
427 "jz 333f\n\t"
428 "222:\n\t"
429 "ic %%r0,0(%[src])\n\t"
430 "stc %%r0,0(%[dst])\n\t"
431 LONG_ADDI " %[src], 1\n\t"
432 LONG_ADDI " %[dst], 1\n\t"
433 LONG_ADDI " %[len], -1\n\t"
434 "jnz 222b\n\t"
435 "333:\n\t"
436 RSEQ_INJECT_ASM(5)
437 /* final store */
438 LONG_S " %[newv], %[v]\n\t"
439 "2:\n\t"
440 RSEQ_INJECT_ASM(6)
441 /* teardown */
442 LONG_L " %[len], %[rseq_scratch2]\n\t"
443 LONG_L " %[dst], %[rseq_scratch1]\n\t"
444 LONG_L " %[src], %[rseq_scratch0]\n\t"
445 RSEQ_ASM_DEFINE_ABORT(4,
446 LONG_L " %[len], %[rseq_scratch2]\n\t"
447 LONG_L " %[dst], %[rseq_scratch1]\n\t"
448 LONG_L " %[src], %[rseq_scratch0]\n\t",
449 abort)
450 RSEQ_ASM_DEFINE_CMPFAIL(5,
451 LONG_L " %[len], %[rseq_scratch2]\n\t"
452 LONG_L " %[dst], %[rseq_scratch1]\n\t"
453 LONG_L " %[src], %[rseq_scratch0]\n\t",
454 cmpfail)
455 #ifdef RSEQ_COMPARE_TWICE
456 RSEQ_ASM_DEFINE_CMPFAIL(6,
457 LONG_L " %[len], %[rseq_scratch2]\n\t"
458 LONG_L " %[dst], %[rseq_scratch1]\n\t"
459 LONG_L " %[src], %[rseq_scratch0]\n\t",
460 error1)
461 RSEQ_ASM_DEFINE_CMPFAIL(7,
462 LONG_L " %[len], %[rseq_scratch2]\n\t"
463 LONG_L " %[dst], %[rseq_scratch1]\n\t"
464 LONG_L " %[src], %[rseq_scratch0]\n\t",
465 error2)
466 #endif
467 : /* gcc asm goto does not allow outputs */
468 : [cpu_id] "r" (cpu),
469 [current_cpu_id] "m" (__rseq_abi.cpu_id),
470 [rseq_cs] "m" (__rseq_abi.rseq_cs),
471 /* final store input */
472 [v] "m" (*v),
473 [expect] "r" (expect),
474 [newv] "r" (newv),
475 /* try memcpy input */
476 [dst] "r" (dst),
477 [src] "r" (src),
478 [len] "r" (len),
479 [rseq_scratch0] "m" (rseq_scratch[0]),
480 [rseq_scratch1] "m" (rseq_scratch[1]),
481 [rseq_scratch2] "m" (rseq_scratch[2])
482 RSEQ_INJECT_INPUT
483 : "memory", "cc", "r0"
484 RSEQ_INJECT_CLOBBER
485 : abort, cmpfail
486 #ifdef RSEQ_COMPARE_TWICE
487 , error1, error2
488 #endif
489 );
490 return 0;
491 abort:
492 RSEQ_INJECT_FAILED
493 return -1;
494 cmpfail:
495 return 1;
496 #ifdef RSEQ_COMPARE_TWICE
497 error1:
498 rseq_bug("cpu_id comparison failed");
499 error2:
500 rseq_bug("expected value comparison failed");
501 #endif
502 }
503
504 /* s390 is TSO. */
505 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)506 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
507 void *dst, void *src, size_t len,
508 intptr_t newv, int cpu)
509 {
510 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
511 newv, cpu);
512 }
513 #endif /* !RSEQ_SKIP_FASTPATH */
514