1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * rseq-arm.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8 #define RSEQ_SIG 0x53053053
9
10 #define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
11 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
12 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
13
14 #define rseq_smp_load_acquire(p) \
15 __extension__ ({ \
16 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
17 rseq_smp_mb(); \
18 ____p1; \
19 })
20
21 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
22
23 #define rseq_smp_store_release(p, v) \
24 do { \
25 rseq_smp_mb(); \
26 RSEQ_WRITE_ONCE(*p, v); \
27 } while (0)
28
29 #ifdef RSEQ_SKIP_FASTPATH
30 #include "rseq-skip.h"
31 #else /* !RSEQ_SKIP_FASTPATH */
32
33 #define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \
34 post_commit_offset, abort_ip) \
35 ".pushsection __rseq_table, \"aw\"\n\t" \
36 ".balign 32\n\t" \
37 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
38 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
39 ".popsection\n\t"
40
41 #define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \
42 __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \
43 (post_commit_ip - start_ip), abort_ip)
44
45 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
46 RSEQ_INJECT_ASM(1) \
47 "adr r0, " __rseq_str(cs_label) "\n\t" \
48 "str r0, %[" __rseq_str(rseq_cs) "]\n\t" \
49 __rseq_str(label) ":\n\t"
50
51 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
52 RSEQ_INJECT_ASM(2) \
53 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \
54 "cmp %[" __rseq_str(cpu_id) "], r0\n\t" \
55 "bne " __rseq_str(label) "\n\t"
56
57 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
58 abort_label, version, flags, \
59 start_ip, post_commit_offset, abort_ip) \
60 ".balign 32\n\t" \
61 __rseq_str(table_label) ":\n\t" \
62 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
63 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
64 ".word " __rseq_str(RSEQ_SIG) "\n\t" \
65 __rseq_str(label) ":\n\t" \
66 teardown \
67 "b %l[" __rseq_str(abort_label) "]\n\t"
68
69 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
70 start_ip, post_commit_ip, abort_ip) \
71 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
72 abort_label, 0x0, 0x0, start_ip, \
73 (post_commit_ip - start_ip), abort_ip)
74
75 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
76 __rseq_str(label) ":\n\t" \
77 teardown \
78 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
79
80 #define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("")
81
82 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)83 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
84 {
85 RSEQ_INJECT_C(9)
86
87 rseq_workaround_gcc_asm_size_guess();
88 __asm__ __volatile__ goto (
89 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
90 /* Start rseq by storing table entry pointer into rseq_cs. */
91 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
92 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
93 RSEQ_INJECT_ASM(3)
94 "ldr r0, %[v]\n\t"
95 "cmp %[expect], r0\n\t"
96 "bne %l[cmpfail]\n\t"
97 RSEQ_INJECT_ASM(4)
98 #ifdef RSEQ_COMPARE_TWICE
99 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
100 "ldr r0, %[v]\n\t"
101 "cmp %[expect], r0\n\t"
102 "bne %l[error2]\n\t"
103 #endif
104 /* final store */
105 "str %[newv], %[v]\n\t"
106 "2:\n\t"
107 RSEQ_INJECT_ASM(5)
108 "b 5f\n\t"
109 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
110 "5:\n\t"
111 : /* gcc asm goto does not allow outputs */
112 : [cpu_id] "r" (cpu),
113 [current_cpu_id] "m" (__rseq_abi.cpu_id),
114 [rseq_cs] "m" (__rseq_abi.rseq_cs),
115 [v] "m" (*v),
116 [expect] "r" (expect),
117 [newv] "r" (newv)
118 RSEQ_INJECT_INPUT
119 : "r0", "memory", "cc"
120 RSEQ_INJECT_CLOBBER
121 : abort, cmpfail
122 #ifdef RSEQ_COMPARE_TWICE
123 , error1, error2
124 #endif
125 );
126 rseq_workaround_gcc_asm_size_guess();
127 return 0;
128 abort:
129 rseq_workaround_gcc_asm_size_guess();
130 RSEQ_INJECT_FAILED
131 return -1;
132 cmpfail:
133 rseq_workaround_gcc_asm_size_guess();
134 return 1;
135 #ifdef RSEQ_COMPARE_TWICE
136 error1:
137 rseq_bug("cpu_id comparison failed");
138 error2:
139 rseq_bug("expected value comparison failed");
140 #endif
141 }
142
143 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)144 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
145 off_t voffp, intptr_t *load, int cpu)
146 {
147 RSEQ_INJECT_C(9)
148
149 rseq_workaround_gcc_asm_size_guess();
150 __asm__ __volatile__ goto (
151 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
152 /* Start rseq by storing table entry pointer into rseq_cs. */
153 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
154 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
155 RSEQ_INJECT_ASM(3)
156 "ldr r0, %[v]\n\t"
157 "cmp %[expectnot], r0\n\t"
158 "beq %l[cmpfail]\n\t"
159 RSEQ_INJECT_ASM(4)
160 #ifdef RSEQ_COMPARE_TWICE
161 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
162 "ldr r0, %[v]\n\t"
163 "cmp %[expectnot], r0\n\t"
164 "beq %l[error2]\n\t"
165 #endif
166 "str r0, %[load]\n\t"
167 "add r0, %[voffp]\n\t"
168 "ldr r0, [r0]\n\t"
169 /* final store */
170 "str r0, %[v]\n\t"
171 "2:\n\t"
172 RSEQ_INJECT_ASM(5)
173 "b 5f\n\t"
174 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
175 "5:\n\t"
176 : /* gcc asm goto does not allow outputs */
177 : [cpu_id] "r" (cpu),
178 [current_cpu_id] "m" (__rseq_abi.cpu_id),
179 [rseq_cs] "m" (__rseq_abi.rseq_cs),
180 /* final store input */
181 [v] "m" (*v),
182 [expectnot] "r" (expectnot),
183 [voffp] "Ir" (voffp),
184 [load] "m" (*load)
185 RSEQ_INJECT_INPUT
186 : "r0", "memory", "cc"
187 RSEQ_INJECT_CLOBBER
188 : abort, cmpfail
189 #ifdef RSEQ_COMPARE_TWICE
190 , error1, error2
191 #endif
192 );
193 rseq_workaround_gcc_asm_size_guess();
194 return 0;
195 abort:
196 rseq_workaround_gcc_asm_size_guess();
197 RSEQ_INJECT_FAILED
198 return -1;
199 cmpfail:
200 rseq_workaround_gcc_asm_size_guess();
201 return 1;
202 #ifdef RSEQ_COMPARE_TWICE
203 error1:
204 rseq_bug("cpu_id comparison failed");
205 error2:
206 rseq_bug("expected value comparison failed");
207 #endif
208 }
209
210 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)211 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
212 {
213 RSEQ_INJECT_C(9)
214
215 rseq_workaround_gcc_asm_size_guess();
216 __asm__ __volatile__ goto (
217 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
218 /* Start rseq by storing table entry pointer into rseq_cs. */
219 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
220 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
221 RSEQ_INJECT_ASM(3)
222 #ifdef RSEQ_COMPARE_TWICE
223 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
224 #endif
225 "ldr r0, %[v]\n\t"
226 "add r0, %[count]\n\t"
227 /* final store */
228 "str r0, %[v]\n\t"
229 "2:\n\t"
230 RSEQ_INJECT_ASM(4)
231 "b 5f\n\t"
232 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
233 "5:\n\t"
234 : /* gcc asm goto does not allow outputs */
235 : [cpu_id] "r" (cpu),
236 [current_cpu_id] "m" (__rseq_abi.cpu_id),
237 [rseq_cs] "m" (__rseq_abi.rseq_cs),
238 [v] "m" (*v),
239 [count] "Ir" (count)
240 RSEQ_INJECT_INPUT
241 : "r0", "memory", "cc"
242 RSEQ_INJECT_CLOBBER
243 : abort
244 #ifdef RSEQ_COMPARE_TWICE
245 , error1
246 #endif
247 );
248 rseq_workaround_gcc_asm_size_guess();
249 return 0;
250 abort:
251 rseq_workaround_gcc_asm_size_guess();
252 RSEQ_INJECT_FAILED
253 return -1;
254 #ifdef RSEQ_COMPARE_TWICE
255 error1:
256 rseq_bug("cpu_id comparison failed");
257 #endif
258 }
259
260 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)261 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
262 intptr_t *v2, intptr_t newv2,
263 intptr_t newv, int cpu)
264 {
265 RSEQ_INJECT_C(9)
266
267 rseq_workaround_gcc_asm_size_guess();
268 __asm__ __volatile__ goto (
269 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
270 /* Start rseq by storing table entry pointer into rseq_cs. */
271 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
272 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
273 RSEQ_INJECT_ASM(3)
274 "ldr r0, %[v]\n\t"
275 "cmp %[expect], r0\n\t"
276 "bne %l[cmpfail]\n\t"
277 RSEQ_INJECT_ASM(4)
278 #ifdef RSEQ_COMPARE_TWICE
279 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
280 "ldr r0, %[v]\n\t"
281 "cmp %[expect], r0\n\t"
282 "bne %l[error2]\n\t"
283 #endif
284 /* try store */
285 "str %[newv2], %[v2]\n\t"
286 RSEQ_INJECT_ASM(5)
287 /* final store */
288 "str %[newv], %[v]\n\t"
289 "2:\n\t"
290 RSEQ_INJECT_ASM(6)
291 "b 5f\n\t"
292 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
293 "5:\n\t"
294 : /* gcc asm goto does not allow outputs */
295 : [cpu_id] "r" (cpu),
296 [current_cpu_id] "m" (__rseq_abi.cpu_id),
297 [rseq_cs] "m" (__rseq_abi.rseq_cs),
298 /* try store input */
299 [v2] "m" (*v2),
300 [newv2] "r" (newv2),
301 /* final store input */
302 [v] "m" (*v),
303 [expect] "r" (expect),
304 [newv] "r" (newv)
305 RSEQ_INJECT_INPUT
306 : "r0", "memory", "cc"
307 RSEQ_INJECT_CLOBBER
308 : abort, cmpfail
309 #ifdef RSEQ_COMPARE_TWICE
310 , error1, error2
311 #endif
312 );
313 rseq_workaround_gcc_asm_size_guess();
314 return 0;
315 abort:
316 rseq_workaround_gcc_asm_size_guess();
317 RSEQ_INJECT_FAILED
318 return -1;
319 cmpfail:
320 rseq_workaround_gcc_asm_size_guess();
321 return 1;
322 #ifdef RSEQ_COMPARE_TWICE
323 error1:
324 rseq_bug("cpu_id comparison failed");
325 error2:
326 rseq_bug("expected value comparison failed");
327 #endif
328 }
329
330 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)331 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
332 intptr_t *v2, intptr_t newv2,
333 intptr_t newv, int cpu)
334 {
335 RSEQ_INJECT_C(9)
336
337 rseq_workaround_gcc_asm_size_guess();
338 __asm__ __volatile__ goto (
339 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
340 /* Start rseq by storing table entry pointer into rseq_cs. */
341 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
342 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
343 RSEQ_INJECT_ASM(3)
344 "ldr r0, %[v]\n\t"
345 "cmp %[expect], r0\n\t"
346 "bne %l[cmpfail]\n\t"
347 RSEQ_INJECT_ASM(4)
348 #ifdef RSEQ_COMPARE_TWICE
349 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
350 "ldr r0, %[v]\n\t"
351 "cmp %[expect], r0\n\t"
352 "bne %l[error2]\n\t"
353 #endif
354 /* try store */
355 "str %[newv2], %[v2]\n\t"
356 RSEQ_INJECT_ASM(5)
357 "dmb\n\t" /* full mb provides store-release */
358 /* final store */
359 "str %[newv], %[v]\n\t"
360 "2:\n\t"
361 RSEQ_INJECT_ASM(6)
362 "b 5f\n\t"
363 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
364 "5:\n\t"
365 : /* gcc asm goto does not allow outputs */
366 : [cpu_id] "r" (cpu),
367 [current_cpu_id] "m" (__rseq_abi.cpu_id),
368 [rseq_cs] "m" (__rseq_abi.rseq_cs),
369 /* try store input */
370 [v2] "m" (*v2),
371 [newv2] "r" (newv2),
372 /* final store input */
373 [v] "m" (*v),
374 [expect] "r" (expect),
375 [newv] "r" (newv)
376 RSEQ_INJECT_INPUT
377 : "r0", "memory", "cc"
378 RSEQ_INJECT_CLOBBER
379 : abort, cmpfail
380 #ifdef RSEQ_COMPARE_TWICE
381 , error1, error2
382 #endif
383 );
384 rseq_workaround_gcc_asm_size_guess();
385 return 0;
386 abort:
387 rseq_workaround_gcc_asm_size_guess();
388 RSEQ_INJECT_FAILED
389 return -1;
390 cmpfail:
391 rseq_workaround_gcc_asm_size_guess();
392 return 1;
393 #ifdef RSEQ_COMPARE_TWICE
394 error1:
395 rseq_bug("cpu_id comparison failed");
396 error2:
397 rseq_bug("expected value comparison failed");
398 #endif
399 }
400
401 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)402 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
403 intptr_t *v2, intptr_t expect2,
404 intptr_t newv, int cpu)
405 {
406 RSEQ_INJECT_C(9)
407
408 rseq_workaround_gcc_asm_size_guess();
409 __asm__ __volatile__ goto (
410 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
411 /* Start rseq by storing table entry pointer into rseq_cs. */
412 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
413 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
414 RSEQ_INJECT_ASM(3)
415 "ldr r0, %[v]\n\t"
416 "cmp %[expect], r0\n\t"
417 "bne %l[cmpfail]\n\t"
418 RSEQ_INJECT_ASM(4)
419 "ldr r0, %[v2]\n\t"
420 "cmp %[expect2], r0\n\t"
421 "bne %l[cmpfail]\n\t"
422 RSEQ_INJECT_ASM(5)
423 #ifdef RSEQ_COMPARE_TWICE
424 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
425 "ldr r0, %[v]\n\t"
426 "cmp %[expect], r0\n\t"
427 "bne %l[error2]\n\t"
428 "ldr r0, %[v2]\n\t"
429 "cmp %[expect2], r0\n\t"
430 "bne %l[error3]\n\t"
431 #endif
432 /* final store */
433 "str %[newv], %[v]\n\t"
434 "2:\n\t"
435 RSEQ_INJECT_ASM(6)
436 "b 5f\n\t"
437 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
438 "5:\n\t"
439 : /* gcc asm goto does not allow outputs */
440 : [cpu_id] "r" (cpu),
441 [current_cpu_id] "m" (__rseq_abi.cpu_id),
442 [rseq_cs] "m" (__rseq_abi.rseq_cs),
443 /* cmp2 input */
444 [v2] "m" (*v2),
445 [expect2] "r" (expect2),
446 /* final store input */
447 [v] "m" (*v),
448 [expect] "r" (expect),
449 [newv] "r" (newv)
450 RSEQ_INJECT_INPUT
451 : "r0", "memory", "cc"
452 RSEQ_INJECT_CLOBBER
453 : abort, cmpfail
454 #ifdef RSEQ_COMPARE_TWICE
455 , error1, error2, error3
456 #endif
457 );
458 rseq_workaround_gcc_asm_size_guess();
459 return 0;
460 abort:
461 rseq_workaround_gcc_asm_size_guess();
462 RSEQ_INJECT_FAILED
463 return -1;
464 cmpfail:
465 rseq_workaround_gcc_asm_size_guess();
466 return 1;
467 #ifdef RSEQ_COMPARE_TWICE
468 error1:
469 rseq_bug("cpu_id comparison failed");
470 error2:
471 rseq_bug("1st expected value comparison failed");
472 error3:
473 rseq_bug("2nd expected value comparison failed");
474 #endif
475 }
476
477 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)478 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
479 void *dst, void *src, size_t len,
480 intptr_t newv, int cpu)
481 {
482 uint32_t rseq_scratch[3];
483
484 RSEQ_INJECT_C(9)
485
486 rseq_workaround_gcc_asm_size_guess();
487 __asm__ __volatile__ goto (
488 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
489 "str %[src], %[rseq_scratch0]\n\t"
490 "str %[dst], %[rseq_scratch1]\n\t"
491 "str %[len], %[rseq_scratch2]\n\t"
492 /* Start rseq by storing table entry pointer into rseq_cs. */
493 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
494 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
495 RSEQ_INJECT_ASM(3)
496 "ldr r0, %[v]\n\t"
497 "cmp %[expect], r0\n\t"
498 "bne 5f\n\t"
499 RSEQ_INJECT_ASM(4)
500 #ifdef RSEQ_COMPARE_TWICE
501 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
502 "ldr r0, %[v]\n\t"
503 "cmp %[expect], r0\n\t"
504 "bne 7f\n\t"
505 #endif
506 /* try memcpy */
507 "cmp %[len], #0\n\t" \
508 "beq 333f\n\t" \
509 "222:\n\t" \
510 "ldrb %%r0, [%[src]]\n\t" \
511 "strb %%r0, [%[dst]]\n\t" \
512 "adds %[src], #1\n\t" \
513 "adds %[dst], #1\n\t" \
514 "subs %[len], #1\n\t" \
515 "bne 222b\n\t" \
516 "333:\n\t" \
517 RSEQ_INJECT_ASM(5)
518 /* final store */
519 "str %[newv], %[v]\n\t"
520 "2:\n\t"
521 RSEQ_INJECT_ASM(6)
522 /* teardown */
523 "ldr %[len], %[rseq_scratch2]\n\t"
524 "ldr %[dst], %[rseq_scratch1]\n\t"
525 "ldr %[src], %[rseq_scratch0]\n\t"
526 "b 8f\n\t"
527 RSEQ_ASM_DEFINE_ABORT(3, 4,
528 /* teardown */
529 "ldr %[len], %[rseq_scratch2]\n\t"
530 "ldr %[dst], %[rseq_scratch1]\n\t"
531 "ldr %[src], %[rseq_scratch0]\n\t",
532 abort, 1b, 2b, 4f)
533 RSEQ_ASM_DEFINE_CMPFAIL(5,
534 /* teardown */
535 "ldr %[len], %[rseq_scratch2]\n\t"
536 "ldr %[dst], %[rseq_scratch1]\n\t"
537 "ldr %[src], %[rseq_scratch0]\n\t",
538 cmpfail)
539 #ifdef RSEQ_COMPARE_TWICE
540 RSEQ_ASM_DEFINE_CMPFAIL(6,
541 /* teardown */
542 "ldr %[len], %[rseq_scratch2]\n\t"
543 "ldr %[dst], %[rseq_scratch1]\n\t"
544 "ldr %[src], %[rseq_scratch0]\n\t",
545 error1)
546 RSEQ_ASM_DEFINE_CMPFAIL(7,
547 /* teardown */
548 "ldr %[len], %[rseq_scratch2]\n\t"
549 "ldr %[dst], %[rseq_scratch1]\n\t"
550 "ldr %[src], %[rseq_scratch0]\n\t",
551 error2)
552 #endif
553 "8:\n\t"
554 : /* gcc asm goto does not allow outputs */
555 : [cpu_id] "r" (cpu),
556 [current_cpu_id] "m" (__rseq_abi.cpu_id),
557 [rseq_cs] "m" (__rseq_abi.rseq_cs),
558 /* final store input */
559 [v] "m" (*v),
560 [expect] "r" (expect),
561 [newv] "r" (newv),
562 /* try memcpy input */
563 [dst] "r" (dst),
564 [src] "r" (src),
565 [len] "r" (len),
566 [rseq_scratch0] "m" (rseq_scratch[0]),
567 [rseq_scratch1] "m" (rseq_scratch[1]),
568 [rseq_scratch2] "m" (rseq_scratch[2])
569 RSEQ_INJECT_INPUT
570 : "r0", "memory", "cc"
571 RSEQ_INJECT_CLOBBER
572 : abort, cmpfail
573 #ifdef RSEQ_COMPARE_TWICE
574 , error1, error2
575 #endif
576 );
577 rseq_workaround_gcc_asm_size_guess();
578 return 0;
579 abort:
580 rseq_workaround_gcc_asm_size_guess();
581 RSEQ_INJECT_FAILED
582 return -1;
583 cmpfail:
584 rseq_workaround_gcc_asm_size_guess();
585 return 1;
586 #ifdef RSEQ_COMPARE_TWICE
587 error1:
588 rseq_workaround_gcc_asm_size_guess();
589 rseq_bug("cpu_id comparison failed");
590 error2:
591 rseq_workaround_gcc_asm_size_guess();
592 rseq_bug("expected value comparison failed");
593 #endif
594 }
595
596 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)597 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
598 void *dst, void *src, size_t len,
599 intptr_t newv, int cpu)
600 {
601 uint32_t rseq_scratch[3];
602
603 RSEQ_INJECT_C(9)
604
605 rseq_workaround_gcc_asm_size_guess();
606 __asm__ __volatile__ goto (
607 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
608 "str %[src], %[rseq_scratch0]\n\t"
609 "str %[dst], %[rseq_scratch1]\n\t"
610 "str %[len], %[rseq_scratch2]\n\t"
611 /* Start rseq by storing table entry pointer into rseq_cs. */
612 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
613 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
614 RSEQ_INJECT_ASM(3)
615 "ldr r0, %[v]\n\t"
616 "cmp %[expect], r0\n\t"
617 "bne 5f\n\t"
618 RSEQ_INJECT_ASM(4)
619 #ifdef RSEQ_COMPARE_TWICE
620 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
621 "ldr r0, %[v]\n\t"
622 "cmp %[expect], r0\n\t"
623 "bne 7f\n\t"
624 #endif
625 /* try memcpy */
626 "cmp %[len], #0\n\t" \
627 "beq 333f\n\t" \
628 "222:\n\t" \
629 "ldrb %%r0, [%[src]]\n\t" \
630 "strb %%r0, [%[dst]]\n\t" \
631 "adds %[src], #1\n\t" \
632 "adds %[dst], #1\n\t" \
633 "subs %[len], #1\n\t" \
634 "bne 222b\n\t" \
635 "333:\n\t" \
636 RSEQ_INJECT_ASM(5)
637 "dmb\n\t" /* full mb provides store-release */
638 /* final store */
639 "str %[newv], %[v]\n\t"
640 "2:\n\t"
641 RSEQ_INJECT_ASM(6)
642 /* teardown */
643 "ldr %[len], %[rseq_scratch2]\n\t"
644 "ldr %[dst], %[rseq_scratch1]\n\t"
645 "ldr %[src], %[rseq_scratch0]\n\t"
646 "b 8f\n\t"
647 RSEQ_ASM_DEFINE_ABORT(3, 4,
648 /* teardown */
649 "ldr %[len], %[rseq_scratch2]\n\t"
650 "ldr %[dst], %[rseq_scratch1]\n\t"
651 "ldr %[src], %[rseq_scratch0]\n\t",
652 abort, 1b, 2b, 4f)
653 RSEQ_ASM_DEFINE_CMPFAIL(5,
654 /* teardown */
655 "ldr %[len], %[rseq_scratch2]\n\t"
656 "ldr %[dst], %[rseq_scratch1]\n\t"
657 "ldr %[src], %[rseq_scratch0]\n\t",
658 cmpfail)
659 #ifdef RSEQ_COMPARE_TWICE
660 RSEQ_ASM_DEFINE_CMPFAIL(6,
661 /* teardown */
662 "ldr %[len], %[rseq_scratch2]\n\t"
663 "ldr %[dst], %[rseq_scratch1]\n\t"
664 "ldr %[src], %[rseq_scratch0]\n\t",
665 error1)
666 RSEQ_ASM_DEFINE_CMPFAIL(7,
667 /* teardown */
668 "ldr %[len], %[rseq_scratch2]\n\t"
669 "ldr %[dst], %[rseq_scratch1]\n\t"
670 "ldr %[src], %[rseq_scratch0]\n\t",
671 error2)
672 #endif
673 "8:\n\t"
674 : /* gcc asm goto does not allow outputs */
675 : [cpu_id] "r" (cpu),
676 [current_cpu_id] "m" (__rseq_abi.cpu_id),
677 [rseq_cs] "m" (__rseq_abi.rseq_cs),
678 /* final store input */
679 [v] "m" (*v),
680 [expect] "r" (expect),
681 [newv] "r" (newv),
682 /* try memcpy input */
683 [dst] "r" (dst),
684 [src] "r" (src),
685 [len] "r" (len),
686 [rseq_scratch0] "m" (rseq_scratch[0]),
687 [rseq_scratch1] "m" (rseq_scratch[1]),
688 [rseq_scratch2] "m" (rseq_scratch[2])
689 RSEQ_INJECT_INPUT
690 : "r0", "memory", "cc"
691 RSEQ_INJECT_CLOBBER
692 : abort, cmpfail
693 #ifdef RSEQ_COMPARE_TWICE
694 , error1, error2
695 #endif
696 );
697 rseq_workaround_gcc_asm_size_guess();
698 return 0;
699 abort:
700 rseq_workaround_gcc_asm_size_guess();
701 RSEQ_INJECT_FAILED
702 return -1;
703 cmpfail:
704 rseq_workaround_gcc_asm_size_guess();
705 return 1;
706 #ifdef RSEQ_COMPARE_TWICE
707 error1:
708 rseq_workaround_gcc_asm_size_guess();
709 rseq_bug("cpu_id comparison failed");
710 error2:
711 rseq_workaround_gcc_asm_size_guess();
712 rseq_bug("expected value comparison failed");
713 #endif
714 }
715
716 #endif /* !RSEQ_SKIP_FASTPATH */
717