1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * rseq-arm64.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
7 */
8
9 #define RSEQ_SIG 0xd428bc00 /* BRK #0x45E0 */
10
11 #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
12 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
13 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
14
15 #define rseq_smp_load_acquire(p) \
16 __extension__ ({ \
17 __typeof(*p) ____p1; \
18 switch (sizeof(*p)) { \
19 case 1: \
20 asm volatile ("ldarb %w0, %1" \
21 : "=r" (*(__u8 *)p) \
22 : "Q" (*p) : "memory"); \
23 break; \
24 case 2: \
25 asm volatile ("ldarh %w0, %1" \
26 : "=r" (*(__u16 *)p) \
27 : "Q" (*p) : "memory"); \
28 break; \
29 case 4: \
30 asm volatile ("ldar %w0, %1" \
31 : "=r" (*(__u32 *)p) \
32 : "Q" (*p) : "memory"); \
33 break; \
34 case 8: \
35 asm volatile ("ldar %0, %1" \
36 : "=r" (*(__u64 *)p) \
37 : "Q" (*p) : "memory"); \
38 break; \
39 } \
40 ____p1; \
41 })
42
43 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
44
45 #define rseq_smp_store_release(p, v) \
46 do { \
47 switch (sizeof(*p)) { \
48 case 1: \
49 asm volatile ("stlrb %w1, %0" \
50 : "=Q" (*p) \
51 : "r" ((__u8)v) \
52 : "memory"); \
53 break; \
54 case 2: \
55 asm volatile ("stlrh %w1, %0" \
56 : "=Q" (*p) \
57 : "r" ((__u16)v) \
58 : "memory"); \
59 break; \
60 case 4: \
61 asm volatile ("stlr %w1, %0" \
62 : "=Q" (*p) \
63 : "r" ((__u32)v) \
64 : "memory"); \
65 break; \
66 case 8: \
67 asm volatile ("stlr %1, %0" \
68 : "=Q" (*p) \
69 : "r" ((__u64)v) \
70 : "memory"); \
71 break; \
72 } \
73 } while (0)
74
75 #ifdef RSEQ_SKIP_FASTPATH
76 #include "rseq-skip.h"
77 #else /* !RSEQ_SKIP_FASTPATH */
78
79 #define RSEQ_ASM_TMP_REG32 "w15"
80 #define RSEQ_ASM_TMP_REG "x15"
81 #define RSEQ_ASM_TMP_REG_2 "x14"
82
83 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
84 post_commit_offset, abort_ip) \
85 " .pushsection __rseq_table, \"aw\"\n" \
86 " .balign 32\n" \
87 __rseq_str(label) ":\n" \
88 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
89 " .quad " __rseq_str(start_ip) ", " \
90 __rseq_str(post_commit_offset) ", " \
91 __rseq_str(abort_ip) "\n" \
92 " .popsection\n"
93
94 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
95 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
96 (post_commit_ip - start_ip), abort_ip)
97
98 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
99 RSEQ_INJECT_ASM(1) \
100 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
101 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
102 ", :lo12:" __rseq_str(cs_label) "\n" \
103 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
104 __rseq_str(label) ":\n"
105
106 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
107 " b 222f\n" \
108 " .inst " __rseq_str(RSEQ_SIG) "\n" \
109 __rseq_str(label) ":\n" \
110 " b %l[" __rseq_str(abort_label) "]\n" \
111 "222:\n"
112
113 #define RSEQ_ASM_OP_STORE(value, var) \
114 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
115
116 #define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
117 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
118
119 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
120 RSEQ_ASM_OP_STORE(value, var) \
121 __rseq_str(post_commit_label) ":\n"
122
123 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
124 RSEQ_ASM_OP_STORE_RELEASE(value, var) \
125 __rseq_str(post_commit_label) ":\n"
126
127 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
128 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
129 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
130 ", %[" __rseq_str(expect) "]\n" \
131 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
132
133 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
134 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
135 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
136 ", %w[" __rseq_str(expect) "]\n" \
137 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
138
139 #define RSEQ_ASM_OP_CMPNE(var, expect, label) \
140 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
141 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
142 ", %[" __rseq_str(expect) "]\n" \
143 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
144
145 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
146 RSEQ_INJECT_ASM(2) \
147 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
148
149 #define RSEQ_ASM_OP_R_LOAD(var) \
150 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
151
152 #define RSEQ_ASM_OP_R_STORE(var) \
153 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
154
155 #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
156 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
157 ", %[" __rseq_str(offset) "]]\n"
158
159 #define RSEQ_ASM_OP_R_ADD(count) \
160 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
161 ", %[" __rseq_str(count) "]\n"
162
163 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
164 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
165 __rseq_str(post_commit_label) ":\n"
166
167 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
168 " cbz %[" __rseq_str(len) "], 333f\n" \
169 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
170 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
171 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
172 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
173 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
174 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
175 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
176 "333:\n"
177
178 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)179 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
180 {
181 RSEQ_INJECT_C(9)
182
183 __asm__ __volatile__ goto (
184 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
185 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
186 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
187 RSEQ_INJECT_ASM(3)
188 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
189 RSEQ_INJECT_ASM(4)
190 #ifdef RSEQ_COMPARE_TWICE
191 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
192 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
193 #endif
194 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
195 RSEQ_INJECT_ASM(5)
196 RSEQ_ASM_DEFINE_ABORT(4, abort)
197 : /* gcc asm goto does not allow outputs */
198 : [cpu_id] "r" (cpu),
199 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
200 [rseq_cs] "m" (__rseq_abi.rseq_cs),
201 [v] "Qo" (*v),
202 [expect] "r" (expect),
203 [newv] "r" (newv)
204 RSEQ_INJECT_INPUT
205 : "memory", RSEQ_ASM_TMP_REG
206 : abort, cmpfail
207 #ifdef RSEQ_COMPARE_TWICE
208 , error1, error2
209 #endif
210 );
211
212 return 0;
213 abort:
214 RSEQ_INJECT_FAILED
215 return -1;
216 cmpfail:
217 return 1;
218 #ifdef RSEQ_COMPARE_TWICE
219 error1:
220 rseq_bug("cpu_id comparison failed");
221 error2:
222 rseq_bug("expected value comparison failed");
223 #endif
224 }
225
226 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)227 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
228 off_t voffp, intptr_t *load, int cpu)
229 {
230 RSEQ_INJECT_C(9)
231
232 __asm__ __volatile__ goto (
233 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
234 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
235 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
236 RSEQ_INJECT_ASM(3)
237 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
238 RSEQ_INJECT_ASM(4)
239 #ifdef RSEQ_COMPARE_TWICE
240 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
241 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
242 #endif
243 RSEQ_ASM_OP_R_LOAD(v)
244 RSEQ_ASM_OP_R_STORE(load)
245 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
246 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
247 RSEQ_INJECT_ASM(5)
248 RSEQ_ASM_DEFINE_ABORT(4, abort)
249 : /* gcc asm goto does not allow outputs */
250 : [cpu_id] "r" (cpu),
251 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
252 [rseq_cs] "m" (__rseq_abi.rseq_cs),
253 [v] "Qo" (*v),
254 [expectnot] "r" (expectnot),
255 [load] "Qo" (*load),
256 [voffp] "r" (voffp)
257 RSEQ_INJECT_INPUT
258 : "memory", RSEQ_ASM_TMP_REG
259 : abort, cmpfail
260 #ifdef RSEQ_COMPARE_TWICE
261 , error1, error2
262 #endif
263 );
264 return 0;
265 abort:
266 RSEQ_INJECT_FAILED
267 return -1;
268 cmpfail:
269 return 1;
270 #ifdef RSEQ_COMPARE_TWICE
271 error1:
272 rseq_bug("cpu_id comparison failed");
273 error2:
274 rseq_bug("expected value comparison failed");
275 #endif
276 }
277
278 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)279 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
280 {
281 RSEQ_INJECT_C(9)
282
283 __asm__ __volatile__ goto (
284 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
285 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
286 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
287 RSEQ_INJECT_ASM(3)
288 #ifdef RSEQ_COMPARE_TWICE
289 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
290 #endif
291 RSEQ_ASM_OP_R_LOAD(v)
292 RSEQ_ASM_OP_R_ADD(count)
293 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
294 RSEQ_INJECT_ASM(4)
295 RSEQ_ASM_DEFINE_ABORT(4, abort)
296 : /* gcc asm goto does not allow outputs */
297 : [cpu_id] "r" (cpu),
298 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
299 [rseq_cs] "m" (__rseq_abi.rseq_cs),
300 [v] "Qo" (*v),
301 [count] "r" (count)
302 RSEQ_INJECT_INPUT
303 : "memory", RSEQ_ASM_TMP_REG
304 : abort
305 #ifdef RSEQ_COMPARE_TWICE
306 , error1
307 #endif
308 );
309 return 0;
310 abort:
311 RSEQ_INJECT_FAILED
312 return -1;
313 #ifdef RSEQ_COMPARE_TWICE
314 error1:
315 rseq_bug("cpu_id comparison failed");
316 #endif
317 }
318
319 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)320 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
321 intptr_t *v2, intptr_t newv2,
322 intptr_t newv, int cpu)
323 {
324 RSEQ_INJECT_C(9)
325
326 __asm__ __volatile__ goto (
327 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
328 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
329 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
330 RSEQ_INJECT_ASM(3)
331 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
332 RSEQ_INJECT_ASM(4)
333 #ifdef RSEQ_COMPARE_TWICE
334 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
335 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
336 #endif
337 RSEQ_ASM_OP_STORE(newv2, v2)
338 RSEQ_INJECT_ASM(5)
339 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
340 RSEQ_INJECT_ASM(6)
341 RSEQ_ASM_DEFINE_ABORT(4, abort)
342 : /* gcc asm goto does not allow outputs */
343 : [cpu_id] "r" (cpu),
344 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
345 [rseq_cs] "m" (__rseq_abi.rseq_cs),
346 [expect] "r" (expect),
347 [v] "Qo" (*v),
348 [newv] "r" (newv),
349 [v2] "Qo" (*v2),
350 [newv2] "r" (newv2)
351 RSEQ_INJECT_INPUT
352 : "memory", RSEQ_ASM_TMP_REG
353 : abort, cmpfail
354 #ifdef RSEQ_COMPARE_TWICE
355 , error1, error2
356 #endif
357 );
358
359 return 0;
360 abort:
361 RSEQ_INJECT_FAILED
362 return -1;
363 cmpfail:
364 return 1;
365 #ifdef RSEQ_COMPARE_TWICE
366 error1:
367 rseq_bug("cpu_id comparison failed");
368 error2:
369 rseq_bug("expected value comparison failed");
370 #endif
371 }
372
373 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)374 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
375 intptr_t *v2, intptr_t newv2,
376 intptr_t newv, int cpu)
377 {
378 RSEQ_INJECT_C(9)
379
380 __asm__ __volatile__ goto (
381 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
382 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
383 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
384 RSEQ_INJECT_ASM(3)
385 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
386 RSEQ_INJECT_ASM(4)
387 #ifdef RSEQ_COMPARE_TWICE
388 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
389 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
390 #endif
391 RSEQ_ASM_OP_STORE(newv2, v2)
392 RSEQ_INJECT_ASM(5)
393 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
394 RSEQ_INJECT_ASM(6)
395 RSEQ_ASM_DEFINE_ABORT(4, abort)
396 : /* gcc asm goto does not allow outputs */
397 : [cpu_id] "r" (cpu),
398 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
399 [rseq_cs] "m" (__rseq_abi.rseq_cs),
400 [expect] "r" (expect),
401 [v] "Qo" (*v),
402 [newv] "r" (newv),
403 [v2] "Qo" (*v2),
404 [newv2] "r" (newv2)
405 RSEQ_INJECT_INPUT
406 : "memory", RSEQ_ASM_TMP_REG
407 : abort, cmpfail
408 #ifdef RSEQ_COMPARE_TWICE
409 , error1, error2
410 #endif
411 );
412
413 return 0;
414 abort:
415 RSEQ_INJECT_FAILED
416 return -1;
417 cmpfail:
418 return 1;
419 #ifdef RSEQ_COMPARE_TWICE
420 error1:
421 rseq_bug("cpu_id comparison failed");
422 error2:
423 rseq_bug("expected value comparison failed");
424 #endif
425 }
426
427 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)428 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
429 intptr_t *v2, intptr_t expect2,
430 intptr_t newv, int cpu)
431 {
432 RSEQ_INJECT_C(9)
433
434 __asm__ __volatile__ goto (
435 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
436 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
437 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
438 RSEQ_INJECT_ASM(3)
439 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
440 RSEQ_INJECT_ASM(4)
441 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
442 RSEQ_INJECT_ASM(5)
443 #ifdef RSEQ_COMPARE_TWICE
444 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
445 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
446 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
447 #endif
448 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
449 RSEQ_INJECT_ASM(6)
450 RSEQ_ASM_DEFINE_ABORT(4, abort)
451 : /* gcc asm goto does not allow outputs */
452 : [cpu_id] "r" (cpu),
453 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
454 [rseq_cs] "m" (__rseq_abi.rseq_cs),
455 [v] "Qo" (*v),
456 [expect] "r" (expect),
457 [v2] "Qo" (*v2),
458 [expect2] "r" (expect2),
459 [newv] "r" (newv)
460 RSEQ_INJECT_INPUT
461 : "memory", RSEQ_ASM_TMP_REG
462 : abort, cmpfail
463 #ifdef RSEQ_COMPARE_TWICE
464 , error1, error2, error3
465 #endif
466 );
467
468 return 0;
469 abort:
470 RSEQ_INJECT_FAILED
471 return -1;
472 cmpfail:
473 return 1;
474 #ifdef RSEQ_COMPARE_TWICE
475 error1:
476 rseq_bug("cpu_id comparison failed");
477 error2:
478 rseq_bug("expected value comparison failed");
479 error3:
480 rseq_bug("2nd expected value comparison failed");
481 #endif
482 }
483
484 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)485 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
486 void *dst, void *src, size_t len,
487 intptr_t newv, int cpu)
488 {
489 RSEQ_INJECT_C(9)
490
491 __asm__ __volatile__ goto (
492 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
493 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
494 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
495 RSEQ_INJECT_ASM(3)
496 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
497 RSEQ_INJECT_ASM(4)
498 #ifdef RSEQ_COMPARE_TWICE
499 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
500 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
501 #endif
502 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
503 RSEQ_INJECT_ASM(5)
504 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
505 RSEQ_INJECT_ASM(6)
506 RSEQ_ASM_DEFINE_ABORT(4, abort)
507 : /* gcc asm goto does not allow outputs */
508 : [cpu_id] "r" (cpu),
509 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
510 [rseq_cs] "m" (__rseq_abi.rseq_cs),
511 [expect] "r" (expect),
512 [v] "Qo" (*v),
513 [newv] "r" (newv),
514 [dst] "r" (dst),
515 [src] "r" (src),
516 [len] "r" (len)
517 RSEQ_INJECT_INPUT
518 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
519 : abort, cmpfail
520 #ifdef RSEQ_COMPARE_TWICE
521 , error1, error2
522 #endif
523 );
524
525 return 0;
526 abort:
527 RSEQ_INJECT_FAILED
528 return -1;
529 cmpfail:
530 return 1;
531 #ifdef RSEQ_COMPARE_TWICE
532 error1:
533 rseq_bug("cpu_id comparison failed");
534 error2:
535 rseq_bug("expected value comparison failed");
536 #endif
537 }
538
539 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)540 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
541 void *dst, void *src, size_t len,
542 intptr_t newv, int cpu)
543 {
544 RSEQ_INJECT_C(9)
545
546 __asm__ __volatile__ goto (
547 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
548 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
549 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
550 RSEQ_INJECT_ASM(3)
551 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
552 RSEQ_INJECT_ASM(4)
553 #ifdef RSEQ_COMPARE_TWICE
554 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
555 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
556 #endif
557 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
558 RSEQ_INJECT_ASM(5)
559 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
560 RSEQ_INJECT_ASM(6)
561 RSEQ_ASM_DEFINE_ABORT(4, abort)
562 : /* gcc asm goto does not allow outputs */
563 : [cpu_id] "r" (cpu),
564 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
565 [rseq_cs] "m" (__rseq_abi.rseq_cs),
566 [expect] "r" (expect),
567 [v] "Qo" (*v),
568 [newv] "r" (newv),
569 [dst] "r" (dst),
570 [src] "r" (src),
571 [len] "r" (len)
572 RSEQ_INJECT_INPUT
573 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
574 : abort, cmpfail
575 #ifdef RSEQ_COMPARE_TWICE
576 , error1, error2
577 #endif
578 );
579
580 return 0;
581 abort:
582 RSEQ_INJECT_FAILED
583 return -1;
584 cmpfail:
585 return 1;
586 #ifdef RSEQ_COMPARE_TWICE
587 error1:
588 rseq_bug("cpu_id comparison failed");
589 error2:
590 rseq_bug("expected value comparison failed");
591 #endif
592 }
593
594 #endif /* !RSEQ_SKIP_FASTPATH */
595