1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * rseq-arm64.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
7 */
8
9 /*
10 * aarch64 -mbig-endian generates mixed endianness code vs data:
11 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
12 * matches code endianness.
13 */
14 #define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
15
16 #ifdef __AARCH64EB__
17 #define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
18 #else
19 #define RSEQ_SIG_DATA RSEQ_SIG_CODE
20 #endif
21
22 #define RSEQ_SIG RSEQ_SIG_DATA
23
24 #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
25 #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
26 #define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
27
28 #define rseq_smp_load_acquire(p) \
29 __extension__ ({ \
30 __typeof(*p) ____p1; \
31 switch (sizeof(*p)) { \
32 case 1: \
33 asm volatile ("ldarb %w0, %1" \
34 : "=r" (*(__u8 *)p) \
35 : "Q" (*p) : "memory"); \
36 break; \
37 case 2: \
38 asm volatile ("ldarh %w0, %1" \
39 : "=r" (*(__u16 *)p) \
40 : "Q" (*p) : "memory"); \
41 break; \
42 case 4: \
43 asm volatile ("ldar %w0, %1" \
44 : "=r" (*(__u32 *)p) \
45 : "Q" (*p) : "memory"); \
46 break; \
47 case 8: \
48 asm volatile ("ldar %0, %1" \
49 : "=r" (*(__u64 *)p) \
50 : "Q" (*p) : "memory"); \
51 break; \
52 } \
53 ____p1; \
54 })
55
56 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
57
58 #define rseq_smp_store_release(p, v) \
59 do { \
60 switch (sizeof(*p)) { \
61 case 1: \
62 asm volatile ("stlrb %w1, %0" \
63 : "=Q" (*p) \
64 : "r" ((__u8)v) \
65 : "memory"); \
66 break; \
67 case 2: \
68 asm volatile ("stlrh %w1, %0" \
69 : "=Q" (*p) \
70 : "r" ((__u16)v) \
71 : "memory"); \
72 break; \
73 case 4: \
74 asm volatile ("stlr %w1, %0" \
75 : "=Q" (*p) \
76 : "r" ((__u32)v) \
77 : "memory"); \
78 break; \
79 case 8: \
80 asm volatile ("stlr %1, %0" \
81 : "=Q" (*p) \
82 : "r" ((__u64)v) \
83 : "memory"); \
84 break; \
85 } \
86 } while (0)
87
88 #ifdef RSEQ_SKIP_FASTPATH
89 #include "rseq-skip.h"
90 #else /* !RSEQ_SKIP_FASTPATH */
91
92 #define RSEQ_ASM_TMP_REG32 "w15"
93 #define RSEQ_ASM_TMP_REG "x15"
94 #define RSEQ_ASM_TMP_REG_2 "x14"
95
96 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
97 post_commit_offset, abort_ip) \
98 " .pushsection __rseq_cs, \"aw\"\n" \
99 " .balign 32\n" \
100 __rseq_str(label) ":\n" \
101 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
102 " .quad " __rseq_str(start_ip) ", " \
103 __rseq_str(post_commit_offset) ", " \
104 __rseq_str(abort_ip) "\n" \
105 " .popsection\n\t" \
106 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
107 " .quad " __rseq_str(label) "b\n" \
108 " .popsection\n"
109
110 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
111 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
112 (post_commit_ip - start_ip), abort_ip)
113
114 /*
115 * Exit points of a rseq critical section consist of all instructions outside
116 * of the critical section where a critical section can either branch to or
117 * reach through the normal course of its execution. The abort IP and the
118 * post-commit IP are already part of the __rseq_cs section and should not be
119 * explicitly defined as additional exit points. Knowing all exit points is
120 * useful to assist debuggers stepping over the critical section.
121 */
122 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
123 " .pushsection __rseq_exit_point_array, \"aw\"\n" \
124 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
125 " .popsection\n"
126
127 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
128 RSEQ_INJECT_ASM(1) \
129 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
130 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
131 ", :lo12:" __rseq_str(cs_label) "\n" \
132 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
133 __rseq_str(label) ":\n"
134
135 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
136 " b 222f\n" \
137 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
138 __rseq_str(label) ":\n" \
139 " b %l[" __rseq_str(abort_label) "]\n" \
140 "222:\n"
141
142 #define RSEQ_ASM_OP_STORE(value, var) \
143 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
144
145 #define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
146 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
147
148 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
149 RSEQ_ASM_OP_STORE(value, var) \
150 __rseq_str(post_commit_label) ":\n"
151
152 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
153 RSEQ_ASM_OP_STORE_RELEASE(value, var) \
154 __rseq_str(post_commit_label) ":\n"
155
156 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
157 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
158 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
159 ", %[" __rseq_str(expect) "]\n" \
160 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
161
162 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
163 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
164 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
165 ", %w[" __rseq_str(expect) "]\n" \
166 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
167
168 #define RSEQ_ASM_OP_CMPNE(var, expect, label) \
169 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
170 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
171 ", %[" __rseq_str(expect) "]\n" \
172 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
173
174 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
175 RSEQ_INJECT_ASM(2) \
176 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
177
178 #define RSEQ_ASM_OP_R_LOAD(var) \
179 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
180
181 #define RSEQ_ASM_OP_R_STORE(var) \
182 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
183
184 #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
185 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
186 ", %[" __rseq_str(offset) "]]\n"
187
188 #define RSEQ_ASM_OP_R_ADD(count) \
189 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
190 ", %[" __rseq_str(count) "]\n"
191
192 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
193 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
194 __rseq_str(post_commit_label) ":\n"
195
196 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
197 " cbz %[" __rseq_str(len) "], 333f\n" \
198 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
199 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
200 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
201 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
202 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
203 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
204 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
205 "333:\n"
206
207 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)208 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
209 {
210 RSEQ_INJECT_C(9)
211
212 __asm__ __volatile__ goto (
213 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
214 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
215 #ifdef RSEQ_COMPARE_TWICE
216 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
217 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
218 #endif
219 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
220 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
221 RSEQ_INJECT_ASM(3)
222 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
223 RSEQ_INJECT_ASM(4)
224 #ifdef RSEQ_COMPARE_TWICE
225 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
226 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
227 #endif
228 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
229 RSEQ_INJECT_ASM(5)
230 RSEQ_ASM_DEFINE_ABORT(4, abort)
231 : /* gcc asm goto does not allow outputs */
232 : [cpu_id] "r" (cpu),
233 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
234 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
235 [v] "Qo" (*v),
236 [expect] "r" (expect),
237 [newv] "r" (newv)
238 RSEQ_INJECT_INPUT
239 : "memory", RSEQ_ASM_TMP_REG
240 : abort, cmpfail
241 #ifdef RSEQ_COMPARE_TWICE
242 , error1, error2
243 #endif
244 );
245 rseq_after_asm_goto();
246 return 0;
247 abort:
248 rseq_after_asm_goto();
249 RSEQ_INJECT_FAILED
250 return -1;
251 cmpfail:
252 rseq_after_asm_goto();
253 return 1;
254 #ifdef RSEQ_COMPARE_TWICE
255 error1:
256 rseq_after_asm_goto();
257 rseq_bug("cpu_id comparison failed");
258 error2:
259 rseq_after_asm_goto();
260 rseq_bug("expected value comparison failed");
261 #endif
262 }
263
264 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,long voffp,intptr_t * load,int cpu)265 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
266 long voffp, intptr_t *load, int cpu)
267 {
268 RSEQ_INJECT_C(9)
269
270 __asm__ __volatile__ goto (
271 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
272 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
273 #ifdef RSEQ_COMPARE_TWICE
274 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
275 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
276 #endif
277 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
278 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
279 RSEQ_INJECT_ASM(3)
280 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
281 RSEQ_INJECT_ASM(4)
282 #ifdef RSEQ_COMPARE_TWICE
283 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
284 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
285 #endif
286 RSEQ_ASM_OP_R_LOAD(v)
287 RSEQ_ASM_OP_R_STORE(load)
288 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
289 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
290 RSEQ_INJECT_ASM(5)
291 RSEQ_ASM_DEFINE_ABORT(4, abort)
292 : /* gcc asm goto does not allow outputs */
293 : [cpu_id] "r" (cpu),
294 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
295 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
296 [v] "Qo" (*v),
297 [expectnot] "r" (expectnot),
298 [load] "Qo" (*load),
299 [voffp] "r" (voffp)
300 RSEQ_INJECT_INPUT
301 : "memory", RSEQ_ASM_TMP_REG
302 : abort, cmpfail
303 #ifdef RSEQ_COMPARE_TWICE
304 , error1, error2
305 #endif
306 );
307 rseq_after_asm_goto();
308 return 0;
309 abort:
310 rseq_after_asm_goto();
311 RSEQ_INJECT_FAILED
312 return -1;
313 cmpfail:
314 rseq_after_asm_goto();
315 return 1;
316 #ifdef RSEQ_COMPARE_TWICE
317 error1:
318 rseq_after_asm_goto();
319 rseq_bug("cpu_id comparison failed");
320 error2:
321 rseq_after_asm_goto();
322 rseq_bug("expected value comparison failed");
323 #endif
324 }
325
326 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)327 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
328 {
329 RSEQ_INJECT_C(9)
330
331 __asm__ __volatile__ goto (
332 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
333 #ifdef RSEQ_COMPARE_TWICE
334 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
335 #endif
336 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
337 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
338 RSEQ_INJECT_ASM(3)
339 #ifdef RSEQ_COMPARE_TWICE
340 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
341 #endif
342 RSEQ_ASM_OP_R_LOAD(v)
343 RSEQ_ASM_OP_R_ADD(count)
344 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
345 RSEQ_INJECT_ASM(4)
346 RSEQ_ASM_DEFINE_ABORT(4, abort)
347 : /* gcc asm goto does not allow outputs */
348 : [cpu_id] "r" (cpu),
349 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
350 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
351 [v] "Qo" (*v),
352 [count] "r" (count)
353 RSEQ_INJECT_INPUT
354 : "memory", RSEQ_ASM_TMP_REG
355 : abort
356 #ifdef RSEQ_COMPARE_TWICE
357 , error1
358 #endif
359 );
360 rseq_after_asm_goto();
361 return 0;
362 abort:
363 rseq_after_asm_goto();
364 RSEQ_INJECT_FAILED
365 return -1;
366 #ifdef RSEQ_COMPARE_TWICE
367 error1:
368 rseq_after_asm_goto();
369 rseq_bug("cpu_id comparison failed");
370 #endif
371 }
372
373 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)374 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
375 intptr_t *v2, intptr_t newv2,
376 intptr_t newv, int cpu)
377 {
378 RSEQ_INJECT_C(9)
379
380 __asm__ __volatile__ goto (
381 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
382 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
383 #ifdef RSEQ_COMPARE_TWICE
384 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
385 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
386 #endif
387 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
388 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
389 RSEQ_INJECT_ASM(3)
390 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
391 RSEQ_INJECT_ASM(4)
392 #ifdef RSEQ_COMPARE_TWICE
393 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
394 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
395 #endif
396 RSEQ_ASM_OP_STORE(newv2, v2)
397 RSEQ_INJECT_ASM(5)
398 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
399 RSEQ_INJECT_ASM(6)
400 RSEQ_ASM_DEFINE_ABORT(4, abort)
401 : /* gcc asm goto does not allow outputs */
402 : [cpu_id] "r" (cpu),
403 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
404 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
405 [expect] "r" (expect),
406 [v] "Qo" (*v),
407 [newv] "r" (newv),
408 [v2] "Qo" (*v2),
409 [newv2] "r" (newv2)
410 RSEQ_INJECT_INPUT
411 : "memory", RSEQ_ASM_TMP_REG
412 : abort, cmpfail
413 #ifdef RSEQ_COMPARE_TWICE
414 , error1, error2
415 #endif
416 );
417 rseq_after_asm_goto();
418 return 0;
419 abort:
420 rseq_after_asm_goto();
421 RSEQ_INJECT_FAILED
422 return -1;
423 cmpfail:
424 rseq_after_asm_goto();
425 return 1;
426 #ifdef RSEQ_COMPARE_TWICE
427 error1:
428 rseq_after_asm_goto();
429 rseq_bug("cpu_id comparison failed");
430 error2:
431 rseq_after_asm_goto();
432 rseq_bug("expected value comparison failed");
433 #endif
434 }
435
436 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)437 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
438 intptr_t *v2, intptr_t newv2,
439 intptr_t newv, int cpu)
440 {
441 RSEQ_INJECT_C(9)
442
443 __asm__ __volatile__ goto (
444 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
445 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
446 #ifdef RSEQ_COMPARE_TWICE
447 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
448 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
449 #endif
450 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
451 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
452 RSEQ_INJECT_ASM(3)
453 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
454 RSEQ_INJECT_ASM(4)
455 #ifdef RSEQ_COMPARE_TWICE
456 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
457 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
458 #endif
459 RSEQ_ASM_OP_STORE(newv2, v2)
460 RSEQ_INJECT_ASM(5)
461 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
462 RSEQ_INJECT_ASM(6)
463 RSEQ_ASM_DEFINE_ABORT(4, abort)
464 : /* gcc asm goto does not allow outputs */
465 : [cpu_id] "r" (cpu),
466 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
467 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
468 [expect] "r" (expect),
469 [v] "Qo" (*v),
470 [newv] "r" (newv),
471 [v2] "Qo" (*v2),
472 [newv2] "r" (newv2)
473 RSEQ_INJECT_INPUT
474 : "memory", RSEQ_ASM_TMP_REG
475 : abort, cmpfail
476 #ifdef RSEQ_COMPARE_TWICE
477 , error1, error2
478 #endif
479 );
480 rseq_after_asm_goto();
481 return 0;
482 abort:
483 rseq_after_asm_goto();
484 RSEQ_INJECT_FAILED
485 return -1;
486 cmpfail:
487 rseq_after_asm_goto();
488 return 1;
489 #ifdef RSEQ_COMPARE_TWICE
490 error1:
491 rseq_after_asm_goto();
492 rseq_bug("cpu_id comparison failed");
493 error2:
494 rseq_after_asm_goto();
495 rseq_bug("expected value comparison failed");
496 #endif
497 }
498
499 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)500 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
501 intptr_t *v2, intptr_t expect2,
502 intptr_t newv, int cpu)
503 {
504 RSEQ_INJECT_C(9)
505
506 __asm__ __volatile__ goto (
507 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
508 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
509 #ifdef RSEQ_COMPARE_TWICE
510 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
511 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
512 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
513 #endif
514 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
515 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
516 RSEQ_INJECT_ASM(3)
517 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
518 RSEQ_INJECT_ASM(4)
519 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
520 RSEQ_INJECT_ASM(5)
521 #ifdef RSEQ_COMPARE_TWICE
522 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
523 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
524 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
525 #endif
526 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
527 RSEQ_INJECT_ASM(6)
528 RSEQ_ASM_DEFINE_ABORT(4, abort)
529 : /* gcc asm goto does not allow outputs */
530 : [cpu_id] "r" (cpu),
531 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
532 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
533 [v] "Qo" (*v),
534 [expect] "r" (expect),
535 [v2] "Qo" (*v2),
536 [expect2] "r" (expect2),
537 [newv] "r" (newv)
538 RSEQ_INJECT_INPUT
539 : "memory", RSEQ_ASM_TMP_REG
540 : abort, cmpfail
541 #ifdef RSEQ_COMPARE_TWICE
542 , error1, error2, error3
543 #endif
544 );
545 rseq_after_asm_goto();
546 return 0;
547 abort:
548 rseq_after_asm_goto();
549 RSEQ_INJECT_FAILED
550 return -1;
551 cmpfail:
552 rseq_after_asm_goto();
553 return 1;
554 #ifdef RSEQ_COMPARE_TWICE
555 error1:
556 rseq_after_asm_goto();
557 rseq_bug("cpu_id comparison failed");
558 error2:
559 rseq_after_asm_goto();
560 rseq_bug("expected value comparison failed");
561 error3:
562 rseq_after_asm_goto();
563 rseq_bug("2nd expected value comparison failed");
564 #endif
565 }
566
567 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)568 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
569 void *dst, void *src, size_t len,
570 intptr_t newv, int cpu)
571 {
572 RSEQ_INJECT_C(9)
573
574 __asm__ __volatile__ goto (
575 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
576 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
577 #ifdef RSEQ_COMPARE_TWICE
578 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
579 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
580 #endif
581 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
582 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
583 RSEQ_INJECT_ASM(3)
584 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
585 RSEQ_INJECT_ASM(4)
586 #ifdef RSEQ_COMPARE_TWICE
587 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
588 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
589 #endif
590 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
591 RSEQ_INJECT_ASM(5)
592 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
593 RSEQ_INJECT_ASM(6)
594 RSEQ_ASM_DEFINE_ABORT(4, abort)
595 : /* gcc asm goto does not allow outputs */
596 : [cpu_id] "r" (cpu),
597 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
598 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
599 [expect] "r" (expect),
600 [v] "Qo" (*v),
601 [newv] "r" (newv),
602 [dst] "r" (dst),
603 [src] "r" (src),
604 [len] "r" (len)
605 RSEQ_INJECT_INPUT
606 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
607 : abort, cmpfail
608 #ifdef RSEQ_COMPARE_TWICE
609 , error1, error2
610 #endif
611 );
612 rseq_after_asm_goto();
613 return 0;
614 abort:
615 rseq_after_asm_goto();
616 RSEQ_INJECT_FAILED
617 return -1;
618 cmpfail:
619 rseq_after_asm_goto();
620 return 1;
621 #ifdef RSEQ_COMPARE_TWICE
622 error1:
623 rseq_after_asm_goto();
624 rseq_bug("cpu_id comparison failed");
625 error2:
626 rseq_after_asm_goto();
627 rseq_bug("expected value comparison failed");
628 #endif
629 }
630
631 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)632 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
633 void *dst, void *src, size_t len,
634 intptr_t newv, int cpu)
635 {
636 RSEQ_INJECT_C(9)
637
638 __asm__ __volatile__ goto (
639 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
640 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
641 #ifdef RSEQ_COMPARE_TWICE
642 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
643 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
644 #endif
645 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
646 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
647 RSEQ_INJECT_ASM(3)
648 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
649 RSEQ_INJECT_ASM(4)
650 #ifdef RSEQ_COMPARE_TWICE
651 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
652 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
653 #endif
654 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
655 RSEQ_INJECT_ASM(5)
656 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
657 RSEQ_INJECT_ASM(6)
658 RSEQ_ASM_DEFINE_ABORT(4, abort)
659 : /* gcc asm goto does not allow outputs */
660 : [cpu_id] "r" (cpu),
661 [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
662 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
663 [expect] "r" (expect),
664 [v] "Qo" (*v),
665 [newv] "r" (newv),
666 [dst] "r" (dst),
667 [src] "r" (src),
668 [len] "r" (len)
669 RSEQ_INJECT_INPUT
670 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
671 : abort, cmpfail
672 #ifdef RSEQ_COMPARE_TWICE
673 , error1, error2
674 #endif
675 );
676 rseq_after_asm_goto();
677 return 0;
678 abort:
679 rseq_after_asm_goto();
680 RSEQ_INJECT_FAILED
681 return -1;
682 cmpfail:
683 rseq_after_asm_goto();
684 return 1;
685 #ifdef RSEQ_COMPARE_TWICE
686 error1:
687 rseq_after_asm_goto();
688 rseq_bug("cpu_id comparison failed");
689 error2:
690 rseq_after_asm_goto();
691 rseq_bug("expected value comparison failed");
692 #endif
693 }
694
695 #endif /* !RSEQ_SKIP_FASTPATH */
696