1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3 * Author: Paul Burton <paul.burton@mips.com>
4 * (C) Copyright 2018 MIPS Tech LLC
5 *
6 * Based on rseq-arm.h:
7 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
8 */
9
10 #define RSEQ_SIG 0x53053053
11
12 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory")
13 #define rseq_smp_rmb() rseq_smp_mb()
14 #define rseq_smp_wmb() rseq_smp_mb()
15
16 #define rseq_smp_load_acquire(p) \
17 __extension__ ({ \
18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
19 rseq_smp_mb(); \
20 ____p1; \
21 })
22
23 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
24
25 #define rseq_smp_store_release(p, v) \
26 do { \
27 rseq_smp_mb(); \
28 RSEQ_WRITE_ONCE(*p, v); \
29 } while (0)
30
31 #ifdef RSEQ_SKIP_FASTPATH
32 #include "rseq-skip.h"
33 #else /* !RSEQ_SKIP_FASTPATH */
34
35 #if _MIPS_SZLONG == 64
36 # define LONG ".dword"
37 # define LONG_LA "dla"
38 # define LONG_L "ld"
39 # define LONG_S "sd"
40 # define LONG_ADDI "daddiu"
41 # define U32_U64_PAD(x) x
42 #elif _MIPS_SZLONG == 32
43 # define LONG ".word"
44 # define LONG_LA "la"
45 # define LONG_L "lw"
46 # define LONG_S "sw"
47 # define LONG_ADDI "addiu"
48 # ifdef __BIG_ENDIAN
49 # define U32_U64_PAD(x) "0x0, " x
50 # else
51 # define U32_U64_PAD(x) x ", 0x0"
52 # endif
53 #else
54 # error unsupported _MIPS_SZLONG
55 #endif
56
57 #define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \
58 post_commit_offset, abort_ip) \
59 ".pushsection __rseq_table, \"aw\"\n\t" \
60 ".balign 32\n\t" \
61 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
62 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
63 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
64 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
65 ".popsection\n\t"
66
67 #define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \
68 __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \
69 (post_commit_ip - start_ip), abort_ip)
70
71 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
72 RSEQ_INJECT_ASM(1) \
73 LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \
74 LONG_S " $4, %[" __rseq_str(rseq_cs) "]\n\t" \
75 __rseq_str(label) ":\n\t"
76
77 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
78 RSEQ_INJECT_ASM(2) \
79 "lw $4, %[" __rseq_str(current_cpu_id) "]\n\t" \
80 "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t"
81
82 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
83 abort_label, version, flags, \
84 start_ip, post_commit_offset, abort_ip) \
85 ".balign 32\n\t" \
86 __rseq_str(table_label) ":\n\t" \
87 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
88 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
89 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
90 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
91 ".word " __rseq_str(RSEQ_SIG) "\n\t" \
92 __rseq_str(label) ":\n\t" \
93 teardown \
94 "b %l[" __rseq_str(abort_label) "]\n\t"
95
96 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
97 start_ip, post_commit_ip, abort_ip) \
98 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
99 abort_label, 0x0, 0x0, start_ip, \
100 (post_commit_ip - start_ip), abort_ip)
101
102 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
103 __rseq_str(label) ":\n\t" \
104 teardown \
105 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
106
107 #define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("")
108
109 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)110 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
111 {
112 RSEQ_INJECT_C(9)
113
114 rseq_workaround_gcc_asm_size_guess();
115 __asm__ __volatile__ goto (
116 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
117 /* Start rseq by storing table entry pointer into rseq_cs. */
118 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
119 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
120 RSEQ_INJECT_ASM(3)
121 LONG_L " $4, %[v]\n\t"
122 "bne $4, %[expect], %l[cmpfail]\n\t"
123 RSEQ_INJECT_ASM(4)
124 #ifdef RSEQ_COMPARE_TWICE
125 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
126 LONG_L " $4, %[v]\n\t"
127 "bne $4, %[expect], %l[error2]\n\t"
128 #endif
129 /* final store */
130 LONG_S " %[newv], %[v]\n\t"
131 "2:\n\t"
132 RSEQ_INJECT_ASM(5)
133 "b 5f\n\t"
134 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
135 "5:\n\t"
136 : /* gcc asm goto does not allow outputs */
137 : [cpu_id] "r" (cpu),
138 [current_cpu_id] "m" (__rseq_abi.cpu_id),
139 [rseq_cs] "m" (__rseq_abi.rseq_cs),
140 [v] "m" (*v),
141 [expect] "r" (expect),
142 [newv] "r" (newv)
143 RSEQ_INJECT_INPUT
144 : "$4", "memory"
145 RSEQ_INJECT_CLOBBER
146 : abort, cmpfail
147 #ifdef RSEQ_COMPARE_TWICE
148 , error1, error2
149 #endif
150 );
151 rseq_workaround_gcc_asm_size_guess();
152 return 0;
153 abort:
154 rseq_workaround_gcc_asm_size_guess();
155 RSEQ_INJECT_FAILED
156 return -1;
157 cmpfail:
158 rseq_workaround_gcc_asm_size_guess();
159 return 1;
160 #ifdef RSEQ_COMPARE_TWICE
161 error1:
162 rseq_bug("cpu_id comparison failed");
163 error2:
164 rseq_bug("expected value comparison failed");
165 #endif
166 }
167
168 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)169 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
170 off_t voffp, intptr_t *load, int cpu)
171 {
172 RSEQ_INJECT_C(9)
173
174 rseq_workaround_gcc_asm_size_guess();
175 __asm__ __volatile__ goto (
176 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
177 /* Start rseq by storing table entry pointer into rseq_cs. */
178 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
179 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
180 RSEQ_INJECT_ASM(3)
181 LONG_L " $4, %[v]\n\t"
182 "beq $4, %[expectnot], %l[cmpfail]\n\t"
183 RSEQ_INJECT_ASM(4)
184 #ifdef RSEQ_COMPARE_TWICE
185 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
186 LONG_L " $4, %[v]\n\t"
187 "beq $4, %[expectnot], %l[error2]\n\t"
188 #endif
189 LONG_S " $4, %[load]\n\t"
190 LONG_ADDI " $4, %[voffp]\n\t"
191 LONG_L " $4, 0($4)\n\t"
192 /* final store */
193 LONG_S " $4, %[v]\n\t"
194 "2:\n\t"
195 RSEQ_INJECT_ASM(5)
196 "b 5f\n\t"
197 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
198 "5:\n\t"
199 : /* gcc asm goto does not allow outputs */
200 : [cpu_id] "r" (cpu),
201 [current_cpu_id] "m" (__rseq_abi.cpu_id),
202 [rseq_cs] "m" (__rseq_abi.rseq_cs),
203 /* final store input */
204 [v] "m" (*v),
205 [expectnot] "r" (expectnot),
206 [voffp] "Ir" (voffp),
207 [load] "m" (*load)
208 RSEQ_INJECT_INPUT
209 : "$4", "memory"
210 RSEQ_INJECT_CLOBBER
211 : abort, cmpfail
212 #ifdef RSEQ_COMPARE_TWICE
213 , error1, error2
214 #endif
215 );
216 rseq_workaround_gcc_asm_size_guess();
217 return 0;
218 abort:
219 rseq_workaround_gcc_asm_size_guess();
220 RSEQ_INJECT_FAILED
221 return -1;
222 cmpfail:
223 rseq_workaround_gcc_asm_size_guess();
224 return 1;
225 #ifdef RSEQ_COMPARE_TWICE
226 error1:
227 rseq_bug("cpu_id comparison failed");
228 error2:
229 rseq_bug("expected value comparison failed");
230 #endif
231 }
232
233 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)234 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
235 {
236 RSEQ_INJECT_C(9)
237
238 rseq_workaround_gcc_asm_size_guess();
239 __asm__ __volatile__ goto (
240 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
241 /* Start rseq by storing table entry pointer into rseq_cs. */
242 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
243 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
244 RSEQ_INJECT_ASM(3)
245 #ifdef RSEQ_COMPARE_TWICE
246 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
247 #endif
248 LONG_L " $4, %[v]\n\t"
249 LONG_ADDI " $4, %[count]\n\t"
250 /* final store */
251 LONG_S " $4, %[v]\n\t"
252 "2:\n\t"
253 RSEQ_INJECT_ASM(4)
254 "b 5f\n\t"
255 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
256 "5:\n\t"
257 : /* gcc asm goto does not allow outputs */
258 : [cpu_id] "r" (cpu),
259 [current_cpu_id] "m" (__rseq_abi.cpu_id),
260 [rseq_cs] "m" (__rseq_abi.rseq_cs),
261 [v] "m" (*v),
262 [count] "Ir" (count)
263 RSEQ_INJECT_INPUT
264 : "$4", "memory"
265 RSEQ_INJECT_CLOBBER
266 : abort
267 #ifdef RSEQ_COMPARE_TWICE
268 , error1
269 #endif
270 );
271 rseq_workaround_gcc_asm_size_guess();
272 return 0;
273 abort:
274 rseq_workaround_gcc_asm_size_guess();
275 RSEQ_INJECT_FAILED
276 return -1;
277 #ifdef RSEQ_COMPARE_TWICE
278 error1:
279 rseq_bug("cpu_id comparison failed");
280 #endif
281 }
282
283 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)284 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
285 intptr_t *v2, intptr_t newv2,
286 intptr_t newv, int cpu)
287 {
288 RSEQ_INJECT_C(9)
289
290 rseq_workaround_gcc_asm_size_guess();
291 __asm__ __volatile__ goto (
292 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
293 /* Start rseq by storing table entry pointer into rseq_cs. */
294 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
295 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
296 RSEQ_INJECT_ASM(3)
297 LONG_L " $4, %[v]\n\t"
298 "bne $4, %[expect], %l[cmpfail]\n\t"
299 RSEQ_INJECT_ASM(4)
300 #ifdef RSEQ_COMPARE_TWICE
301 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
302 LONG_L " $4, %[v]\n\t"
303 "bne $4, %[expect], %l[error2]\n\t"
304 #endif
305 /* try store */
306 LONG_S " %[newv2], %[v2]\n\t"
307 RSEQ_INJECT_ASM(5)
308 /* final store */
309 LONG_S " %[newv], %[v]\n\t"
310 "2:\n\t"
311 RSEQ_INJECT_ASM(6)
312 "b 5f\n\t"
313 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
314 "5:\n\t"
315 : /* gcc asm goto does not allow outputs */
316 : [cpu_id] "r" (cpu),
317 [current_cpu_id] "m" (__rseq_abi.cpu_id),
318 [rseq_cs] "m" (__rseq_abi.rseq_cs),
319 /* try store input */
320 [v2] "m" (*v2),
321 [newv2] "r" (newv2),
322 /* final store input */
323 [v] "m" (*v),
324 [expect] "r" (expect),
325 [newv] "r" (newv)
326 RSEQ_INJECT_INPUT
327 : "$4", "memory"
328 RSEQ_INJECT_CLOBBER
329 : abort, cmpfail
330 #ifdef RSEQ_COMPARE_TWICE
331 , error1, error2
332 #endif
333 );
334 rseq_workaround_gcc_asm_size_guess();
335 return 0;
336 abort:
337 rseq_workaround_gcc_asm_size_guess();
338 RSEQ_INJECT_FAILED
339 return -1;
340 cmpfail:
341 rseq_workaround_gcc_asm_size_guess();
342 return 1;
343 #ifdef RSEQ_COMPARE_TWICE
344 error1:
345 rseq_bug("cpu_id comparison failed");
346 error2:
347 rseq_bug("expected value comparison failed");
348 #endif
349 }
350
351 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)352 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
353 intptr_t *v2, intptr_t newv2,
354 intptr_t newv, int cpu)
355 {
356 RSEQ_INJECT_C(9)
357
358 rseq_workaround_gcc_asm_size_guess();
359 __asm__ __volatile__ goto (
360 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
361 /* Start rseq by storing table entry pointer into rseq_cs. */
362 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
363 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
364 RSEQ_INJECT_ASM(3)
365 LONG_L " $4, %[v]\n\t"
366 "bne $4, %[expect], %l[cmpfail]\n\t"
367 RSEQ_INJECT_ASM(4)
368 #ifdef RSEQ_COMPARE_TWICE
369 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
370 LONG_L " $4, %[v]\n\t"
371 "bne $4, %[expect], %l[error2]\n\t"
372 #endif
373 /* try store */
374 LONG_S " %[newv2], %[v2]\n\t"
375 RSEQ_INJECT_ASM(5)
376 "sync\n\t" /* full sync provides store-release */
377 /* final store */
378 LONG_S " %[newv], %[v]\n\t"
379 "2:\n\t"
380 RSEQ_INJECT_ASM(6)
381 "b 5f\n\t"
382 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
383 "5:\n\t"
384 : /* gcc asm goto does not allow outputs */
385 : [cpu_id] "r" (cpu),
386 [current_cpu_id] "m" (__rseq_abi.cpu_id),
387 [rseq_cs] "m" (__rseq_abi.rseq_cs),
388 /* try store input */
389 [v2] "m" (*v2),
390 [newv2] "r" (newv2),
391 /* final store input */
392 [v] "m" (*v),
393 [expect] "r" (expect),
394 [newv] "r" (newv)
395 RSEQ_INJECT_INPUT
396 : "$4", "memory"
397 RSEQ_INJECT_CLOBBER
398 : abort, cmpfail
399 #ifdef RSEQ_COMPARE_TWICE
400 , error1, error2
401 #endif
402 );
403 rseq_workaround_gcc_asm_size_guess();
404 return 0;
405 abort:
406 rseq_workaround_gcc_asm_size_guess();
407 RSEQ_INJECT_FAILED
408 return -1;
409 cmpfail:
410 rseq_workaround_gcc_asm_size_guess();
411 return 1;
412 #ifdef RSEQ_COMPARE_TWICE
413 error1:
414 rseq_bug("cpu_id comparison failed");
415 error2:
416 rseq_bug("expected value comparison failed");
417 #endif
418 }
419
420 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)421 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
422 intptr_t *v2, intptr_t expect2,
423 intptr_t newv, int cpu)
424 {
425 RSEQ_INJECT_C(9)
426
427 rseq_workaround_gcc_asm_size_guess();
428 __asm__ __volatile__ goto (
429 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
430 /* Start rseq by storing table entry pointer into rseq_cs. */
431 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
432 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
433 RSEQ_INJECT_ASM(3)
434 LONG_L " $4, %[v]\n\t"
435 "bne $4, %[expect], %l[cmpfail]\n\t"
436 RSEQ_INJECT_ASM(4)
437 LONG_L " $4, %[v2]\n\t"
438 "bne $4, %[expect2], %l[cmpfail]\n\t"
439 RSEQ_INJECT_ASM(5)
440 #ifdef RSEQ_COMPARE_TWICE
441 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
442 LONG_L " $4, %[v]\n\t"
443 "bne $4, %[expect], %l[error2]\n\t"
444 LONG_L " $4, %[v2]\n\t"
445 "bne $4, %[expect2], %l[error3]\n\t"
446 #endif
447 /* final store */
448 LONG_S " %[newv], %[v]\n\t"
449 "2:\n\t"
450 RSEQ_INJECT_ASM(6)
451 "b 5f\n\t"
452 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
453 "5:\n\t"
454 : /* gcc asm goto does not allow outputs */
455 : [cpu_id] "r" (cpu),
456 [current_cpu_id] "m" (__rseq_abi.cpu_id),
457 [rseq_cs] "m" (__rseq_abi.rseq_cs),
458 /* cmp2 input */
459 [v2] "m" (*v2),
460 [expect2] "r" (expect2),
461 /* final store input */
462 [v] "m" (*v),
463 [expect] "r" (expect),
464 [newv] "r" (newv)
465 RSEQ_INJECT_INPUT
466 : "$4", "memory"
467 RSEQ_INJECT_CLOBBER
468 : abort, cmpfail
469 #ifdef RSEQ_COMPARE_TWICE
470 , error1, error2, error3
471 #endif
472 );
473 rseq_workaround_gcc_asm_size_guess();
474 return 0;
475 abort:
476 rseq_workaround_gcc_asm_size_guess();
477 RSEQ_INJECT_FAILED
478 return -1;
479 cmpfail:
480 rseq_workaround_gcc_asm_size_guess();
481 return 1;
482 #ifdef RSEQ_COMPARE_TWICE
483 error1:
484 rseq_bug("cpu_id comparison failed");
485 error2:
486 rseq_bug("1st expected value comparison failed");
487 error3:
488 rseq_bug("2nd expected value comparison failed");
489 #endif
490 }
491
492 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)493 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
494 void *dst, void *src, size_t len,
495 intptr_t newv, int cpu)
496 {
497 uintptr_t rseq_scratch[3];
498
499 RSEQ_INJECT_C(9)
500
501 rseq_workaround_gcc_asm_size_guess();
502 __asm__ __volatile__ goto (
503 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
504 LONG_S " %[src], %[rseq_scratch0]\n\t"
505 LONG_S " %[dst], %[rseq_scratch1]\n\t"
506 LONG_S " %[len], %[rseq_scratch2]\n\t"
507 /* Start rseq by storing table entry pointer into rseq_cs. */
508 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
509 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
510 RSEQ_INJECT_ASM(3)
511 LONG_L " $4, %[v]\n\t"
512 "bne $4, %[expect], 5f\n\t"
513 RSEQ_INJECT_ASM(4)
514 #ifdef RSEQ_COMPARE_TWICE
515 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
516 LONG_L " $4, %[v]\n\t"
517 "bne $4, %[expect], 7f\n\t"
518 #endif
519 /* try memcpy */
520 "beqz %[len], 333f\n\t" \
521 "222:\n\t" \
522 "lb $4, 0(%[src])\n\t" \
523 "sb $4, 0(%[dst])\n\t" \
524 LONG_ADDI " %[src], 1\n\t" \
525 LONG_ADDI " %[dst], 1\n\t" \
526 LONG_ADDI " %[len], -1\n\t" \
527 "bnez %[len], 222b\n\t" \
528 "333:\n\t" \
529 RSEQ_INJECT_ASM(5)
530 /* final store */
531 LONG_S " %[newv], %[v]\n\t"
532 "2:\n\t"
533 RSEQ_INJECT_ASM(6)
534 /* teardown */
535 LONG_L " %[len], %[rseq_scratch2]\n\t"
536 LONG_L " %[dst], %[rseq_scratch1]\n\t"
537 LONG_L " %[src], %[rseq_scratch0]\n\t"
538 "b 8f\n\t"
539 RSEQ_ASM_DEFINE_ABORT(3, 4,
540 /* teardown */
541 LONG_L " %[len], %[rseq_scratch2]\n\t"
542 LONG_L " %[dst], %[rseq_scratch1]\n\t"
543 LONG_L " %[src], %[rseq_scratch0]\n\t",
544 abort, 1b, 2b, 4f)
545 RSEQ_ASM_DEFINE_CMPFAIL(5,
546 /* teardown */
547 LONG_L " %[len], %[rseq_scratch2]\n\t"
548 LONG_L " %[dst], %[rseq_scratch1]\n\t"
549 LONG_L " %[src], %[rseq_scratch0]\n\t",
550 cmpfail)
551 #ifdef RSEQ_COMPARE_TWICE
552 RSEQ_ASM_DEFINE_CMPFAIL(6,
553 /* teardown */
554 LONG_L " %[len], %[rseq_scratch2]\n\t"
555 LONG_L " %[dst], %[rseq_scratch1]\n\t"
556 LONG_L " %[src], %[rseq_scratch0]\n\t",
557 error1)
558 RSEQ_ASM_DEFINE_CMPFAIL(7,
559 /* teardown */
560 LONG_L " %[len], %[rseq_scratch2]\n\t"
561 LONG_L " %[dst], %[rseq_scratch1]\n\t"
562 LONG_L " %[src], %[rseq_scratch0]\n\t",
563 error2)
564 #endif
565 "8:\n\t"
566 : /* gcc asm goto does not allow outputs */
567 : [cpu_id] "r" (cpu),
568 [current_cpu_id] "m" (__rseq_abi.cpu_id),
569 [rseq_cs] "m" (__rseq_abi.rseq_cs),
570 /* final store input */
571 [v] "m" (*v),
572 [expect] "r" (expect),
573 [newv] "r" (newv),
574 /* try memcpy input */
575 [dst] "r" (dst),
576 [src] "r" (src),
577 [len] "r" (len),
578 [rseq_scratch0] "m" (rseq_scratch[0]),
579 [rseq_scratch1] "m" (rseq_scratch[1]),
580 [rseq_scratch2] "m" (rseq_scratch[2])
581 RSEQ_INJECT_INPUT
582 : "$4", "memory"
583 RSEQ_INJECT_CLOBBER
584 : abort, cmpfail
585 #ifdef RSEQ_COMPARE_TWICE
586 , error1, error2
587 #endif
588 );
589 rseq_workaround_gcc_asm_size_guess();
590 return 0;
591 abort:
592 rseq_workaround_gcc_asm_size_guess();
593 RSEQ_INJECT_FAILED
594 return -1;
595 cmpfail:
596 rseq_workaround_gcc_asm_size_guess();
597 return 1;
598 #ifdef RSEQ_COMPARE_TWICE
599 error1:
600 rseq_workaround_gcc_asm_size_guess();
601 rseq_bug("cpu_id comparison failed");
602 error2:
603 rseq_workaround_gcc_asm_size_guess();
604 rseq_bug("expected value comparison failed");
605 #endif
606 }
607
608 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)609 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
610 void *dst, void *src, size_t len,
611 intptr_t newv, int cpu)
612 {
613 uintptr_t rseq_scratch[3];
614
615 RSEQ_INJECT_C(9)
616
617 rseq_workaround_gcc_asm_size_guess();
618 __asm__ __volatile__ goto (
619 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
620 LONG_S " %[src], %[rseq_scratch0]\n\t"
621 LONG_S " %[dst], %[rseq_scratch1]\n\t"
622 LONG_S " %[len], %[rseq_scratch2]\n\t"
623 /* Start rseq by storing table entry pointer into rseq_cs. */
624 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
625 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
626 RSEQ_INJECT_ASM(3)
627 LONG_L " $4, %[v]\n\t"
628 "bne $4, %[expect], 5f\n\t"
629 RSEQ_INJECT_ASM(4)
630 #ifdef RSEQ_COMPARE_TWICE
631 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
632 LONG_L " $4, %[v]\n\t"
633 "bne $4, %[expect], 7f\n\t"
634 #endif
635 /* try memcpy */
636 "beqz %[len], 333f\n\t" \
637 "222:\n\t" \
638 "lb $4, 0(%[src])\n\t" \
639 "sb $4, 0(%[dst])\n\t" \
640 LONG_ADDI " %[src], 1\n\t" \
641 LONG_ADDI " %[dst], 1\n\t" \
642 LONG_ADDI " %[len], -1\n\t" \
643 "bnez %[len], 222b\n\t" \
644 "333:\n\t" \
645 RSEQ_INJECT_ASM(5)
646 "sync\n\t" /* full sync provides store-release */
647 /* final store */
648 LONG_S " %[newv], %[v]\n\t"
649 "2:\n\t"
650 RSEQ_INJECT_ASM(6)
651 /* teardown */
652 LONG_L " %[len], %[rseq_scratch2]\n\t"
653 LONG_L " %[dst], %[rseq_scratch1]\n\t"
654 LONG_L " %[src], %[rseq_scratch0]\n\t"
655 "b 8f\n\t"
656 RSEQ_ASM_DEFINE_ABORT(3, 4,
657 /* teardown */
658 LONG_L " %[len], %[rseq_scratch2]\n\t"
659 LONG_L " %[dst], %[rseq_scratch1]\n\t"
660 LONG_L " %[src], %[rseq_scratch0]\n\t",
661 abort, 1b, 2b, 4f)
662 RSEQ_ASM_DEFINE_CMPFAIL(5,
663 /* teardown */
664 LONG_L " %[len], %[rseq_scratch2]\n\t"
665 LONG_L " %[dst], %[rseq_scratch1]\n\t"
666 LONG_L " %[src], %[rseq_scratch0]\n\t",
667 cmpfail)
668 #ifdef RSEQ_COMPARE_TWICE
669 RSEQ_ASM_DEFINE_CMPFAIL(6,
670 /* teardown */
671 LONG_L " %[len], %[rseq_scratch2]\n\t"
672 LONG_L " %[dst], %[rseq_scratch1]\n\t"
673 LONG_L " %[src], %[rseq_scratch0]\n\t",
674 error1)
675 RSEQ_ASM_DEFINE_CMPFAIL(7,
676 /* teardown */
677 LONG_L " %[len], %[rseq_scratch2]\n\t"
678 LONG_L " %[dst], %[rseq_scratch1]\n\t"
679 LONG_L " %[src], %[rseq_scratch0]\n\t",
680 error2)
681 #endif
682 "8:\n\t"
683 : /* gcc asm goto does not allow outputs */
684 : [cpu_id] "r" (cpu),
685 [current_cpu_id] "m" (__rseq_abi.cpu_id),
686 [rseq_cs] "m" (__rseq_abi.rseq_cs),
687 /* final store input */
688 [v] "m" (*v),
689 [expect] "r" (expect),
690 [newv] "r" (newv),
691 /* try memcpy input */
692 [dst] "r" (dst),
693 [src] "r" (src),
694 [len] "r" (len),
695 [rseq_scratch0] "m" (rseq_scratch[0]),
696 [rseq_scratch1] "m" (rseq_scratch[1]),
697 [rseq_scratch2] "m" (rseq_scratch[2])
698 RSEQ_INJECT_INPUT
699 : "$4", "memory"
700 RSEQ_INJECT_CLOBBER
701 : abort, cmpfail
702 #ifdef RSEQ_COMPARE_TWICE
703 , error1, error2
704 #endif
705 );
706 rseq_workaround_gcc_asm_size_guess();
707 return 0;
708 abort:
709 rseq_workaround_gcc_asm_size_guess();
710 RSEQ_INJECT_FAILED
711 return -1;
712 cmpfail:
713 rseq_workaround_gcc_asm_size_guess();
714 return 1;
715 #ifdef RSEQ_COMPARE_TWICE
716 error1:
717 rseq_workaround_gcc_asm_size_guess();
718 rseq_bug("cpu_id comparison failed");
719 error2:
720 rseq_workaround_gcc_asm_size_guess();
721 rseq_bug("expected value comparison failed");
722 #endif
723 }
724
725 #endif /* !RSEQ_SKIP_FASTPATH */
726