• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3  * Author: Paul Burton <paul.burton@mips.com>
4  * (C) Copyright 2018 MIPS Tech LLC
5  *
6  * Based on rseq-arm.h:
7  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
8  */
9 
10 #define RSEQ_SIG	0x53053053
11 
12 #define rseq_smp_mb()	__asm__ __volatile__ ("sync" ::: "memory")
13 #define rseq_smp_rmb()	rseq_smp_mb()
14 #define rseq_smp_wmb()	rseq_smp_mb()
15 
16 #define rseq_smp_load_acquire(p)					\
17 __extension__ ({							\
18 	__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);			\
19 	rseq_smp_mb();							\
20 	____p1;								\
21 })
22 
23 #define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()
24 
25 #define rseq_smp_store_release(p, v)					\
26 do {									\
27 	rseq_smp_mb();							\
28 	RSEQ_WRITE_ONCE(*p, v);						\
29 } while (0)
30 
31 #ifdef RSEQ_SKIP_FASTPATH
32 #include "rseq-skip.h"
33 #else /* !RSEQ_SKIP_FASTPATH */
34 
35 #if _MIPS_SZLONG == 64
36 # define LONG			".dword"
37 # define LONG_LA		"dla"
38 # define LONG_L			"ld"
39 # define LONG_S			"sd"
40 # define LONG_ADDI		"daddiu"
41 # define U32_U64_PAD(x)		x
42 #elif _MIPS_SZLONG == 32
43 # define LONG			".word"
44 # define LONG_LA		"la"
45 # define LONG_L			"lw"
46 # define LONG_S			"sw"
47 # define LONG_ADDI		"addiu"
48 # ifdef __BIG_ENDIAN
49 #  define U32_U64_PAD(x)	"0x0, " x
50 # else
51 #  define U32_U64_PAD(x)	x ", 0x0"
52 # endif
53 #else
54 # error unsupported _MIPS_SZLONG
55 #endif
56 
57 #define __RSEQ_ASM_DEFINE_TABLE(version, flags,	start_ip, \
58 				post_commit_offset, abort_ip) \
59 		".pushsection __rseq_table, \"aw\"\n\t" \
60 		".balign 32\n\t" \
61 		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
62 		LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
63 		LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
64 		LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
65 		".popsection\n\t"
66 
67 #define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \
68 	__RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \
69 				(post_commit_ip - start_ip), abort_ip)
70 
71 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
72 		RSEQ_INJECT_ASM(1) \
73 		LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \
74 		LONG_S  " $4, %[" __rseq_str(rseq_cs) "]\n\t" \
75 		__rseq_str(label) ":\n\t"
76 
77 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
78 		RSEQ_INJECT_ASM(2) \
79 		"lw  $4, %[" __rseq_str(current_cpu_id) "]\n\t" \
80 		"bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t"
81 
82 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
83 				abort_label, version, flags, \
84 				start_ip, post_commit_offset, abort_ip) \
85 		".balign 32\n\t" \
86 		__rseq_str(table_label) ":\n\t" \
87 		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
88 		LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
89 		LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
90 		LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
91 		".word " __rseq_str(RSEQ_SIG) "\n\t" \
92 		__rseq_str(label) ":\n\t" \
93 		teardown \
94 		"b %l[" __rseq_str(abort_label) "]\n\t"
95 
96 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
97 			      start_ip, post_commit_ip, abort_ip) \
98 	__RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
99 				abort_label, 0x0, 0x0, start_ip, \
100 				(post_commit_ip - start_ip), abort_ip)
101 
102 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
103 		__rseq_str(label) ":\n\t" \
104 		teardown \
105 		"b %l[" __rseq_str(cmpfail_label) "]\n\t"
106 
107 #define rseq_workaround_gcc_asm_size_guess()	__asm__ __volatile__("")
108 
109 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)110 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
111 {
112 	RSEQ_INJECT_C(9)
113 
114 	rseq_workaround_gcc_asm_size_guess();
115 	__asm__ __volatile__ goto (
116 		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
117 		/* Start rseq by storing table entry pointer into rseq_cs. */
118 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
119 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
120 		RSEQ_INJECT_ASM(3)
121 		LONG_L " $4, %[v]\n\t"
122 		"bne $4, %[expect], %l[cmpfail]\n\t"
123 		RSEQ_INJECT_ASM(4)
124 #ifdef RSEQ_COMPARE_TWICE
125 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
126 		LONG_L " $4, %[v]\n\t"
127 		"bne $4, %[expect], %l[error2]\n\t"
128 #endif
129 		/* final store */
130 		LONG_S " %[newv], %[v]\n\t"
131 		"2:\n\t"
132 		RSEQ_INJECT_ASM(5)
133 		"b 5f\n\t"
134 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
135 		"5:\n\t"
136 		: /* gcc asm goto does not allow outputs */
137 		: [cpu_id]		"r" (cpu),
138 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
139 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
140 		  [v]			"m" (*v),
141 		  [expect]		"r" (expect),
142 		  [newv]		"r" (newv)
143 		  RSEQ_INJECT_INPUT
144 		: "$4", "memory"
145 		  RSEQ_INJECT_CLOBBER
146 		: abort, cmpfail
147 #ifdef RSEQ_COMPARE_TWICE
148 		  , error1, error2
149 #endif
150 	);
151 	rseq_workaround_gcc_asm_size_guess();
152 	return 0;
153 abort:
154 	rseq_workaround_gcc_asm_size_guess();
155 	RSEQ_INJECT_FAILED
156 	return -1;
157 cmpfail:
158 	rseq_workaround_gcc_asm_size_guess();
159 	return 1;
160 #ifdef RSEQ_COMPARE_TWICE
161 error1:
162 	rseq_bug("cpu_id comparison failed");
163 error2:
164 	rseq_bug("expected value comparison failed");
165 #endif
166 }
167 
168 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)169 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
170 			       off_t voffp, intptr_t *load, int cpu)
171 {
172 	RSEQ_INJECT_C(9)
173 
174 	rseq_workaround_gcc_asm_size_guess();
175 	__asm__ __volatile__ goto (
176 		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
177 		/* Start rseq by storing table entry pointer into rseq_cs. */
178 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
179 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
180 		RSEQ_INJECT_ASM(3)
181 		LONG_L " $4, %[v]\n\t"
182 		"beq $4, %[expectnot], %l[cmpfail]\n\t"
183 		RSEQ_INJECT_ASM(4)
184 #ifdef RSEQ_COMPARE_TWICE
185 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
186 		LONG_L " $4, %[v]\n\t"
187 		"beq $4, %[expectnot], %l[error2]\n\t"
188 #endif
189 		LONG_S " $4, %[load]\n\t"
190 		LONG_ADDI " $4, %[voffp]\n\t"
191 		LONG_L " $4, 0($4)\n\t"
192 		/* final store */
193 		LONG_S " $4, %[v]\n\t"
194 		"2:\n\t"
195 		RSEQ_INJECT_ASM(5)
196 		"b 5f\n\t"
197 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
198 		"5:\n\t"
199 		: /* gcc asm goto does not allow outputs */
200 		: [cpu_id]		"r" (cpu),
201 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
202 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
203 		  /* final store input */
204 		  [v]			"m" (*v),
205 		  [expectnot]		"r" (expectnot),
206 		  [voffp]		"Ir" (voffp),
207 		  [load]		"m" (*load)
208 		  RSEQ_INJECT_INPUT
209 		: "$4", "memory"
210 		  RSEQ_INJECT_CLOBBER
211 		: abort, cmpfail
212 #ifdef RSEQ_COMPARE_TWICE
213 		  , error1, error2
214 #endif
215 	);
216 	rseq_workaround_gcc_asm_size_guess();
217 	return 0;
218 abort:
219 	rseq_workaround_gcc_asm_size_guess();
220 	RSEQ_INJECT_FAILED
221 	return -1;
222 cmpfail:
223 	rseq_workaround_gcc_asm_size_guess();
224 	return 1;
225 #ifdef RSEQ_COMPARE_TWICE
226 error1:
227 	rseq_bug("cpu_id comparison failed");
228 error2:
229 	rseq_bug("expected value comparison failed");
230 #endif
231 }
232 
233 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)234 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
235 {
236 	RSEQ_INJECT_C(9)
237 
238 	rseq_workaround_gcc_asm_size_guess();
239 	__asm__ __volatile__ goto (
240 		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
241 		/* Start rseq by storing table entry pointer into rseq_cs. */
242 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
243 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
244 		RSEQ_INJECT_ASM(3)
245 #ifdef RSEQ_COMPARE_TWICE
246 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
247 #endif
248 		LONG_L " $4, %[v]\n\t"
249 		LONG_ADDI " $4, %[count]\n\t"
250 		/* final store */
251 		LONG_S " $4, %[v]\n\t"
252 		"2:\n\t"
253 		RSEQ_INJECT_ASM(4)
254 		"b 5f\n\t"
255 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
256 		"5:\n\t"
257 		: /* gcc asm goto does not allow outputs */
258 		: [cpu_id]		"r" (cpu),
259 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
260 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
261 		  [v]			"m" (*v),
262 		  [count]		"Ir" (count)
263 		  RSEQ_INJECT_INPUT
264 		: "$4", "memory"
265 		  RSEQ_INJECT_CLOBBER
266 		: abort
267 #ifdef RSEQ_COMPARE_TWICE
268 		  , error1
269 #endif
270 	);
271 	rseq_workaround_gcc_asm_size_guess();
272 	return 0;
273 abort:
274 	rseq_workaround_gcc_asm_size_guess();
275 	RSEQ_INJECT_FAILED
276 	return -1;
277 #ifdef RSEQ_COMPARE_TWICE
278 error1:
279 	rseq_bug("cpu_id comparison failed");
280 #endif
281 }
282 
283 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)284 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
285 				 intptr_t *v2, intptr_t newv2,
286 				 intptr_t newv, int cpu)
287 {
288 	RSEQ_INJECT_C(9)
289 
290 	rseq_workaround_gcc_asm_size_guess();
291 	__asm__ __volatile__ goto (
292 		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
293 		/* Start rseq by storing table entry pointer into rseq_cs. */
294 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
295 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
296 		RSEQ_INJECT_ASM(3)
297 		LONG_L " $4, %[v]\n\t"
298 		"bne $4, %[expect], %l[cmpfail]\n\t"
299 		RSEQ_INJECT_ASM(4)
300 #ifdef RSEQ_COMPARE_TWICE
301 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
302 		LONG_L " $4, %[v]\n\t"
303 		"bne $4, %[expect], %l[error2]\n\t"
304 #endif
305 		/* try store */
306 		LONG_S " %[newv2], %[v2]\n\t"
307 		RSEQ_INJECT_ASM(5)
308 		/* final store */
309 		LONG_S " %[newv], %[v]\n\t"
310 		"2:\n\t"
311 		RSEQ_INJECT_ASM(6)
312 		"b 5f\n\t"
313 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
314 		"5:\n\t"
315 		: /* gcc asm goto does not allow outputs */
316 		: [cpu_id]		"r" (cpu),
317 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
318 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
319 		  /* try store input */
320 		  [v2]			"m" (*v2),
321 		  [newv2]		"r" (newv2),
322 		  /* final store input */
323 		  [v]			"m" (*v),
324 		  [expect]		"r" (expect),
325 		  [newv]		"r" (newv)
326 		  RSEQ_INJECT_INPUT
327 		: "$4", "memory"
328 		  RSEQ_INJECT_CLOBBER
329 		: abort, cmpfail
330 #ifdef RSEQ_COMPARE_TWICE
331 		  , error1, error2
332 #endif
333 	);
334 	rseq_workaround_gcc_asm_size_guess();
335 	return 0;
336 abort:
337 	rseq_workaround_gcc_asm_size_guess();
338 	RSEQ_INJECT_FAILED
339 	return -1;
340 cmpfail:
341 	rseq_workaround_gcc_asm_size_guess();
342 	return 1;
343 #ifdef RSEQ_COMPARE_TWICE
344 error1:
345 	rseq_bug("cpu_id comparison failed");
346 error2:
347 	rseq_bug("expected value comparison failed");
348 #endif
349 }
350 
351 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)352 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
353 					 intptr_t *v2, intptr_t newv2,
354 					 intptr_t newv, int cpu)
355 {
356 	RSEQ_INJECT_C(9)
357 
358 	rseq_workaround_gcc_asm_size_guess();
359 	__asm__ __volatile__ goto (
360 		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
361 		/* Start rseq by storing table entry pointer into rseq_cs. */
362 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
363 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
364 		RSEQ_INJECT_ASM(3)
365 		LONG_L " $4, %[v]\n\t"
366 		"bne $4, %[expect], %l[cmpfail]\n\t"
367 		RSEQ_INJECT_ASM(4)
368 #ifdef RSEQ_COMPARE_TWICE
369 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
370 		LONG_L " $4, %[v]\n\t"
371 		"bne $4, %[expect], %l[error2]\n\t"
372 #endif
373 		/* try store */
374 		LONG_S " %[newv2], %[v2]\n\t"
375 		RSEQ_INJECT_ASM(5)
376 		"sync\n\t"	/* full sync provides store-release */
377 		/* final store */
378 		LONG_S " %[newv], %[v]\n\t"
379 		"2:\n\t"
380 		RSEQ_INJECT_ASM(6)
381 		"b 5f\n\t"
382 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
383 		"5:\n\t"
384 		: /* gcc asm goto does not allow outputs */
385 		: [cpu_id]		"r" (cpu),
386 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
387 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
388 		  /* try store input */
389 		  [v2]			"m" (*v2),
390 		  [newv2]		"r" (newv2),
391 		  /* final store input */
392 		  [v]			"m" (*v),
393 		  [expect]		"r" (expect),
394 		  [newv]		"r" (newv)
395 		  RSEQ_INJECT_INPUT
396 		: "$4", "memory"
397 		  RSEQ_INJECT_CLOBBER
398 		: abort, cmpfail
399 #ifdef RSEQ_COMPARE_TWICE
400 		  , error1, error2
401 #endif
402 	);
403 	rseq_workaround_gcc_asm_size_guess();
404 	return 0;
405 abort:
406 	rseq_workaround_gcc_asm_size_guess();
407 	RSEQ_INJECT_FAILED
408 	return -1;
409 cmpfail:
410 	rseq_workaround_gcc_asm_size_guess();
411 	return 1;
412 #ifdef RSEQ_COMPARE_TWICE
413 error1:
414 	rseq_bug("cpu_id comparison failed");
415 error2:
416 	rseq_bug("expected value comparison failed");
417 #endif
418 }
419 
420 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)421 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
422 			      intptr_t *v2, intptr_t expect2,
423 			      intptr_t newv, int cpu)
424 {
425 	RSEQ_INJECT_C(9)
426 
427 	rseq_workaround_gcc_asm_size_guess();
428 	__asm__ __volatile__ goto (
429 		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
430 		/* Start rseq by storing table entry pointer into rseq_cs. */
431 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
432 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
433 		RSEQ_INJECT_ASM(3)
434 		LONG_L " $4, %[v]\n\t"
435 		"bne $4, %[expect], %l[cmpfail]\n\t"
436 		RSEQ_INJECT_ASM(4)
437 		LONG_L " $4, %[v2]\n\t"
438 		"bne $4, %[expect2], %l[cmpfail]\n\t"
439 		RSEQ_INJECT_ASM(5)
440 #ifdef RSEQ_COMPARE_TWICE
441 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
442 		LONG_L " $4, %[v]\n\t"
443 		"bne $4, %[expect], %l[error2]\n\t"
444 		LONG_L " $4, %[v2]\n\t"
445 		"bne $4, %[expect2], %l[error3]\n\t"
446 #endif
447 		/* final store */
448 		LONG_S " %[newv], %[v]\n\t"
449 		"2:\n\t"
450 		RSEQ_INJECT_ASM(6)
451 		"b 5f\n\t"
452 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
453 		"5:\n\t"
454 		: /* gcc asm goto does not allow outputs */
455 		: [cpu_id]		"r" (cpu),
456 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
457 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
458 		  /* cmp2 input */
459 		  [v2]			"m" (*v2),
460 		  [expect2]		"r" (expect2),
461 		  /* final store input */
462 		  [v]			"m" (*v),
463 		  [expect]		"r" (expect),
464 		  [newv]		"r" (newv)
465 		  RSEQ_INJECT_INPUT
466 		: "$4", "memory"
467 		  RSEQ_INJECT_CLOBBER
468 		: abort, cmpfail
469 #ifdef RSEQ_COMPARE_TWICE
470 		  , error1, error2, error3
471 #endif
472 	);
473 	rseq_workaround_gcc_asm_size_guess();
474 	return 0;
475 abort:
476 	rseq_workaround_gcc_asm_size_guess();
477 	RSEQ_INJECT_FAILED
478 	return -1;
479 cmpfail:
480 	rseq_workaround_gcc_asm_size_guess();
481 	return 1;
482 #ifdef RSEQ_COMPARE_TWICE
483 error1:
484 	rseq_bug("cpu_id comparison failed");
485 error2:
486 	rseq_bug("1st expected value comparison failed");
487 error3:
488 	rseq_bug("2nd expected value comparison failed");
489 #endif
490 }
491 
492 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)493 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
494 				 void *dst, void *src, size_t len,
495 				 intptr_t newv, int cpu)
496 {
497 	uintptr_t rseq_scratch[3];
498 
499 	RSEQ_INJECT_C(9)
500 
501 	rseq_workaround_gcc_asm_size_guess();
502 	__asm__ __volatile__ goto (
503 		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
504 		LONG_S " %[src], %[rseq_scratch0]\n\t"
505 		LONG_S "  %[dst], %[rseq_scratch1]\n\t"
506 		LONG_S " %[len], %[rseq_scratch2]\n\t"
507 		/* Start rseq by storing table entry pointer into rseq_cs. */
508 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
509 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
510 		RSEQ_INJECT_ASM(3)
511 		LONG_L " $4, %[v]\n\t"
512 		"bne $4, %[expect], 5f\n\t"
513 		RSEQ_INJECT_ASM(4)
514 #ifdef RSEQ_COMPARE_TWICE
515 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
516 		LONG_L " $4, %[v]\n\t"
517 		"bne $4, %[expect], 7f\n\t"
518 #endif
519 		/* try memcpy */
520 		"beqz %[len], 333f\n\t" \
521 		"222:\n\t" \
522 		"lb   $4, 0(%[src])\n\t" \
523 		"sb   $4, 0(%[dst])\n\t" \
524 		LONG_ADDI " %[src], 1\n\t" \
525 		LONG_ADDI " %[dst], 1\n\t" \
526 		LONG_ADDI " %[len], -1\n\t" \
527 		"bnez %[len], 222b\n\t" \
528 		"333:\n\t" \
529 		RSEQ_INJECT_ASM(5)
530 		/* final store */
531 		LONG_S " %[newv], %[v]\n\t"
532 		"2:\n\t"
533 		RSEQ_INJECT_ASM(6)
534 		/* teardown */
535 		LONG_L " %[len], %[rseq_scratch2]\n\t"
536 		LONG_L " %[dst], %[rseq_scratch1]\n\t"
537 		LONG_L " %[src], %[rseq_scratch0]\n\t"
538 		"b 8f\n\t"
539 		RSEQ_ASM_DEFINE_ABORT(3, 4,
540 				      /* teardown */
541 				      LONG_L " %[len], %[rseq_scratch2]\n\t"
542 				      LONG_L " %[dst], %[rseq_scratch1]\n\t"
543 				      LONG_L " %[src], %[rseq_scratch0]\n\t",
544 				      abort, 1b, 2b, 4f)
545 		RSEQ_ASM_DEFINE_CMPFAIL(5,
546 					/* teardown */
547 					LONG_L " %[len], %[rseq_scratch2]\n\t"
548 					LONG_L " %[dst], %[rseq_scratch1]\n\t"
549 					LONG_L " %[src], %[rseq_scratch0]\n\t",
550 					cmpfail)
551 #ifdef RSEQ_COMPARE_TWICE
552 		RSEQ_ASM_DEFINE_CMPFAIL(6,
553 					/* teardown */
554 					LONG_L " %[len], %[rseq_scratch2]\n\t"
555 					LONG_L " %[dst], %[rseq_scratch1]\n\t"
556 					LONG_L " %[src], %[rseq_scratch0]\n\t",
557 					error1)
558 		RSEQ_ASM_DEFINE_CMPFAIL(7,
559 					/* teardown */
560 					LONG_L " %[len], %[rseq_scratch2]\n\t"
561 					LONG_L " %[dst], %[rseq_scratch1]\n\t"
562 					LONG_L " %[src], %[rseq_scratch0]\n\t",
563 					error2)
564 #endif
565 		"8:\n\t"
566 		: /* gcc asm goto does not allow outputs */
567 		: [cpu_id]		"r" (cpu),
568 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
569 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
570 		  /* final store input */
571 		  [v]			"m" (*v),
572 		  [expect]		"r" (expect),
573 		  [newv]		"r" (newv),
574 		  /* try memcpy input */
575 		  [dst]			"r" (dst),
576 		  [src]			"r" (src),
577 		  [len]			"r" (len),
578 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
579 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
580 		  [rseq_scratch2]	"m" (rseq_scratch[2])
581 		  RSEQ_INJECT_INPUT
582 		: "$4", "memory"
583 		  RSEQ_INJECT_CLOBBER
584 		: abort, cmpfail
585 #ifdef RSEQ_COMPARE_TWICE
586 		  , error1, error2
587 #endif
588 	);
589 	rseq_workaround_gcc_asm_size_guess();
590 	return 0;
591 abort:
592 	rseq_workaround_gcc_asm_size_guess();
593 	RSEQ_INJECT_FAILED
594 	return -1;
595 cmpfail:
596 	rseq_workaround_gcc_asm_size_guess();
597 	return 1;
598 #ifdef RSEQ_COMPARE_TWICE
599 error1:
600 	rseq_workaround_gcc_asm_size_guess();
601 	rseq_bug("cpu_id comparison failed");
602 error2:
603 	rseq_workaround_gcc_asm_size_guess();
604 	rseq_bug("expected value comparison failed");
605 #endif
606 }
607 
608 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)609 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
610 					 void *dst, void *src, size_t len,
611 					 intptr_t newv, int cpu)
612 {
613 	uintptr_t rseq_scratch[3];
614 
615 	RSEQ_INJECT_C(9)
616 
617 	rseq_workaround_gcc_asm_size_guess();
618 	__asm__ __volatile__ goto (
619 		RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
620 		LONG_S " %[src], %[rseq_scratch0]\n\t"
621 		LONG_S " %[dst], %[rseq_scratch1]\n\t"
622 		LONG_S " %[len], %[rseq_scratch2]\n\t"
623 		/* Start rseq by storing table entry pointer into rseq_cs. */
624 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
625 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
626 		RSEQ_INJECT_ASM(3)
627 		LONG_L " $4, %[v]\n\t"
628 		"bne $4, %[expect], 5f\n\t"
629 		RSEQ_INJECT_ASM(4)
630 #ifdef RSEQ_COMPARE_TWICE
631 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
632 		LONG_L " $4, %[v]\n\t"
633 		"bne $4, %[expect], 7f\n\t"
634 #endif
635 		/* try memcpy */
636 		"beqz %[len], 333f\n\t" \
637 		"222:\n\t" \
638 		"lb   $4, 0(%[src])\n\t" \
639 		"sb   $4, 0(%[dst])\n\t" \
640 		LONG_ADDI " %[src], 1\n\t" \
641 		LONG_ADDI " %[dst], 1\n\t" \
642 		LONG_ADDI " %[len], -1\n\t" \
643 		"bnez %[len], 222b\n\t" \
644 		"333:\n\t" \
645 		RSEQ_INJECT_ASM(5)
646 		"sync\n\t"	/* full sync provides store-release */
647 		/* final store */
648 		LONG_S " %[newv], %[v]\n\t"
649 		"2:\n\t"
650 		RSEQ_INJECT_ASM(6)
651 		/* teardown */
652 		LONG_L " %[len], %[rseq_scratch2]\n\t"
653 		LONG_L " %[dst], %[rseq_scratch1]\n\t"
654 		LONG_L " %[src], %[rseq_scratch0]\n\t"
655 		"b 8f\n\t"
656 		RSEQ_ASM_DEFINE_ABORT(3, 4,
657 				      /* teardown */
658 				      LONG_L " %[len], %[rseq_scratch2]\n\t"
659 				      LONG_L " %[dst], %[rseq_scratch1]\n\t"
660 				      LONG_L " %[src], %[rseq_scratch0]\n\t",
661 				      abort, 1b, 2b, 4f)
662 		RSEQ_ASM_DEFINE_CMPFAIL(5,
663 					/* teardown */
664 					LONG_L " %[len], %[rseq_scratch2]\n\t"
665 					LONG_L " %[dst], %[rseq_scratch1]\n\t"
666 					LONG_L " %[src], %[rseq_scratch0]\n\t",
667 					cmpfail)
668 #ifdef RSEQ_COMPARE_TWICE
669 		RSEQ_ASM_DEFINE_CMPFAIL(6,
670 					/* teardown */
671 					LONG_L " %[len], %[rseq_scratch2]\n\t"
672 					LONG_L " %[dst], %[rseq_scratch1]\n\t"
673 					LONG_L " %[src], %[rseq_scratch0]\n\t",
674 					error1)
675 		RSEQ_ASM_DEFINE_CMPFAIL(7,
676 					/* teardown */
677 					LONG_L " %[len], %[rseq_scratch2]\n\t"
678 					LONG_L " %[dst], %[rseq_scratch1]\n\t"
679 					LONG_L " %[src], %[rseq_scratch0]\n\t",
680 					error2)
681 #endif
682 		"8:\n\t"
683 		: /* gcc asm goto does not allow outputs */
684 		: [cpu_id]		"r" (cpu),
685 		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
686 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
687 		  /* final store input */
688 		  [v]			"m" (*v),
689 		  [expect]		"r" (expect),
690 		  [newv]		"r" (newv),
691 		  /* try memcpy input */
692 		  [dst]			"r" (dst),
693 		  [src]			"r" (src),
694 		  [len]			"r" (len),
695 		  [rseq_scratch0]	"m" (rseq_scratch[0]),
696 		  [rseq_scratch1]	"m" (rseq_scratch[1]),
697 		  [rseq_scratch2]	"m" (rseq_scratch[2])
698 		  RSEQ_INJECT_INPUT
699 		: "$4", "memory"
700 		  RSEQ_INJECT_CLOBBER
701 		: abort, cmpfail
702 #ifdef RSEQ_COMPARE_TWICE
703 		  , error1, error2
704 #endif
705 	);
706 	rseq_workaround_gcc_asm_size_guess();
707 	return 0;
708 abort:
709 	rseq_workaround_gcc_asm_size_guess();
710 	RSEQ_INJECT_FAILED
711 	return -1;
712 cmpfail:
713 	rseq_workaround_gcc_asm_size_guess();
714 	return 1;
715 #ifdef RSEQ_COMPARE_TWICE
716 error1:
717 	rseq_workaround_gcc_asm_size_guess();
718 	rseq_bug("cpu_id comparison failed");
719 error2:
720 	rseq_workaround_gcc_asm_size_guess();
721 	rseq_bug("expected value comparison failed");
722 #endif
723 }
724 
725 #endif /* !RSEQ_SKIP_FASTPATH */
726