• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2 /*
3  * rseq-arm64.h
4  *
5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6  * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
7  */
8 
9 #define RSEQ_SIG	0xd428bc00	/* BRK #0x45E0 */
10 
11 #define rseq_smp_mb()	__asm__ __volatile__ ("dmb ish" ::: "memory")
12 #define rseq_smp_rmb()	__asm__ __volatile__ ("dmb ishld" ::: "memory")
13 #define rseq_smp_wmb()	__asm__ __volatile__ ("dmb ishst" ::: "memory")
14 
15 #define rseq_smp_load_acquire(p)						\
16 __extension__ ({								\
17 	__typeof(*p) ____p1;							\
18 	switch (sizeof(*p)) {							\
19 	case 1:									\
20 		asm volatile ("ldarb %w0, %1"					\
21 			: "=r" (*(__u8 *)p)					\
22 			: "Q" (*p) : "memory");					\
23 		break;								\
24 	case 2:									\
25 		asm volatile ("ldarh %w0, %1"					\
26 			: "=r" (*(__u16 *)p)					\
27 			: "Q" (*p) : "memory");					\
28 		break;								\
29 	case 4:									\
30 		asm volatile ("ldar %w0, %1"					\
31 			: "=r" (*(__u32 *)p)					\
32 			: "Q" (*p) : "memory");					\
33 		break;								\
34 	case 8:									\
35 		asm volatile ("ldar %0, %1"					\
36 			: "=r" (*(__u64 *)p)					\
37 			: "Q" (*p) : "memory");					\
38 		break;								\
39 	}									\
40 	____p1;									\
41 })
42 
43 #define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()
44 
45 #define rseq_smp_store_release(p, v)						\
46 do {										\
47 	switch (sizeof(*p)) {							\
48 	case 1:									\
49 		asm volatile ("stlrb %w1, %0"					\
50 				: "=Q" (*p)					\
51 				: "r" ((__u8)v)					\
52 				: "memory");					\
53 		break;								\
54 	case 2:									\
55 		asm volatile ("stlrh %w1, %0"					\
56 				: "=Q" (*p)					\
57 				: "r" ((__u16)v)				\
58 				: "memory");					\
59 		break;								\
60 	case 4:									\
61 		asm volatile ("stlr %w1, %0"					\
62 				: "=Q" (*p)					\
63 				: "r" ((__u32)v)				\
64 				: "memory");					\
65 		break;								\
66 	case 8:									\
67 		asm volatile ("stlr %1, %0"					\
68 				: "=Q" (*p)					\
69 				: "r" ((__u64)v)				\
70 				: "memory");					\
71 		break;								\
72 	}									\
73 } while (0)
74 
75 #ifdef RSEQ_SKIP_FASTPATH
76 #include "rseq-skip.h"
77 #else /* !RSEQ_SKIP_FASTPATH */
78 
79 #define RSEQ_ASM_TMP_REG32	"w15"
80 #define RSEQ_ASM_TMP_REG	"x15"
81 #define RSEQ_ASM_TMP_REG_2	"x14"
82 
83 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,		\
84 				post_commit_offset, abort_ip)			\
85 	"	.pushsection	__rseq_table, \"aw\"\n"				\
86 	"	.balign	32\n"							\
87 	__rseq_str(label) ":\n"							\
88 	"	.long	" __rseq_str(version) ", " __rseq_str(flags) "\n"	\
89 	"	.quad	" __rseq_str(start_ip) ", "				\
90 			  __rseq_str(post_commit_offset) ", "			\
91 			  __rseq_str(abort_ip) "\n"				\
92 	"	.popsection\n"
93 
94 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)	\
95 	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,			\
96 				(post_commit_ip - start_ip), abort_ip)
97 
98 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
99 	RSEQ_INJECT_ASM(1)							\
100 	"	adrp	" RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n"	\
101 	"	add	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
102 			", :lo12:" __rseq_str(cs_label) "\n"			\
103 	"	str	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n"	\
104 	__rseq_str(label) ":\n"
105 
106 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label)				\
107 	"	b	222f\n"							\
108 	"	.inst 	"	__rseq_str(RSEQ_SIG) "\n"			\
109 	__rseq_str(label) ":\n"							\
110 	"	b	%l[" __rseq_str(abort_label) "]\n"			\
111 	"222:\n"
112 
113 #define RSEQ_ASM_OP_STORE(value, var)						\
114 	"	str	%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
115 
116 #define RSEQ_ASM_OP_STORE_RELEASE(value, var)					\
117 	"	stlr	%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
118 
119 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)			\
120 	RSEQ_ASM_OP_STORE(value, var)						\
121 	__rseq_str(post_commit_label) ":\n"
122 
123 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)		\
124 	RSEQ_ASM_OP_STORE_RELEASE(value, var)					\
125 	__rseq_str(post_commit_label) ":\n"
126 
127 #define RSEQ_ASM_OP_CMPEQ(var, expect, label)					\
128 	"	ldr	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"		\
129 	"	sub	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
130 			", %[" __rseq_str(expect) "]\n"				\
131 	"	cbnz	" RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
132 
133 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label)					\
134 	"	ldr	" RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n"	\
135 	"	sub	" RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32		\
136 			", %w[" __rseq_str(expect) "]\n"			\
137 	"	cbnz	" RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
138 
139 #define RSEQ_ASM_OP_CMPNE(var, expect, label)					\
140 	"	ldr	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"		\
141 	"	sub	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
142 			", %[" __rseq_str(expect) "]\n"				\
143 	"	cbz	" RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
144 
145 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)			\
146 	RSEQ_INJECT_ASM(2)							\
147 	RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
148 
149 #define RSEQ_ASM_OP_R_LOAD(var)							\
150 	"	ldr	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
151 
152 #define RSEQ_ASM_OP_R_STORE(var)						\
153 	"	str	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
154 
155 #define RSEQ_ASM_OP_R_LOAD_OFF(offset)						\
156 	"	ldr	" RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG		\
157 			", %[" __rseq_str(offset) "]]\n"
158 
159 #define RSEQ_ASM_OP_R_ADD(count)						\
160 	"	add	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
161 			", %[" __rseq_str(count) "]\n"
162 
163 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)			\
164 	"	str	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"		\
165 	__rseq_str(post_commit_label) ":\n"
166 
167 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)					\
168 	"	cbz	%[" __rseq_str(len) "], 333f\n"				\
169 	"	mov	" RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n"	\
170 	"222:	sub	" RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n"	\
171 	"	ldrb	" RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]"	\
172 			", " RSEQ_ASM_TMP_REG_2 "]\n"				\
173 	"	strb	" RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]"	\
174 			", " RSEQ_ASM_TMP_REG_2 "]\n"				\
175 	"	cbnz	" RSEQ_ASM_TMP_REG_2 ", 222b\n"				\
176 	"333:\n"
177 
178 static inline __attribute__((always_inline))
rseq_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t newv,int cpu)179 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
180 {
181 	RSEQ_INJECT_C(9)
182 
183 	__asm__ __volatile__ goto (
184 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
185 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
186 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
187 		RSEQ_INJECT_ASM(3)
188 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
189 		RSEQ_INJECT_ASM(4)
190 #ifdef RSEQ_COMPARE_TWICE
191 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
192 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
193 #endif
194 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
195 		RSEQ_INJECT_ASM(5)
196 		RSEQ_ASM_DEFINE_ABORT(4, abort)
197 		: /* gcc asm goto does not allow outputs */
198 		: [cpu_id]		"r" (cpu),
199 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
200 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
201 		  [v]			"Qo" (*v),
202 		  [expect]		"r" (expect),
203 		  [newv]		"r" (newv)
204 		  RSEQ_INJECT_INPUT
205 		: "memory", RSEQ_ASM_TMP_REG
206 		: abort, cmpfail
207 #ifdef RSEQ_COMPARE_TWICE
208 		  , error1, error2
209 #endif
210 	);
211 
212 	return 0;
213 abort:
214 	RSEQ_INJECT_FAILED
215 	return -1;
216 cmpfail:
217 	return 1;
218 #ifdef RSEQ_COMPARE_TWICE
219 error1:
220 	rseq_bug("cpu_id comparison failed");
221 error2:
222 	rseq_bug("expected value comparison failed");
223 #endif
224 }
225 
226 static inline __attribute__((always_inline))
rseq_cmpnev_storeoffp_load(intptr_t * v,intptr_t expectnot,off_t voffp,intptr_t * load,int cpu)227 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
228 			       off_t voffp, intptr_t *load, int cpu)
229 {
230 	RSEQ_INJECT_C(9)
231 
232 	__asm__ __volatile__ goto (
233 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
234 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
235 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
236 		RSEQ_INJECT_ASM(3)
237 		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
238 		RSEQ_INJECT_ASM(4)
239 #ifdef RSEQ_COMPARE_TWICE
240 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
241 		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
242 #endif
243 		RSEQ_ASM_OP_R_LOAD(v)
244 		RSEQ_ASM_OP_R_STORE(load)
245 		RSEQ_ASM_OP_R_LOAD_OFF(voffp)
246 		RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
247 		RSEQ_INJECT_ASM(5)
248 		RSEQ_ASM_DEFINE_ABORT(4, abort)
249 		: /* gcc asm goto does not allow outputs */
250 		: [cpu_id]		"r" (cpu),
251 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
252 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
253 		  [v]			"Qo" (*v),
254 		  [expectnot]		"r" (expectnot),
255 		  [load]		"Qo" (*load),
256 		  [voffp]		"r" (voffp)
257 		  RSEQ_INJECT_INPUT
258 		: "memory", RSEQ_ASM_TMP_REG
259 		: abort, cmpfail
260 #ifdef RSEQ_COMPARE_TWICE
261 		  , error1, error2
262 #endif
263 	);
264 	return 0;
265 abort:
266 	RSEQ_INJECT_FAILED
267 	return -1;
268 cmpfail:
269 	return 1;
270 #ifdef RSEQ_COMPARE_TWICE
271 error1:
272 	rseq_bug("cpu_id comparison failed");
273 error2:
274 	rseq_bug("expected value comparison failed");
275 #endif
276 }
277 
278 static inline __attribute__((always_inline))
rseq_addv(intptr_t * v,intptr_t count,int cpu)279 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
280 {
281 	RSEQ_INJECT_C(9)
282 
283 	__asm__ __volatile__ goto (
284 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
285 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
286 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
287 		RSEQ_INJECT_ASM(3)
288 #ifdef RSEQ_COMPARE_TWICE
289 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
290 #endif
291 		RSEQ_ASM_OP_R_LOAD(v)
292 		RSEQ_ASM_OP_R_ADD(count)
293 		RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
294 		RSEQ_INJECT_ASM(4)
295 		RSEQ_ASM_DEFINE_ABORT(4, abort)
296 		: /* gcc asm goto does not allow outputs */
297 		: [cpu_id]		"r" (cpu),
298 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
299 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
300 		  [v]			"Qo" (*v),
301 		  [count]		"r" (count)
302 		  RSEQ_INJECT_INPUT
303 		: "memory", RSEQ_ASM_TMP_REG
304 		: abort
305 #ifdef RSEQ_COMPARE_TWICE
306 		  , error1
307 #endif
308 	);
309 	return 0;
310 abort:
311 	RSEQ_INJECT_FAILED
312 	return -1;
313 #ifdef RSEQ_COMPARE_TWICE
314 error1:
315 	rseq_bug("cpu_id comparison failed");
316 #endif
317 }
318 
319 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)320 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
321 				 intptr_t *v2, intptr_t newv2,
322 				 intptr_t newv, int cpu)
323 {
324 	RSEQ_INJECT_C(9)
325 
326 	__asm__ __volatile__ goto (
327 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
328 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
329 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
330 		RSEQ_INJECT_ASM(3)
331 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
332 		RSEQ_INJECT_ASM(4)
333 #ifdef RSEQ_COMPARE_TWICE
334 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
335 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
336 #endif
337 		RSEQ_ASM_OP_STORE(newv2, v2)
338 		RSEQ_INJECT_ASM(5)
339 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
340 		RSEQ_INJECT_ASM(6)
341 		RSEQ_ASM_DEFINE_ABORT(4, abort)
342 		: /* gcc asm goto does not allow outputs */
343 		: [cpu_id]		"r" (cpu),
344 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
345 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
346 		  [expect]		"r" (expect),
347 		  [v]			"Qo" (*v),
348 		  [newv]		"r" (newv),
349 		  [v2]			"Qo" (*v2),
350 		  [newv2]		"r" (newv2)
351 		  RSEQ_INJECT_INPUT
352 		: "memory", RSEQ_ASM_TMP_REG
353 		: abort, cmpfail
354 #ifdef RSEQ_COMPARE_TWICE
355 		  , error1, error2
356 #endif
357 	);
358 
359 	return 0;
360 abort:
361 	RSEQ_INJECT_FAILED
362 	return -1;
363 cmpfail:
364 	return 1;
365 #ifdef RSEQ_COMPARE_TWICE
366 error1:
367 	rseq_bug("cpu_id comparison failed");
368 error2:
369 	rseq_bug("expected value comparison failed");
370 #endif
371 }
372 
373 static inline __attribute__((always_inline))
rseq_cmpeqv_trystorev_storev_release(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t newv2,intptr_t newv,int cpu)374 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
375 					 intptr_t *v2, intptr_t newv2,
376 					 intptr_t newv, int cpu)
377 {
378 	RSEQ_INJECT_C(9)
379 
380 	__asm__ __volatile__ goto (
381 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
382 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
383 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
384 		RSEQ_INJECT_ASM(3)
385 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
386 		RSEQ_INJECT_ASM(4)
387 #ifdef RSEQ_COMPARE_TWICE
388 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
389 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
390 #endif
391 		RSEQ_ASM_OP_STORE(newv2, v2)
392 		RSEQ_INJECT_ASM(5)
393 		RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
394 		RSEQ_INJECT_ASM(6)
395 		RSEQ_ASM_DEFINE_ABORT(4, abort)
396 		: /* gcc asm goto does not allow outputs */
397 		: [cpu_id]		"r" (cpu),
398 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
399 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
400 		  [expect]		"r" (expect),
401 		  [v]			"Qo" (*v),
402 		  [newv]		"r" (newv),
403 		  [v2]			"Qo" (*v2),
404 		  [newv2]		"r" (newv2)
405 		  RSEQ_INJECT_INPUT
406 		: "memory", RSEQ_ASM_TMP_REG
407 		: abort, cmpfail
408 #ifdef RSEQ_COMPARE_TWICE
409 		  , error1, error2
410 #endif
411 	);
412 
413 	return 0;
414 abort:
415 	RSEQ_INJECT_FAILED
416 	return -1;
417 cmpfail:
418 	return 1;
419 #ifdef RSEQ_COMPARE_TWICE
420 error1:
421 	rseq_bug("cpu_id comparison failed");
422 error2:
423 	rseq_bug("expected value comparison failed");
424 #endif
425 }
426 
427 static inline __attribute__((always_inline))
rseq_cmpeqv_cmpeqv_storev(intptr_t * v,intptr_t expect,intptr_t * v2,intptr_t expect2,intptr_t newv,int cpu)428 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
429 			      intptr_t *v2, intptr_t expect2,
430 			      intptr_t newv, int cpu)
431 {
432 	RSEQ_INJECT_C(9)
433 
434 	__asm__ __volatile__ goto (
435 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
436 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
437 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
438 		RSEQ_INJECT_ASM(3)
439 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
440 		RSEQ_INJECT_ASM(4)
441 		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
442 		RSEQ_INJECT_ASM(5)
443 #ifdef RSEQ_COMPARE_TWICE
444 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
445 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
446 		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
447 #endif
448 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
449 		RSEQ_INJECT_ASM(6)
450 		RSEQ_ASM_DEFINE_ABORT(4, abort)
451 		: /* gcc asm goto does not allow outputs */
452 		: [cpu_id]		"r" (cpu),
453 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
454 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
455 		  [v]			"Qo" (*v),
456 		  [expect]		"r" (expect),
457 		  [v2]			"Qo" (*v2),
458 		  [expect2]		"r" (expect2),
459 		  [newv]		"r" (newv)
460 		  RSEQ_INJECT_INPUT
461 		: "memory", RSEQ_ASM_TMP_REG
462 		: abort, cmpfail
463 #ifdef RSEQ_COMPARE_TWICE
464 		  , error1, error2, error3
465 #endif
466 	);
467 
468 	return 0;
469 abort:
470 	RSEQ_INJECT_FAILED
471 	return -1;
472 cmpfail:
473 	return 1;
474 #ifdef RSEQ_COMPARE_TWICE
475 error1:
476 	rseq_bug("cpu_id comparison failed");
477 error2:
478 	rseq_bug("expected value comparison failed");
479 error3:
480 	rseq_bug("2nd expected value comparison failed");
481 #endif
482 }
483 
484 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)485 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
486 				 void *dst, void *src, size_t len,
487 				 intptr_t newv, int cpu)
488 {
489 	RSEQ_INJECT_C(9)
490 
491 	__asm__ __volatile__ goto (
492 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
493 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
494 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
495 		RSEQ_INJECT_ASM(3)
496 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
497 		RSEQ_INJECT_ASM(4)
498 #ifdef RSEQ_COMPARE_TWICE
499 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
500 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
501 #endif
502 		RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
503 		RSEQ_INJECT_ASM(5)
504 		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
505 		RSEQ_INJECT_ASM(6)
506 		RSEQ_ASM_DEFINE_ABORT(4, abort)
507 		: /* gcc asm goto does not allow outputs */
508 		: [cpu_id]		"r" (cpu),
509 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
510 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
511 		  [expect]		"r" (expect),
512 		  [v]			"Qo" (*v),
513 		  [newv]		"r" (newv),
514 		  [dst]			"r" (dst),
515 		  [src]			"r" (src),
516 		  [len]			"r" (len)
517 		  RSEQ_INJECT_INPUT
518 		: "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
519 		: abort, cmpfail
520 #ifdef RSEQ_COMPARE_TWICE
521 		  , error1, error2
522 #endif
523 	);
524 
525 	return 0;
526 abort:
527 	RSEQ_INJECT_FAILED
528 	return -1;
529 cmpfail:
530 	return 1;
531 #ifdef RSEQ_COMPARE_TWICE
532 error1:
533 	rseq_bug("cpu_id comparison failed");
534 error2:
535 	rseq_bug("expected value comparison failed");
536 #endif
537 }
538 
539 static inline __attribute__((always_inline))
rseq_cmpeqv_trymemcpy_storev_release(intptr_t * v,intptr_t expect,void * dst,void * src,size_t len,intptr_t newv,int cpu)540 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
541 					 void *dst, void *src, size_t len,
542 					 intptr_t newv, int cpu)
543 {
544 	RSEQ_INJECT_C(9)
545 
546 	__asm__ __volatile__ goto (
547 		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
548 		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
549 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
550 		RSEQ_INJECT_ASM(3)
551 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
552 		RSEQ_INJECT_ASM(4)
553 #ifdef RSEQ_COMPARE_TWICE
554 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
555 		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
556 #endif
557 		RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
558 		RSEQ_INJECT_ASM(5)
559 		RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
560 		RSEQ_INJECT_ASM(6)
561 		RSEQ_ASM_DEFINE_ABORT(4, abort)
562 		: /* gcc asm goto does not allow outputs */
563 		: [cpu_id]		"r" (cpu),
564 		  [current_cpu_id]	"Qo" (__rseq_abi.cpu_id),
565 		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
566 		  [expect]		"r" (expect),
567 		  [v]			"Qo" (*v),
568 		  [newv]		"r" (newv),
569 		  [dst]			"r" (dst),
570 		  [src]			"r" (src),
571 		  [len]			"r" (len)
572 		  RSEQ_INJECT_INPUT
573 		: "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
574 		: abort, cmpfail
575 #ifdef RSEQ_COMPARE_TWICE
576 		  , error1, error2
577 #endif
578 	);
579 
580 	return 0;
581 abort:
582 	RSEQ_INJECT_FAILED
583 	return -1;
584 cmpfail:
585 	return 1;
586 #ifdef RSEQ_COMPARE_TWICE
587 error1:
588 	rseq_bug("cpu_id comparison failed");
589 error2:
590 	rseq_bug("expected value comparison failed");
591 #endif
592 }
593 
594 #endif /* !RSEQ_SKIP_FASTPATH */
595