• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
2
3@var8 = global i8 0
4@var16 = global i16 0
5@var32 = global i32 0
6@var64 = global i64 0
7
8define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
9; CHECK: test_atomic_load_add_i8:
10   %old = atomicrmw add i8* @var8, i8 %offset seq_cst
11; CHECK: dmb ish
12; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
13; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
14
15; CHECK: .LBB{{[0-9]+}}_1:
16; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
17  ; w0 below is a reasonable guess but could change: it certainly comes into the
18  ;  function there.
19; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
20; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
21; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
22; CHECK: dmb ish
23
24; CHECK: mov x0, x[[OLD]]
25   ret i8 %old
26}
27
28define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
29; CHECK: test_atomic_load_add_i16:
30   %old = atomicrmw add i16* @var16, i16 %offset seq_cst
31; CHECK: dmb ish
32; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
33; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
34
35; CHECK: .LBB{{[0-9]+}}_1:
36; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
37  ; w0 below is a reasonable guess but could change: it certainly comes into the
38  ;  function there.
39; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
40; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
41; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
42; CHECK: dmb ish
43
44; CHECK: mov x0, x[[OLD]]
45   ret i16 %old
46}
47
48define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
49; CHECK: test_atomic_load_add_i32:
50   %old = atomicrmw add i32* @var32, i32 %offset seq_cst
51; CHECK: dmb ish
52; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
53; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
54
55; CHECK: .LBB{{[0-9]+}}_1:
56; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
57  ; w0 below is a reasonable guess but could change: it certainly comes into the
58  ;  function there.
59; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
60; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
61; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
62; CHECK: dmb ish
63
64; CHECK: mov x0, x[[OLD]]
65   ret i32 %old
66}
67
68define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
69; CHECK: test_atomic_load_add_i64:
70   %old = atomicrmw add i64* @var64, i64 %offset seq_cst
71; CHECK: dmb ish
72; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
73; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
74
75; CHECK: .LBB{{[0-9]+}}_1:
76; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
77  ; x0 below is a reasonable guess but could change: it certainly comes into the
78  ; function there.
79; CHECK-NEXT: add [[NEW:x[0-9]+]], x[[OLD]], x0
80; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
81; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
82; CHECK: dmb ish
83
84; CHECK: mov x0, x[[OLD]]
85   ret i64 %old
86}
87
88define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
89; CHECK: test_atomic_load_sub_i8:
90   %old = atomicrmw sub i8* @var8, i8 %offset seq_cst
91; CHECK: dmb ish
92; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
93; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
94
95; CHECK: .LBB{{[0-9]+}}_1:
96; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
97  ; w0 below is a reasonable guess but could change: it certainly comes into the
98  ;  function there.
99; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
100; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
101; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
102; CHECK: dmb ish
103
104; CHECK: mov x0, x[[OLD]]
105   ret i8 %old
106}
107
108define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
109; CHECK: test_atomic_load_sub_i16:
110   %old = atomicrmw sub i16* @var16, i16 %offset seq_cst
111; CHECK: dmb ish
112; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
113; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
114
115; CHECK: .LBB{{[0-9]+}}_1:
116; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
117  ; w0 below is a reasonable guess but could change: it certainly comes into the
118  ;  function there.
119; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
120; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
121; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
122; CHECK: dmb ish
123
124; CHECK: mov x0, x[[OLD]]
125   ret i16 %old
126}
127
128define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
129; CHECK: test_atomic_load_sub_i32:
130   %old = atomicrmw sub i32* @var32, i32 %offset seq_cst
131; CHECK: dmb ish
132; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
133; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
134
135; CHECK: .LBB{{[0-9]+}}_1:
136; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
137  ; w0 below is a reasonable guess but could change: it certainly comes into the
138  ;  function there.
139; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
140; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
141; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
142; CHECK: dmb ish
143
144; CHECK: mov x0, x[[OLD]]
145   ret i32 %old
146}
147
148define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
149; CHECK: test_atomic_load_sub_i64:
150   %old = atomicrmw sub i64* @var64, i64 %offset seq_cst
151; CHECK: dmb ish
152; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
153; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
154
155; CHECK: .LBB{{[0-9]+}}_1:
156; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
157  ; x0 below is a reasonable guess but could change: it certainly comes into the
158  ; function there.
159; CHECK-NEXT: sub [[NEW:x[0-9]+]], x[[OLD]], x0
160; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
161; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
162; CHECK: dmb ish
163
164; CHECK: mov x0, x[[OLD]]
165   ret i64 %old
166}
167
168define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
169; CHECK: test_atomic_load_and_i8:
170   %old = atomicrmw and i8* @var8, i8 %offset seq_cst
171; CHECK: dmb ish
172; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
173; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
174
175; CHECK: .LBB{{[0-9]+}}_1:
176; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
177  ; w0 below is a reasonable guess but could change: it certainly comes into the
178  ;  function there.
179; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
180; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
181; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
182; CHECK: dmb ish
183
184; CHECK: mov x0, x[[OLD]]
185   ret i8 %old
186}
187
188define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
189; CHECK: test_atomic_load_and_i16:
190   %old = atomicrmw and i16* @var16, i16 %offset seq_cst
191; CHECK: dmb ish
192; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
193; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
194
195; CHECK: .LBB{{[0-9]+}}_1:
196; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
197  ; w0 below is a reasonable guess but could change: it certainly comes into the
198  ;  function there.
199; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
200; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
201; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
202; CHECK: dmb ish
203
204; CHECK: mov x0, x[[OLD]]
205   ret i16 %old
206}
207
208define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
209; CHECK: test_atomic_load_and_i32:
210   %old = atomicrmw and i32* @var32, i32 %offset seq_cst
211; CHECK: dmb ish
212; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
213; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
214
215; CHECK: .LBB{{[0-9]+}}_1:
216; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
217  ; w0 below is a reasonable guess but could change: it certainly comes into the
218  ;  function there.
219; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
220; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
221; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
222; CHECK: dmb ish
223
224; CHECK: mov x0, x[[OLD]]
225   ret i32 %old
226}
227
228define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
229; CHECK: test_atomic_load_and_i64:
230   %old = atomicrmw and i64* @var64, i64 %offset seq_cst
231; CHECK: dmb ish
232; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
233; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
234
235; CHECK: .LBB{{[0-9]+}}_1:
236; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
237  ; x0 below is a reasonable guess but could change: it certainly comes into the
238  ; function there.
239; CHECK-NEXT: and [[NEW:x[0-9]+]], x[[OLD]], x0
240; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
241; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
242; CHECK: dmb ish
243
244; CHECK: mov x0, x[[OLD]]
245   ret i64 %old
246}
247
248define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
249; CHECK: test_atomic_load_or_i8:
250   %old = atomicrmw or i8* @var8, i8 %offset seq_cst
251; CHECK: dmb ish
252; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
253; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
254
255; CHECK: .LBB{{[0-9]+}}_1:
256; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
257  ; w0 below is a reasonable guess but could change: it certainly comes into the
258  ;  function there.
259; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
260; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
261; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
262; CHECK: dmb ish
263
264; CHECK: mov x0, x[[OLD]]
265   ret i8 %old
266}
267
268define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
269; CHECK: test_atomic_load_or_i16:
270   %old = atomicrmw or i16* @var16, i16 %offset seq_cst
271; CHECK: dmb ish
272; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
273; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
274
275; CHECK: .LBB{{[0-9]+}}_1:
276; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
277  ; w0 below is a reasonable guess but could change: it certainly comes into the
278  ;  function there.
279; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
280; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
281; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
282; CHECK: dmb ish
283
284; CHECK: mov x0, x[[OLD]]
285   ret i16 %old
286}
287
288define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
289; CHECK: test_atomic_load_or_i32:
290   %old = atomicrmw or i32* @var32, i32 %offset seq_cst
291; CHECK: dmb ish
292; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
293; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
294
295; CHECK: .LBB{{[0-9]+}}_1:
296; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
297  ; w0 below is a reasonable guess but could change: it certainly comes into the
298  ;  function there.
299; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
300; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
301; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
302; CHECK: dmb ish
303
304; CHECK: mov x0, x[[OLD]]
305   ret i32 %old
306}
307
308define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
309; CHECK: test_atomic_load_or_i64:
310   %old = atomicrmw or i64* @var64, i64 %offset seq_cst
311; CHECK: dmb ish
312; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
313; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
314
315; CHECK: .LBB{{[0-9]+}}_1:
316; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
317  ; x0 below is a reasonable guess but could change: it certainly comes into the
318  ; function there.
319; CHECK-NEXT: orr [[NEW:x[0-9]+]], x[[OLD]], x0
320; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
321; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
322; CHECK: dmb ish
323
324; CHECK: mov x0, x[[OLD]]
325   ret i64 %old
326}
327
328define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
329; CHECK: test_atomic_load_xor_i8:
330   %old = atomicrmw xor i8* @var8, i8 %offset seq_cst
331; CHECK: dmb ish
332; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
333; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
334
335; CHECK: .LBB{{[0-9]+}}_1:
336; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
337  ; w0 below is a reasonable guess but could change: it certainly comes into the
338  ;  function there.
339; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
340; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
341; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
342; CHECK: dmb ish
343
344; CHECK: mov x0, x[[OLD]]
345   ret i8 %old
346}
347
348define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
349; CHECK: test_atomic_load_xor_i16:
350   %old = atomicrmw xor i16* @var16, i16 %offset seq_cst
351; CHECK: dmb ish
352; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
353; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
354
355; CHECK: .LBB{{[0-9]+}}_1:
356; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
357  ; w0 below is a reasonable guess but could change: it certainly comes into the
358  ;  function there.
359; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
360; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
361; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
362; CHECK: dmb ish
363
364; CHECK: mov x0, x[[OLD]]
365   ret i16 %old
366}
367
368define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
369; CHECK: test_atomic_load_xor_i32:
370   %old = atomicrmw xor i32* @var32, i32 %offset seq_cst
371; CHECK: dmb ish
372; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
373; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
374
375; CHECK: .LBB{{[0-9]+}}_1:
376; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
377  ; w0 below is a reasonable guess but could change: it certainly comes into the
378  ;  function there.
379; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
380; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
381; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
382; CHECK: dmb ish
383
384; CHECK: mov x0, x[[OLD]]
385   ret i32 %old
386}
387
388define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
389; CHECK: test_atomic_load_xor_i64:
390   %old = atomicrmw xor i64* @var64, i64 %offset seq_cst
391; CHECK: dmb ish
392; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
393; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
394
395; CHECK: .LBB{{[0-9]+}}_1:
396; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
397  ; x0 below is a reasonable guess but could change: it certainly comes into the
398  ; function there.
399; CHECK-NEXT: eor [[NEW:x[0-9]+]], x[[OLD]], x0
400; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
401; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
402; CHECK: dmb ish
403
404; CHECK: mov x0, x[[OLD]]
405   ret i64 %old
406}
407
408define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
409; CHECK: test_atomic_load_xchg_i8:
410   %old = atomicrmw xchg i8* @var8, i8 %offset seq_cst
411; CHECK: dmb ish
412; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
413; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
414
415; CHECK: .LBB{{[0-9]+}}_1:
416; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
417  ; w0 below is a reasonable guess but could change: it certainly comes into the
418  ; function there.
419; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
420; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
421; CHECK: dmb ish
422
423; CHECK: mov x0, x[[OLD]]
424   ret i8 %old
425}
426
427define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
428; CHECK: test_atomic_load_xchg_i16:
429   %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst
430; CHECK: dmb ish
431; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
432; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
433
434; CHECK: .LBB{{[0-9]+}}_1:
435; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
436  ; w0 below is a reasonable guess but could change: it certainly comes into the
437  ; function there.
438; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
439; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
440; CHECK: dmb ish
441
442; CHECK: mov x0, x[[OLD]]
443   ret i16 %old
444}
445
446define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
447; CHECK: test_atomic_load_xchg_i32:
448   %old = atomicrmw xchg i32* @var32, i32 %offset seq_cst
449; CHECK: dmb ish
450; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
451; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
452
453; CHECK: .LBB{{[0-9]+}}_1:
454; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
455  ; w0 below is a reasonable guess but could change: it certainly comes into the
456  ;  function there.
457; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
458; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
459; CHECK: dmb ish
460
461; CHECK: mov x0, x[[OLD]]
462   ret i32 %old
463}
464
465define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
466; CHECK: test_atomic_load_xchg_i64:
467   %old = atomicrmw xchg i64* @var64, i64 %offset seq_cst
468; CHECK: dmb ish
469; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
470; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
471
472; CHECK: .LBB{{[0-9]+}}_1:
473; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
474  ; x0 below is a reasonable guess but could change: it certainly comes into the
475  ; function there.
476; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], x0, [x[[ADDR]]]
477; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
478; CHECK: dmb ish
479
480; CHECK: mov x0, x[[OLD]]
481   ret i64 %old
482}
483
484
485define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
486; CHECK: test_atomic_load_min_i8:
487   %old = atomicrmw min i8* @var8, i8 %offset seq_cst
488; CHECK: dmb ish
489; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
490; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
491
492; CHECK: .LBB{{[0-9]+}}_1:
493; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
494  ; w0 below is a reasonable guess but could change: it certainly comes into the
495  ;  function there.
496; CHECK-NEXT: cmp w0, w[[OLD]], sxtb
497; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
498; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
499; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
500; CHECK: dmb ish
501
502; CHECK: mov x0, x[[OLD]]
503   ret i8 %old
504}
505
506define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
507; CHECK: test_atomic_load_min_i16:
508   %old = atomicrmw min i16* @var16, i16 %offset seq_cst
509; CHECK: dmb ish
510; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
511; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
512
513; CHECK: .LBB{{[0-9]+}}_1:
514; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
515  ; w0 below is a reasonable guess but could change: it certainly comes into the
516  ;  function there.
517; CHECK-NEXT: cmp w0, w[[OLD]], sxth
518; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
519; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
520; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
521; CHECK: dmb ish
522
523; CHECK: mov x0, x[[OLD]]
524   ret i16 %old
525}
526
527define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
528; CHECK: test_atomic_load_min_i32:
529   %old = atomicrmw min i32* @var32, i32 %offset seq_cst
530; CHECK: dmb ish
531; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
532; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
533
534; CHECK: .LBB{{[0-9]+}}_1:
535; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
536  ; w0 below is a reasonable guess but could change: it certainly comes into the
537  ;  function there.
538; CHECK-NEXT: cmp w0, w[[OLD]]
539; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
540; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
541; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
542; CHECK: dmb ish
543
544; CHECK: mov x0, x[[OLD]]
545   ret i32 %old
546}
547
548define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
549; CHECK: test_atomic_load_min_i64:
550   %old = atomicrmw min i64* @var64, i64 %offset seq_cst
551; CHECK: dmb ish
552; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
553; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
554
555; CHECK: .LBB{{[0-9]+}}_1:
556; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
557  ; x0 below is a reasonable guess but could change: it certainly comes into the
558  ; function there.
559; CHECK-NEXT: cmp x0, x[[OLD]]
560; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, gt
561; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
562; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
563; CHECK: dmb ish
564
565; CHECK: mov x0, x[[OLD]]
566   ret i64 %old
567}
568
569define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
570; CHECK: test_atomic_load_max_i8:
571   %old = atomicrmw max i8* @var8, i8 %offset seq_cst
572; CHECK: dmb ish
573; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
574; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
575
576; CHECK: .LBB{{[0-9]+}}_1:
577; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
578  ; w0 below is a reasonable guess but could change: it certainly comes into the
579  ;  function there.
580; CHECK-NEXT: cmp w0, w[[OLD]], sxtb
581; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
582; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
583; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
584; CHECK: dmb ish
585
586; CHECK: mov x0, x[[OLD]]
587   ret i8 %old
588}
589
590define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
591; CHECK: test_atomic_load_max_i16:
592   %old = atomicrmw max i16* @var16, i16 %offset seq_cst
593; CHECK: dmb ish
594; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
595; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
596
597; CHECK: .LBB{{[0-9]+}}_1:
598; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
599  ; w0 below is a reasonable guess but could change: it certainly comes into the
600  ;  function there.
601; CHECK-NEXT: cmp w0, w[[OLD]], sxth
602; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
603; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
604; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
605; CHECK: dmb ish
606
607; CHECK: mov x0, x[[OLD]]
608   ret i16 %old
609}
610
611define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
612; CHECK: test_atomic_load_max_i32:
613   %old = atomicrmw max i32* @var32, i32 %offset seq_cst
614; CHECK: dmb ish
615; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
616; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
617
618; CHECK: .LBB{{[0-9]+}}_1:
619; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
620  ; w0 below is a reasonable guess but could change: it certainly comes into the
621  ;  function there.
622; CHECK-NEXT: cmp w0, w[[OLD]]
623; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lt
624; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
625; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
626; CHECK: dmb ish
627
628; CHECK: mov x0, x[[OLD]]
629   ret i32 %old
630}
631
632define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
633; CHECK: test_atomic_load_max_i64:
634   %old = atomicrmw max i64* @var64, i64 %offset seq_cst
635; CHECK: dmb ish
636; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
637; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
638
639; CHECK: .LBB{{[0-9]+}}_1:
640; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
641  ; x0 below is a reasonable guess but could change: it certainly comes into the
642  ; function there.
643; CHECK-NEXT: cmp x0, x[[OLD]]
644; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, lt
645; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
646; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
647; CHECK: dmb ish
648
649; CHECK: mov x0, x[[OLD]]
650   ret i64 %old
651}
652
653define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
654; CHECK: test_atomic_load_umin_i8:
655   %old = atomicrmw umin i8* @var8, i8 %offset seq_cst
656; CHECK: dmb ish
657; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
658; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
659
660; CHECK: .LBB{{[0-9]+}}_1:
661; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
662  ; w0 below is a reasonable guess but could change: it certainly comes into the
663  ;  function there.
664; CHECK-NEXT: cmp w0, w[[OLD]], uxtb
665; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
666; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
667; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
668; CHECK: dmb ish
669
670; CHECK: mov x0, x[[OLD]]
671   ret i8 %old
672}
673
674define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
675; CHECK: test_atomic_load_umin_i16:
676   %old = atomicrmw umin i16* @var16, i16 %offset seq_cst
677; CHECK: dmb ish
678; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
679; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
680
681; CHECK: .LBB{{[0-9]+}}_1:
682; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
683  ; w0 below is a reasonable guess but could change: it certainly comes into the
684  ;  function there.
685; CHECK-NEXT: cmp w0, w[[OLD]], uxth
686; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
687; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
688; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
689; CHECK: dmb ish
690
691; CHECK: mov x0, x[[OLD]]
692   ret i16 %old
693}
694
695define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
696; CHECK: test_atomic_load_umin_i32:
697   %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
698; CHECK: dmb ish
699; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
700; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
701
702; CHECK: .LBB{{[0-9]+}}_1:
703; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
704  ; w0 below is a reasonable guess but could change: it certainly comes into the
705  ;  function there.
706; CHECK-NEXT: cmp w0, w[[OLD]]
707; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
708; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
709; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
710; CHECK: dmb ish
711
712; CHECK: mov x0, x[[OLD]]
713   ret i32 %old
714}
715
716define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
717; CHECK: test_atomic_load_umin_i64:
718   %old = atomicrmw umin i64* @var64, i64 %offset seq_cst
719; CHECK: dmb ish
720; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
721; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
722
723; CHECK: .LBB{{[0-9]+}}_1:
724; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
725  ; x0 below is a reasonable guess but could change: it certainly comes into the
726  ; function there.
727; CHECK-NEXT: cmp x0, x[[OLD]]
728; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, hi
729; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
730; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
731; CHECK: dmb ish
732
733; CHECK: mov x0, x[[OLD]]
734   ret i64 %old
735}
736
737define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
738; CHECK: test_atomic_load_umax_i8:
739   %old = atomicrmw umax i8* @var8, i8 %offset seq_cst
740; CHECK: dmb ish
741; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
742; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
743
744; CHECK: .LBB{{[0-9]+}}_1:
745; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
746  ; w0 below is a reasonable guess but could change: it certainly comes into the
747  ;  function there.
748; CHECK-NEXT: cmp w0, w[[OLD]], uxtb
749; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
750; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
751; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
752; CHECK: dmb ish
753
754; CHECK: mov x0, x[[OLD]]
755   ret i8 %old
756}
757
758define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
759; CHECK: test_atomic_load_umax_i16:
760   %old = atomicrmw umax i16* @var16, i16 %offset seq_cst
761; CHECK: dmb ish
762; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
763; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
764
765; CHECK: .LBB{{[0-9]+}}_1:
766; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
767  ; w0 below is a reasonable guess but could change: it certainly comes into the
768  ;  function there.
769; CHECK-NEXT: cmp w0, w[[OLD]], uxth
770; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
771; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
772; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
773; CHECK: dmb ish
774
775; CHECK: mov x0, x[[OLD]]
776   ret i16 %old
777}
778
779define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
780; CHECK: test_atomic_load_umax_i32:
781   %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
782; CHECK: dmb ish
783; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
784; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
785
786; CHECK: .LBB{{[0-9]+}}_1:
787; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
788  ; w0 below is a reasonable guess but could change: it certainly comes into the
789  ;  function there.
790; CHECK-NEXT: cmp w0, w[[OLD]]
791; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, lo
792; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
793; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
794; CHECK: dmb ish
795
796; CHECK: mov x0, x[[OLD]]
797   ret i32 %old
798}
799
800define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
801; CHECK: test_atomic_load_umax_i64:
802   %old = atomicrmw umax i64* @var64, i64 %offset seq_cst
803; CHECK: dmb ish
804; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
805; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
806
807; CHECK: .LBB{{[0-9]+}}_1:
808; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
809  ; x0 below is a reasonable guess but could change: it certainly comes into the
810  ; function there.
811; CHECK-NEXT: cmp x0, x[[OLD]]
812; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, lo
813; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
814; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
815; CHECK: dmb ish
816
817; CHECK: mov x0, x[[OLD]]
818   ret i64 %old
819}
820
821define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
822; CHECK: test_atomic_cmpxchg_i8:
823   %old = cmpxchg i8* @var8, i8 %wanted, i8 %new seq_cst
824; CHECK: dmb ish
825; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
826; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
827
828; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
829; CHECK-NEXT: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
830  ; w0 below is a reasonable guess but could change: it certainly comes into the
831  ;  function there.
832; CHECK-NEXT: cmp w[[OLD]], w0
833; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
834  ; As above, w1 is a reasonable guess.
835; CHECK: stxrb [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
836; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
837; CHECK: dmb ish
838
839; CHECK: mov x0, x[[OLD]]
840   ret i8 %old
841}
842
843define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
844; CHECK: test_atomic_cmpxchg_i16:
845   %old = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst
846; CHECK: dmb ish
847; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
848; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var16
849
850; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
851; CHECK-NEXT: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
852  ; w0 below is a reasonable guess but could change: it certainly comes into the
853  ;  function there.
854; CHECK-NEXT: cmp w[[OLD]], w0
855; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
856  ; As above, w1 is a reasonable guess.
857; CHECK: stxrh [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
858; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
859; CHECK: dmb ish
860
861; CHECK: mov x0, x[[OLD]]
862   ret i16 %old
863}
864
865define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
866; CHECK: test_atomic_cmpxchg_i32:
867   %old = cmpxchg i32* @var32, i32 %wanted, i32 %new seq_cst
868; CHECK: dmb ish
869; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
870; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var32
871
872; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
873; CHECK-NEXT: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
874  ; w0 below is a reasonable guess but could change: it certainly comes into the
875  ;  function there.
876; CHECK-NEXT: cmp w[[OLD]], w0
877; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
878  ; As above, w1 is a reasonable guess.
879; CHECK: stxr [[STATUS:w[0-9]+]], w1, [x[[ADDR]]]
880; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
881; CHECK: dmb ish
882
883; CHECK: mov x0, x[[OLD]]
884   ret i32 %old
885}
886
887define i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
888; CHECK: test_atomic_cmpxchg_i64:
889   %old = cmpxchg i64* @var64, i64 %wanted, i64 %new seq_cst
890; CHECK: dmb ish
891; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
892; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var64
893
894; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
895; CHECK-NEXT: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
896  ; w0 below is a reasonable guess but could change: it certainly comes into the
897  ;  function there.
898; CHECK-NEXT: cmp x[[OLD]], x0
899; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
900  ; As above, w1 is a reasonable guess.
901; CHECK: stxr [[STATUS:w[0-9]+]], x1, [x[[ADDR]]]
902; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
903; CHECK: dmb ish
904
905; CHECK: mov x0, x[[OLD]]
906   ret i64 %old
907}
908
909define i8 @test_atomic_load_monotonic_i8() nounwind {
910; CHECK: test_atomic_load_monotonic_i8:
911  %val = load atomic i8* @var8 monotonic, align 1
912; CHECK-NOT: dmb
913; CHECK: adrp x[[HIADDR:[0-9]+]], var8
914; CHECK: ldrb w0, [x[[HIADDR]], #:lo12:var8]
915; CHECK-NOT: dmb
916
917  ret i8 %val
918}
919
920define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
921; CHECK: test_atomic_load_monotonic_regoff_i8:
922  %addr_int = add i64 %base, %off
923  %addr = inttoptr i64 %addr_int to i8*
924
925  %val = load atomic i8* %addr monotonic, align 1
926; CHECK-NOT: dmb
927; CHECK: ldrb w0, [x0, x1]
928; CHECK-NOT: dmb
929
930  ret i8 %val
931}
932
933define i8 @test_atomic_load_acquire_i8() nounwind {
934; CHECK: test_atomic_load_acquire_i8:
935  %val = load atomic i8* @var8 acquire, align 1
936; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
937; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], #:lo12:var8
938
939; CHECK: ldarb w0, [x[[ADDR]]]
940  ret i8 %val
941}
942
943define i8 @test_atomic_load_seq_cst_i8() nounwind {
944; CHECK: test_atomic_load_seq_cst_i8:
945  %val = load atomic i8* @var8 seq_cst, align 1
946; CHECK: adrp x[[HIADDR:[0-9]+]], var8
947; CHECK: ldrb w0, [x[[HIADDR]], #:lo12:var8]
948; CHECK: dmb ish
949  ret i8 %val
950}
951
952define i16 @test_atomic_load_monotonic_i16() nounwind {
953; CHECK: test_atomic_load_monotonic_i16:
954  %val = load atomic i16* @var16 monotonic, align 2
955; CHECK-NOT: dmb
956; CHECK: adrp x[[HIADDR:[0-9]+]], var16
957; CHECK: ldrh w0, [x[[HIADDR]], #:lo12:var16]
958; CHECK-NOT: dmb
959
960  ret i16 %val
961}
962
963define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
964; CHECK: test_atomic_load_monotonic_regoff_i32:
965  %addr_int = add i64 %base, %off
966  %addr = inttoptr i64 %addr_int to i32*
967
968  %val = load atomic i32* %addr monotonic, align 4
969; CHECK-NOT: dmb
970; CHECK: ldr w0, [x0, x1]
971; CHECK-NOT: dmb
972
973  ret i32 %val
974}
975
976define i64 @test_atomic_load_seq_cst_i64() nounwind {
977; CHECK: test_atomic_load_seq_cst_i64:
978  %val = load atomic i64* @var64 seq_cst, align 8
979; CHECK: adrp x[[HIADDR:[0-9]+]], var64
980; CHECK: ldr x0, [x[[HIADDR]], #:lo12:var64]
981; CHECK: dmb ish
982  ret i64 %val
983}
984
985define void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
986; CHECK: test_atomic_store_monotonic_i8:
987  store atomic i8 %val, i8* @var8 monotonic, align 1
988; CHECK: adrp x[[HIADDR:[0-9]+]], var8
989; CHECK: strb w0, [x[[HIADDR]], #:lo12:var8]
990
991  ret void
992}
993
994define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
995; CHECK: test_atomic_store_monotonic_regoff_i8:
996
997  %addr_int = add i64 %base, %off
998  %addr = inttoptr i64 %addr_int to i8*
999
1000  store atomic i8 %val, i8* %addr monotonic, align 1
1001; CHECK: strb w2, [x0, x1]
1002
1003  ret void
1004}
1005define void @test_atomic_store_release_i8(i8 %val) nounwind {
1006; CHECK: test_atomic_store_release_i8:
1007  store atomic i8 %val, i8* @var8 release, align 1
1008; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1009; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var8
1010; CHECK: stlrb w0, [x[[ADDR]]]
1011
1012  ret void
1013}
1014
1015define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
1016; CHECK: test_atomic_store_seq_cst_i8:
1017  store atomic i8 %val, i8* @var8 seq_cst, align 1
1018; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1019; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var8
1020; CHECK: stlrb w0, [x[[ADDR]]]
1021; CHECK: dmb ish
1022
1023  ret void
1024}
1025
1026define void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
1027; CHECK: test_atomic_store_monotonic_i16:
1028  store atomic i16 %val, i16* @var16 monotonic, align 2
1029; CHECK: adrp x[[HIADDR:[0-9]+]], var16
1030; CHECK: strh w0, [x[[HIADDR]], #:lo12:var16]
1031
1032  ret void
1033}
1034
1035define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
1036; CHECK: test_atomic_store_monotonic_regoff_i32:
1037
1038  %addr_int = add i64 %base, %off
1039  %addr = inttoptr i64 %addr_int to i32*
1040
1041  store atomic i32 %val, i32* %addr monotonic, align 4
1042; CHECK: str w2, [x0, x1]
1043
1044  ret void
1045}
1046
1047define void @test_atomic_store_release_i64(i64 %val) nounwind {
1048; CHECK: test_atomic_store_release_i64:
1049  store atomic i64 %val, i64* @var64 release, align 8
1050; CHECK: adrp [[HIADDR:x[0-9]+]], var64
1051; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], #:lo12:var64
1052; CHECK: stlr x0, [x[[ADDR]]]
1053
1054  ret void
1055}
1056