• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs < %s | FileCheck %s
2; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-REG
3; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+outline-atomics < %s | FileCheck %s --check-prefix=OUTLINE_ATOMICS
4
5
6; Point of CHECK-REG is to make sure UNPREDICTABLE instructions aren't created
7; (i.e. reusing a register for status & data in store exclusive).
8; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], w[[NEW]], [x{{[0-9]+}}]
9; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], x[[NEW]], [x{{[0-9]+}}]
10
11@var8 = global i8 0
12@var16 = global i16 0
13@var32 = global i32 0
14@var64 = global i64 0
15
16define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
17; CHECK-LABEL: test_atomic_load_add_i8:
18; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i8:
19; OUTLINE_ATOMICS:       // %bb.0:
20; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
21; OUTLINE_ATOMICS-NEXT:    adrp x1, var8
22; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var8
23; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd1_acq_rel
24; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
25; OUTLINE_ATOMICS-NEXT:    ret
26   %old = atomicrmw add i8* @var8, i8 %offset seq_cst
27; CHECK-NOT: dmb
28; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
29; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
30
31; CHECK: .LBB{{[0-9]+}}_1:
32; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
33  ; w0 below is a reasonable guess but could change: it certainly comes into the
34  ;  function there.
35; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
36; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
37; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
38; CHECK-NOT: dmb
39
40; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
41   ret i8 %old
42}
43
44define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
45; CHECK-LABEL: test_atomic_load_add_i16:
46; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i16:
47; OUTLINE_ATOMICS:       // %bb.0:
48; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
49; OUTLINE_ATOMICS-NEXT:    adrp x1, var16
50; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var16
51; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd2_acq
52; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
53; OUTLINE_ATOMICS-NEXT:    ret
54   %old = atomicrmw add i16* @var16, i16 %offset acquire
55; CHECK-NOT: dmb
56; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
57; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
58
59; CHECK: .LBB{{[0-9]+}}_1:
60; ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
61  ; w0 below is a reasonable guess but could change: it certainly comes into the
62  ;  function there.
63; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
64; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
65; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
66; CHECK-NOT: dmb
67
68; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
69   ret i16 %old
70}
71
72define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
73; CHECK-LABEL: test_atomic_load_add_i32:
74; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i32:
75; OUTLINE_ATOMICS:       // %bb.0:
76; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
77; OUTLINE_ATOMICS-NEXT:    adrp x1, var32
78; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var32
79; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd4_rel
80; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
81; OUTLINE_ATOMICS-NEXT:    ret
82   %old = atomicrmw add i32* @var32, i32 %offset release
83; CHECK-NOT: dmb
84; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
85; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
86
87; CHECK: .LBB{{[0-9]+}}_1:
88; ; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
89  ; w0 below is a reasonable guess but could change: it certainly comes into the
90  ;  function there.
91; CHECK-NEXT: add [[NEW:w[0-9]+]], w[[OLD]], w0
92; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
93; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
94; CHECK-NOT: dmb
95
96; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
97   ret i32 %old
98}
99
100define i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
101; CHECK-LABEL: test_atomic_load_add_i64:
102; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i64:
103; OUTLINE_ATOMICS:       // %bb.0:
104; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
105; OUTLINE_ATOMICS-NEXT:    adrp x1, var64
106; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var64
107; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd8_relax
108; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
109; OUTLINE_ATOMICS-NEXT:    ret
110   %old = atomicrmw add i64* @var64, i64 %offset monotonic
111; CHECK-NOT: dmb
112; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
113; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
114
115; CHECK: .LBB{{[0-9]+}}_1:
116; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
117  ; x0 below is a reasonable guess but could change: it certainly comes into the
118  ; function there.
119; CHECK-NEXT: add [[NEW:x[0-9]+]], x[[OLD]], x0
120; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
121; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
122; CHECK-NOT: dmb
123
124; CHECK: mov x0, x[[OLD]]
125   ret i64 %old
126}
127
128define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
129; CHECK-LABEL: test_atomic_load_sub_i8:
130; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i8:
131; OUTLINE_ATOMICS:       // %bb.0:
132; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
133; OUTLINE_ATOMICS-NEXT:    neg w0, w0
134; OUTLINE_ATOMICS-NEXT:    adrp x1, var8
135; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var8
136; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd1_relax
137; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
138; OUTLINE_ATOMICS-NEXT:    ret
139   %old = atomicrmw sub i8* @var8, i8 %offset monotonic
140; CHECK-NOT: dmb
141; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
142; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
143
144; CHECK: .LBB{{[0-9]+}}_1:
145; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
146  ; w0 below is a reasonable guess but could change: it certainly comes into the
147  ;  function there.
148; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
149; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
150; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
151; CHECK-NOT: dmb
152
153; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
154   ret i8 %old
155}
156
157define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
158; CHECK-LABEL: test_atomic_load_sub_i16:
159; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i16:
160; OUTLINE_ATOMICS:       // %bb.0:
161; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
162; OUTLINE_ATOMICS-NEXT:    neg w0, w0
163; OUTLINE_ATOMICS-NEXT:    adrp x1, var16
164; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var16
165; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd2_rel
166; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
167; OUTLINE_ATOMICS-NEXT:    ret
168   %old = atomicrmw sub i16* @var16, i16 %offset release
169; CHECK-NOT: dmb
170; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
171; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
172
173; CHECK: .LBB{{[0-9]+}}_1:
174; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
175  ; w0 below is a reasonable guess but could change: it certainly comes into the
176  ;  function there.
177; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
178; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
179; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
180; CHECK-NOT: dmb
181
182; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
183   ret i16 %old
184}
185
186define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
187; CHECK-LABEL: test_atomic_load_sub_i32:
188; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i32:
189; OUTLINE_ATOMICS:       // %bb.0:
190; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
191; OUTLINE_ATOMICS-NEXT:    neg w0, w0
192; OUTLINE_ATOMICS-NEXT:    adrp x1, var32
193; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var32
194; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd4_acq
195; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
196; OUTLINE_ATOMICS-NEXT:    ret
197   %old = atomicrmw sub i32* @var32, i32 %offset acquire
198; CHECK-NOT: dmb
199; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
200; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
201
202; CHECK: .LBB{{[0-9]+}}_1:
203; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
204  ; w0 below is a reasonable guess but could change: it certainly comes into the
205  ;  function there.
206; CHECK-NEXT: sub [[NEW:w[0-9]+]], w[[OLD]], w0
207; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
208; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
209; CHECK-NOT: dmb
210
211; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
212   ret i32 %old
213}
214
215define i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
216; CHECK-LABEL: test_atomic_load_sub_i64:
217; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i64:
218; OUTLINE_ATOMICS:       // %bb.0:
219; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
220; OUTLINE_ATOMICS-NEXT:    neg x0, x0
221; OUTLINE_ATOMICS-NEXT:    adrp x1, var64
222; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var64
223; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd8_acq_rel
224; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
225; OUTLINE_ATOMICS-NEXT:    ret
226   %old = atomicrmw sub i64* @var64, i64 %offset seq_cst
227; CHECK-NOT: dmb
228; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
229; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
230
231; CHECK: .LBB{{[0-9]+}}_1:
232; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
233  ; x0 below is a reasonable guess but could change: it certainly comes into the
234  ; function there.
235; CHECK-NEXT: sub [[NEW:x[0-9]+]], x[[OLD]], x0
236; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
237; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
238; CHECK-NOT: dmb
239
240; CHECK: mov x0, x[[OLD]]
241   ret i64 %old
242}
243
244define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
245; CHECK-LABEL: test_atomic_load_and_i8:
246; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i8:
247; OUTLINE_ATOMICS:       // %bb.0:
248; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
249; OUTLINE_ATOMICS-NEXT:    mvn w0, w0
250; OUTLINE_ATOMICS-NEXT:    adrp x1, var8
251; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var8
252; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldclr1_rel
253; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
254; OUTLINE_ATOMICS-NEXT:    ret
255   %old = atomicrmw and i8* @var8, i8 %offset release
256; CHECK-NOT: dmb
257; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
258; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
259
260; CHECK: .LBB{{[0-9]+}}_1:
261; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
262  ; w0 below is a reasonable guess but could change: it certainly comes into the
263  ;  function there.
264; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
265; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
266; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
267; CHECK-NOT: dmb
268
269; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
270   ret i8 %old
271}
272
273define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
274; CHECK-LABEL: test_atomic_load_and_i16:
275; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i16:
276; OUTLINE_ATOMICS:       // %bb.0:
277; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
278; OUTLINE_ATOMICS-NEXT:    mvn w0, w0
279; OUTLINE_ATOMICS-NEXT:    adrp x1, var16
280; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var16
281; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldclr2_relax
282; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
283; OUTLINE_ATOMICS-NEXT:    ret
284   %old = atomicrmw and i16* @var16, i16 %offset monotonic
285; CHECK-NOT: dmb
286; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
287; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
288
289; CHECK: .LBB{{[0-9]+}}_1:
290; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
291  ; w0 below is a reasonable guess but could change: it certainly comes into the
292  ;  function there.
293; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
294; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
295; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
296; CHECK-NOT: dmb
297
298; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
299   ret i16 %old
300}
301
302define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
303; CHECK-LABEL: test_atomic_load_and_i32:
304; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i32:
305; OUTLINE_ATOMICS:       // %bb.0:
306; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
307; OUTLINE_ATOMICS-NEXT:    mvn w0, w0
308; OUTLINE_ATOMICS-NEXT:    adrp x1, var32
309; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var32
310; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldclr4_acq_rel
311; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
312; OUTLINE_ATOMICS-NEXT:    ret
313   %old = atomicrmw and i32* @var32, i32 %offset seq_cst
314; CHECK-NOT: dmb
315; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
316; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
317
318; CHECK: .LBB{{[0-9]+}}_1:
319; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
320  ; w0 below is a reasonable guess but could change: it certainly comes into the
321  ;  function there.
322; CHECK-NEXT: and [[NEW:w[0-9]+]], w[[OLD]], w0
323; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
324; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
325; CHECK-NOT: dmb
326
327; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
328   ret i32 %old
329}
330
331define i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
332; CHECK-LABEL: test_atomic_load_and_i64:
333; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i64:
334; OUTLINE_ATOMICS:       // %bb.0:
335; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
336; OUTLINE_ATOMICS-NEXT:    mvn x0, x0
337; OUTLINE_ATOMICS-NEXT:    adrp x1, var64
338; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var64
339; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldclr8_acq
340; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
341; OUTLINE_ATOMICS-NEXT:    ret
342   %old = atomicrmw and i64* @var64, i64 %offset acquire
343; CHECK-NOT: dmb
344; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
345; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
346
347; CHECK: .LBB{{[0-9]+}}_1:
348; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
349  ; x0 below is a reasonable guess but could change: it certainly comes into the
350  ; function there.
351; CHECK-NEXT: and [[NEW:x[0-9]+]], x[[OLD]], x0
352; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
353; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
354; CHECK-NOT: dmb
355
356; CHECK: mov x0, x[[OLD]]
357   ret i64 %old
358}
359
360define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
361; CHECK-LABEL: test_atomic_load_or_i8:
362; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i8:
363; OUTLINE_ATOMICS:       // %bb.0:
364; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
365; OUTLINE_ATOMICS-NEXT:    adrp x1, var8
366; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var8
367; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldset1_acq_rel
368; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
369; OUTLINE_ATOMICS-NEXT:    ret
370   %old = atomicrmw or i8* @var8, i8 %offset seq_cst
371; CHECK-NOT: dmb
372; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
373; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
374
375; CHECK: .LBB{{[0-9]+}}_1:
376; ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
377  ; w0 below is a reasonable guess but could change: it certainly comes into the
378  ;  function there.
379; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
380; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
381; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
382; CHECK-NOT: dmb
383
384; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
385   ret i8 %old
386}
387
388define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
389; CHECK-LABEL: test_atomic_load_or_i16:
390; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i16:
391; OUTLINE_ATOMICS:       // %bb.0:
392; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
393; OUTLINE_ATOMICS-NEXT:    adrp x1, var16
394; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var16
395; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldset2_relax
396; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
397; OUTLINE_ATOMICS-NEXT:    ret
398   %old = atomicrmw or i16* @var16, i16 %offset monotonic
399; CHECK-NOT: dmb
400; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
401; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
402
403; CHECK: .LBB{{[0-9]+}}_1:
404; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
405  ; w0 below is a reasonable guess but could change: it certainly comes into the
406  ;  function there.
407; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
408; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
409; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
410; CHECK-NOT: dmb
411
412; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
413   ret i16 %old
414}
415
416define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
417; CHECK-LABEL: test_atomic_load_or_i32:
418; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i32:
419; OUTLINE_ATOMICS:       // %bb.0:
420; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
421; OUTLINE_ATOMICS-NEXT:    adrp x1, var32
422; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var32
423; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldset4_acq
424; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
425; OUTLINE_ATOMICS-NEXT:    ret
426   %old = atomicrmw or i32* @var32, i32 %offset acquire
427; CHECK-NOT: dmb
428; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
429; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
430
431; CHECK: .LBB{{[0-9]+}}_1:
432; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
433  ; w0 below is a reasonable guess but could change: it certainly comes into the
434  ;  function there.
435; CHECK-NEXT: orr [[NEW:w[0-9]+]], w[[OLD]], w0
436; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
437; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
438; CHECK-NOT: dmb
439
440; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
441   ret i32 %old
442}
443
444define i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
445; CHECK-LABEL: test_atomic_load_or_i64:
446; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i64:
447; OUTLINE_ATOMICS:       // %bb.0:
448; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
449; OUTLINE_ATOMICS-NEXT:    adrp x1, var64
450; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var64
451; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldset8_rel
452; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
453; OUTLINE_ATOMICS-NEXT:    ret
454   %old = atomicrmw or i64* @var64, i64 %offset release
455; CHECK-NOT: dmb
456; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
457; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
458
459; CHECK: .LBB{{[0-9]+}}_1:
460; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
461  ; x0 below is a reasonable guess but could change: it certainly comes into the
462  ; function there.
463; CHECK-NEXT: orr [[NEW:x[0-9]+]], x[[OLD]], x0
464; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
465; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
466; CHECK-NOT: dmb
467
468; CHECK: mov x0, x[[OLD]]
469   ret i64 %old
470}
471
472define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
473; CHECK-LABEL: test_atomic_load_xor_i8:
474; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i8:
475; OUTLINE_ATOMICS:       // %bb.0:
476; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
477; OUTLINE_ATOMICS-NEXT:    adrp x1, var8
478; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var8
479; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldeor1_acq
480; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
481; OUTLINE_ATOMICS-NEXT:    ret
482   %old = atomicrmw xor i8* @var8, i8 %offset acquire
483; CHECK-NOT: dmb
484; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
485; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
486
487; CHECK: .LBB{{[0-9]+}}_1:
488; ; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
489  ; w0 below is a reasonable guess but could change: it certainly comes into the
490  ;  function there.
491; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
492; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
493; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
494; CHECK-NOT: dmb
495
496; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
497   ret i8 %old
498}
499
500define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
501; CHECK-LABEL: test_atomic_load_xor_i16:
502; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i16:
503; OUTLINE_ATOMICS:       // %bb.0:
504; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
505; OUTLINE_ATOMICS-NEXT:    adrp x1, var16
506; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var16
507; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldeor2_rel
508; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
509; OUTLINE_ATOMICS-NEXT:    ret
510   %old = atomicrmw xor i16* @var16, i16 %offset release
511; CHECK-NOT: dmb
512; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
513; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
514
515; CHECK: .LBB{{[0-9]+}}_1:
516; ; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
517  ; w0 below is a reasonable guess but could change: it certainly comes into the
518  ;  function there.
519; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
520; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
521; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
522; CHECK-NOT: dmb
523
524; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
525   ret i16 %old
526}
527
528define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
529; CHECK-LABEL: test_atomic_load_xor_i32:
530; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i32:
531; OUTLINE_ATOMICS:       // %bb.0:
532; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
533; OUTLINE_ATOMICS-NEXT:    adrp x1, var32
534; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var32
535; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldeor4_acq_rel
536; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
537; OUTLINE_ATOMICS-NEXT:    ret
538   %old = atomicrmw xor i32* @var32, i32 %offset seq_cst
539; CHECK-NOT: dmb
540; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
541; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
542
543; CHECK: .LBB{{[0-9]+}}_1:
544; ; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
545  ; w0 below is a reasonable guess but could change: it certainly comes into the
546  ;  function there.
547; CHECK-NEXT: eor [[NEW:w[0-9]+]], w[[OLD]], w0
548; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
549; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
550; CHECK-NOT: dmb
551
552; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
553   ret i32 %old
554}
555
556define i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
557; CHECK-LABEL: test_atomic_load_xor_i64:
558; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i64:
559; OUTLINE_ATOMICS:       // %bb.0:
560; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
561; OUTLINE_ATOMICS-NEXT:    adrp x1, var64
562; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var64
563; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldeor8_relax
564; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
565; OUTLINE_ATOMICS-NEXT:    ret
566   %old = atomicrmw xor i64* @var64, i64 %offset monotonic
567; CHECK-NOT: dmb
568; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
569; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
570
571; CHECK: .LBB{{[0-9]+}}_1:
572; ; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
573  ; x0 below is a reasonable guess but could change: it certainly comes into the
574  ; function there.
575; CHECK-NEXT: eor [[NEW:x[0-9]+]], x[[OLD]], x0
576; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
577; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
578; CHECK-NOT: dmb
579
580; CHECK: mov x0, x[[OLD]]
581   ret i64 %old
582}
583
584define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
585; CHECK-LABEL: test_atomic_load_xchg_i8:
586; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i8:
587; OUTLINE_ATOMICS:       // %bb.0:
588; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
589; OUTLINE_ATOMICS-NEXT:    adrp x1, var8
590; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var8
591; OUTLINE_ATOMICS-NEXT:    bl __aarch64_swp1_relax
592; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
593; OUTLINE_ATOMICS-NEXT:    ret
594   %old = atomicrmw xchg i8* @var8, i8 %offset monotonic
595; CHECK-NOT: dmb
596; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
597; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
598
599; CHECK: .LBB{{[0-9]+}}_1:
600; ; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
601  ; w0 below is a reasonable guess but could change: it certainly comes into the
602  ; function there.
603; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
604; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
605; CHECK-NOT: dmb
606
607; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
608   ret i8 %old
609}
610
611define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
612; CHECK-LABEL: test_atomic_load_xchg_i16:
613; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i16:
614; OUTLINE_ATOMICS:       // %bb.0:
615; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
616; OUTLINE_ATOMICS-NEXT:    adrp x1, var16
617; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var16
618; OUTLINE_ATOMICS-NEXT:    bl __aarch64_swp2_acq_rel
619; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
620; OUTLINE_ATOMICS-NEXT:    ret
621   %old = atomicrmw xchg i16* @var16, i16 %offset seq_cst
622; CHECK-NOT: dmb
623; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
624; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
625
626; CHECK: .LBB{{[0-9]+}}_1:
627; ; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
628  ; w0 below is a reasonable guess but could change: it certainly comes into the
629  ; function there.
630; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], w0, [x[[ADDR]]]
631; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
632; CHECK-NOT: dmb
633
634; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
635   ret i16 %old
636}
637
638define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
639; CHECK-LABEL: test_atomic_load_xchg_i32:
640; CHECK: mov {{[xw]}}8, w[[OLD:[0-9]+]]
641; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i32:
642; OUTLINE_ATOMICS:       // %bb.0:
643; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
644; OUTLINE_ATOMICS-NEXT:    adrp x1, var32
645; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var32
646; OUTLINE_ATOMICS-NEXT:    bl __aarch64_swp4_rel
647; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
648; OUTLINE_ATOMICS-NEXT:    ret
649   %old = atomicrmw xchg i32* @var32, i32 %offset release
650; CHECK-NOT: dmb
651; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
652; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
653
654; CHECK: .LBB{{[0-9]+}}_1:
655; ; CHECK: ldxr {{[xw]}}[[OLD]], [x[[ADDR]]]
656  ; w0 below is a reasonable guess but could change: it certainly comes into the
657  ;  function there.
658; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], w8, [x[[ADDR]]]
659; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
660; CHECK-NOT: dmb
661   ret i32 %old
662}
663
664define i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
665; CHECK-LABEL: test_atomic_load_xchg_i64:
666; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i64:
667; OUTLINE_ATOMICS:       // %bb.0:
668; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
669; OUTLINE_ATOMICS-NEXT:    adrp x1, var64
670; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var64
671; OUTLINE_ATOMICS-NEXT:    bl __aarch64_swp8_acq
672; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
673; OUTLINE_ATOMICS-NEXT:    ret
674   %old = atomicrmw xchg i64* @var64, i64 %offset acquire
675; CHECK-NOT: dmb
676; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
677; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
678
679; CHECK: .LBB{{[0-9]+}}_1:
680; ; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
681  ; x0 below is a reasonable guess but could change: it certainly comes into the
682  ; function there.
683; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], x0, [x[[ADDR]]]
684; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
685; CHECK-NOT: dmb
686
687; CHECK: mov x0, x[[OLD]]
688   ret i64 %old
689}
690
691
692define i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
693; CHECK-LABEL: test_atomic_load_min_i8:
694; OUTLINE_ATOMICS-LABEL: test_atomic_load_min_i8:
695; OUTLINE_ATOMICS:       // %bb.0:
696; OUTLINE_ATOMICS-NEXT:    adrp x9, var8
697; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
698; OUTLINE_ATOMICS-NEXT:  .LBB24_1: // %atomicrmw.start
699; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
700; OUTLINE_ATOMICS-NEXT:    ldaxrb w10, [x9]
701; OUTLINE_ATOMICS-NEXT:    sxtb w8, w10
702; OUTLINE_ATOMICS-NEXT:    cmp w8, w0, sxtb
703; OUTLINE_ATOMICS-NEXT:    csel w10, w10, w0, le
704; OUTLINE_ATOMICS-NEXT:    stxrb w11, w10, [x9]
705; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB24_1
706; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
707; OUTLINE_ATOMICS-NEXT:    mov w0, w8
708; OUTLINE_ATOMICS-NEXT:    ret
709   %old = atomicrmw min i8* @var8, i8 %offset acquire
710; CHECK-NOT: dmb
711; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
712; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
713
714; CHECK: .LBB{{[0-9]+}}_1:
715; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
716  ; w0 below is a reasonable guess but could change: it certainly comes into the
717  ;  function there.
718
719; CHECK-NEXT: sxtb w[[OLD_EXT:[0-9]+]], w[[OLD]]
720; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxtb
721; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
722
723; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
724; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
725; CHECK-NOT: dmb
726
727; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
728   ret i8 %old
729}
730
731define i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
732; CHECK-LABEL: test_atomic_load_min_i16:
733; OUTLINE_ATOMICS-LABEL: test_atomic_load_min_i16:
734; OUTLINE_ATOMICS:       // %bb.0:
735; OUTLINE_ATOMICS-NEXT:    adrp x9, var16
736; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
737; OUTLINE_ATOMICS-NEXT:  .LBB25_1: // %atomicrmw.start
738; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
739; OUTLINE_ATOMICS-NEXT:    ldxrh w10, [x9]
740; OUTLINE_ATOMICS-NEXT:    sxth w8, w10
741; OUTLINE_ATOMICS-NEXT:    cmp w8, w0, sxth
742; OUTLINE_ATOMICS-NEXT:    csel w10, w10, w0, le
743; OUTLINE_ATOMICS-NEXT:    stlxrh w11, w10, [x9]
744; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB25_1
745; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
746; OUTLINE_ATOMICS-NEXT:    mov w0, w8
747; OUTLINE_ATOMICS-NEXT:    ret
748   %old = atomicrmw min i16* @var16, i16 %offset release
749; CHECK-NOT: dmb
750; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
751; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
752
753; CHECK: .LBB{{[0-9]+}}_1:
754; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
755  ; w0 below is a reasonable guess but could change: it certainly comes into the
756  ;  function there.
757
758; CHECK-NEXT: sxth w[[OLD_EXT:[0-9]+]], w[[OLD]]
759; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxth
760; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
761
762
763; CHECK-NEXT: stlxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
764; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
765; CHECK-NOT: dmb
766
767; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
768   ret i16 %old
769}
770
771define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
772; CHECK-LABEL: test_atomic_load_min_i32:
773; OUTLINE_ATOMICS-LABEL: test_atomic_load_min_i32:
774; OUTLINE_ATOMICS:       // %bb.0:
775; OUTLINE_ATOMICS-NEXT:    adrp x9, var32
776; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var32
777; OUTLINE_ATOMICS-NEXT:  .LBB26_1: // %atomicrmw.start
778; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
779; OUTLINE_ATOMICS-NEXT:    ldxr w8, [x9]
780; OUTLINE_ATOMICS-NEXT:    cmp w8, w0
781; OUTLINE_ATOMICS-NEXT:    csel w10, w8, w0, le
782; OUTLINE_ATOMICS-NEXT:    stxr w11, w10, [x9]
783; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB26_1
784; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
785; OUTLINE_ATOMICS-NEXT:    mov w0, w8
786; OUTLINE_ATOMICS-NEXT:    ret
787   %old = atomicrmw min i32* @var32, i32 %offset monotonic
788; CHECK-NOT: dmb
789; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
790; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
791
792; CHECK: .LBB{{[0-9]+}}_1:
793; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
794  ; w0 below is a reasonable guess but could change: it certainly comes into the
795  ;  function there.
796
797; CHECK-NEXT: cmp w[[OLD]], w0
798; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, le
799
800
801; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
802; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
803; CHECK-NOT: dmb
804
805; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
806   ret i32 %old
807}
808
809define i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
810; CHECK-LABEL: test_atomic_load_min_i64:
811; OUTLINE_ATOMICS-LABEL: test_atomic_load_min_i64:
812; OUTLINE_ATOMICS:       // %bb.0:
813; OUTLINE_ATOMICS-NEXT:    adrp x9, var64
814; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var64
815; OUTLINE_ATOMICS-NEXT:  .LBB27_1: // %atomicrmw.start
816; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
817; OUTLINE_ATOMICS-NEXT:    ldaxr x8, [x9]
818; OUTLINE_ATOMICS-NEXT:    cmp x8, x0
819; OUTLINE_ATOMICS-NEXT:    csel x10, x8, x0, le
820; OUTLINE_ATOMICS-NEXT:    stlxr w11, x10, [x9]
821; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB27_1
822; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
823; OUTLINE_ATOMICS-NEXT:    mov x0, x8
824; OUTLINE_ATOMICS-NEXT:    ret
825   %old = atomicrmw min i64* @var64, i64 %offset seq_cst
826; CHECK-NOT: dmb
827; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
828; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
829
830; CHECK: .LBB{{[0-9]+}}_1:
831; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
832  ; x0 below is a reasonable guess but could change: it certainly comes into the
833  ; function there.
834
835; CHECK-NEXT: cmp x[[OLD]], x0
836; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, le
837
838
839; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
840; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
841; CHECK-NOT: dmb
842
843; CHECK: mov x0, x[[OLD]]
844   ret i64 %old
845}
846
847define i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
848; CHECK-LABEL: test_atomic_load_max_i8:
849; OUTLINE_ATOMICS-LABEL: test_atomic_load_max_i8:
850; OUTLINE_ATOMICS:       // %bb.0:
851; OUTLINE_ATOMICS-NEXT:    adrp x9, var8
852; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
853; OUTLINE_ATOMICS-NEXT:  .LBB28_1: // %atomicrmw.start
854; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
855; OUTLINE_ATOMICS-NEXT:    ldaxrb w10, [x9]
856; OUTLINE_ATOMICS-NEXT:    sxtb w8, w10
857; OUTLINE_ATOMICS-NEXT:    cmp w8, w0, sxtb
858; OUTLINE_ATOMICS-NEXT:    csel w10, w10, w0, gt
859; OUTLINE_ATOMICS-NEXT:    stlxrb w11, w10, [x9]
860; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB28_1
861; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
862; OUTLINE_ATOMICS-NEXT:    mov w0, w8
863; OUTLINE_ATOMICS-NEXT:    ret
864   %old = atomicrmw max i8* @var8, i8 %offset seq_cst
865; CHECK-NOT: dmb
866; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
867; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
868
869; CHECK: .LBB{{[0-9]+}}_1:
870; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
871  ; w0 below is a reasonable guess but could change: it certainly comes into the
872  ;  function there.
873
874; CHECK-NEXT: sxtb w[[OLD_EXT:[0-9]+]], w[[OLD]]
875; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxtb
876; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
877
878
879; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
880; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
881; CHECK-NOT: dmb
882
883; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
884   ret i8 %old
885}
886
887define i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
888; CHECK-LABEL: test_atomic_load_max_i16:
889; OUTLINE_ATOMICS-LABEL: test_atomic_load_max_i16:
890; OUTLINE_ATOMICS:       // %bb.0:
891; OUTLINE_ATOMICS-NEXT:    adrp x9, var16
892; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
893; OUTLINE_ATOMICS-NEXT:  .LBB29_1: // %atomicrmw.start
894; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
895; OUTLINE_ATOMICS-NEXT:    ldaxrh w10, [x9]
896; OUTLINE_ATOMICS-NEXT:    sxth w8, w10
897; OUTLINE_ATOMICS-NEXT:    cmp w8, w0, sxth
898; OUTLINE_ATOMICS-NEXT:    csel w10, w10, w0, gt
899; OUTLINE_ATOMICS-NEXT:    stxrh w11, w10, [x9]
900; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB29_1
901; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
902; OUTLINE_ATOMICS-NEXT:    mov w0, w8
903; OUTLINE_ATOMICS-NEXT:    ret
904   %old = atomicrmw max i16* @var16, i16 %offset acquire
905; CHECK-NOT: dmb
906; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
907; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
908
909; CHECK: .LBB{{[0-9]+}}_1:
910; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
911  ; w0 below is a reasonable guess but could change: it certainly comes into the
912  ;  function there.
913
914; CHECK-NEXT: sxth w[[OLD_EXT:[0-9]+]], w[[OLD]]
915; CHECK-NEXT: cmp w[[OLD_EXT]], w0, sxth
916; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
917
918
919; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
920; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
921; CHECK-NOT: dmb
922
923; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD_EXT]]
924   ret i16 %old
925}
926
927define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
928; CHECK-LABEL: test_atomic_load_max_i32:
929; OUTLINE_ATOMICS-LABEL: test_atomic_load_max_i32:
930; OUTLINE_ATOMICS:       // %bb.0:
931; OUTLINE_ATOMICS-NEXT:    adrp x9, var32
932; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var32
933; OUTLINE_ATOMICS-NEXT:  .LBB30_1: // %atomicrmw.start
934; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
935; OUTLINE_ATOMICS-NEXT:    ldxr w8, [x9]
936; OUTLINE_ATOMICS-NEXT:    cmp w8, w0
937; OUTLINE_ATOMICS-NEXT:    csel w10, w8, w0, gt
938; OUTLINE_ATOMICS-NEXT:    stlxr w11, w10, [x9]
939; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB30_1
940; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
941; OUTLINE_ATOMICS-NEXT:    mov w0, w8
942; OUTLINE_ATOMICS-NEXT:    ret
943   %old = atomicrmw max i32* @var32, i32 %offset release
944; CHECK-NOT: dmb
945; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
946; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
947
948; CHECK: .LBB{{[0-9]+}}_1:
949; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
950  ; w0 below is a reasonable guess but could change: it certainly comes into the
951  ;  function there.
952
953; CHECK-NEXT: cmp w[[OLD]], w0
954; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, gt
955
956
957; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
958; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
959; CHECK-NOT: dmb
960
961; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
962   ret i32 %old
963}
964
965define i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
966; CHECK-LABEL: test_atomic_load_max_i64:
967; OUTLINE_ATOMICS-LABEL: test_atomic_load_max_i64:
968; OUTLINE_ATOMICS:       // %bb.0:
969; OUTLINE_ATOMICS-NEXT:    adrp x9, var64
970; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var64
971; OUTLINE_ATOMICS-NEXT:  .LBB31_1: // %atomicrmw.start
972; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
973; OUTLINE_ATOMICS-NEXT:    ldxr x8, [x9]
974; OUTLINE_ATOMICS-NEXT:    cmp x8, x0
975; OUTLINE_ATOMICS-NEXT:    csel x10, x8, x0, gt
976; OUTLINE_ATOMICS-NEXT:    stxr w11, x10, [x9]
977; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB31_1
978; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
979; OUTLINE_ATOMICS-NEXT:    mov x0, x8
980; OUTLINE_ATOMICS-NEXT:    ret
981   %old = atomicrmw max i64* @var64, i64 %offset monotonic
982; CHECK-NOT: dmb
983; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
984; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
985
986; CHECK: .LBB{{[0-9]+}}_1:
987; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
988  ; x0 below is a reasonable guess but could change: it certainly comes into the
989  ; function there.
990
991; CHECK-NEXT: cmp x[[OLD]], x0
992; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, gt
993
994
995; CHECK-NEXT: stxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
996; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
997; CHECK-NOT: dmb
998
999; CHECK: mov x0, x[[OLD]]
1000   ret i64 %old
1001}
1002
1003define i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
1004; CHECK-LABEL: test_atomic_load_umin_i8:
1005; OUTLINE_ATOMICS-LABEL: test_atomic_load_umin_i8:
1006; OUTLINE_ATOMICS:       // %bb.0:
1007; OUTLINE_ATOMICS-NEXT:    adrp x9, var8
1008; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
1009; OUTLINE_ATOMICS-NEXT:  .LBB32_1: // %atomicrmw.start
1010; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
1011; OUTLINE_ATOMICS-NEXT:    ldxrb w8, [x9]
1012; OUTLINE_ATOMICS-NEXT:    cmp w8, w0, uxtb
1013; OUTLINE_ATOMICS-NEXT:    csel w10, w8, w0, ls
1014; OUTLINE_ATOMICS-NEXT:    stxrb w11, w10, [x9]
1015; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB32_1
1016; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
1017; OUTLINE_ATOMICS-NEXT:    mov w0, w8
1018; OUTLINE_ATOMICS-NEXT:    ret
1019   %old = atomicrmw umin i8* @var8, i8 %offset monotonic
1020; CHECK-NOT: dmb
1021; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1022; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1023
1024; CHECK: .LBB{{[0-9]+}}_1:
1025; CHECK: ldxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
1026  ; w0 below is a reasonable guess but could change: it certainly comes into the
1027  ;  function there.
1028
1029; CHECK-NEXT: cmp w[[OLD]], w0, uxtb
1030; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
1031
1032
1033; CHECK-NEXT: stxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
1034; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
1035; CHECK-NOT: dmb
1036
1037; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
1038   ret i8 %old
1039}
1040
1041define i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
1042; CHECK-LABEL: test_atomic_load_umin_i16:
1043; OUTLINE_ATOMICS-LABEL: test_atomic_load_umin_i16:
1044; OUTLINE_ATOMICS:       // %bb.0:
1045; OUTLINE_ATOMICS-NEXT:    adrp x9, var16
1046; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
1047; OUTLINE_ATOMICS-NEXT:  .LBB33_1: // %atomicrmw.start
1048; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
1049; OUTLINE_ATOMICS-NEXT:    ldaxrh w8, [x9]
1050; OUTLINE_ATOMICS-NEXT:    cmp w8, w0, uxth
1051; OUTLINE_ATOMICS-NEXT:    csel w10, w8, w0, ls
1052; OUTLINE_ATOMICS-NEXT:    stxrh w11, w10, [x9]
1053; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB33_1
1054; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
1055; OUTLINE_ATOMICS-NEXT:    mov w0, w8
1056; OUTLINE_ATOMICS-NEXT:    ret
1057   %old = atomicrmw umin i16* @var16, i16 %offset acquire
1058; CHECK-NOT: dmb
1059; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1060; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1061
1062; CHECK: .LBB{{[0-9]+}}_1:
1063; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
1064  ; w0 below is a reasonable guess but could change: it certainly comes into the
1065  ;  function there.
1066
1067; CHECK-NEXT: cmp w[[OLD]], w0, uxth
1068; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
1069
1070
1071; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
1072; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
1073; CHECK-NOT: dmb
1074
1075; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
1076   ret i16 %old
1077}
1078
1079define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
1080; CHECK-LABEL: test_atomic_load_umin_i32:
1081; OUTLINE_ATOMICS-LABEL: test_atomic_load_umin_i32:
1082; OUTLINE_ATOMICS:       // %bb.0:
1083; OUTLINE_ATOMICS-NEXT:    adrp x9, var32
1084; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var32
1085; OUTLINE_ATOMICS-NEXT:  .LBB34_1: // %atomicrmw.start
1086; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
1087; OUTLINE_ATOMICS-NEXT:    ldaxr w8, [x9]
1088; OUTLINE_ATOMICS-NEXT:    cmp w8, w0
1089; OUTLINE_ATOMICS-NEXT:    csel w10, w8, w0, ls
1090; OUTLINE_ATOMICS-NEXT:    stlxr w11, w10, [x9]
1091; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB34_1
1092; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
1093; OUTLINE_ATOMICS-NEXT:    mov w0, w8
1094; OUTLINE_ATOMICS-NEXT:    ret
1095   %old = atomicrmw umin i32* @var32, i32 %offset seq_cst
1096; CHECK-NOT: dmb
1097; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1098; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1099
1100; CHECK: .LBB{{[0-9]+}}_1:
1101; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
1102  ; w0 below is a reasonable guess but could change: it certainly comes into the
1103  ;  function there.
1104
1105; CHECK-NEXT: cmp w[[OLD]], w0
1106; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, ls
1107
1108
1109; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
1110; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
1111; CHECK-NOT: dmb
1112
1113; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
1114   ret i32 %old
1115}
1116
1117define i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
1118; CHECK-LABEL: test_atomic_load_umin_i64:
1119; OUTLINE_ATOMICS-LABEL: test_atomic_load_umin_i64:
1120; OUTLINE_ATOMICS:       // %bb.0:
1121; OUTLINE_ATOMICS-NEXT:    adrp x9, var64
1122; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var64
1123; OUTLINE_ATOMICS-NEXT:  .LBB35_1: // %atomicrmw.start
1124; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
1125; OUTLINE_ATOMICS-NEXT:    ldaxr x8, [x9]
1126; OUTLINE_ATOMICS-NEXT:    cmp x8, x0
1127; OUTLINE_ATOMICS-NEXT:    csel x10, x8, x0, ls
1128; OUTLINE_ATOMICS-NEXT:    stlxr w11, x10, [x9]
1129; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB35_1
1130; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
1131; OUTLINE_ATOMICS-NEXT:    mov x0, x8
1132; OUTLINE_ATOMICS-NEXT:    ret
1133   %old = atomicrmw umin i64* @var64, i64 %offset acq_rel
1134; CHECK-NOT: dmb
1135; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1136; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1137
1138; CHECK: .LBB{{[0-9]+}}_1:
1139; CHECK: ldaxr x[[OLD:[0-9]+]], [x[[ADDR]]]
1140  ; x0 below is a reasonable guess but could change: it certainly comes into the
1141  ; function there.
1142
1143; CHECK-NEXT: cmp x[[OLD]], x0
1144; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, ls
1145
1146
1147; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
1148; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
1149; CHECK-NOT: dmb
1150
1151; CHECK: mov x0, x[[OLD]]
1152   ret i64 %old
1153}
1154
1155define i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
1156; CHECK-LABEL: test_atomic_load_umax_i8:
1157; OUTLINE_ATOMICS-LABEL: test_atomic_load_umax_i8:
1158; OUTLINE_ATOMICS:       // %bb.0:
1159; OUTLINE_ATOMICS-NEXT:    adrp x9, var8
1160; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
1161; OUTLINE_ATOMICS-NEXT:  .LBB36_1: // %atomicrmw.start
1162; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
1163; OUTLINE_ATOMICS-NEXT:    ldaxrb w8, [x9]
1164; OUTLINE_ATOMICS-NEXT:    cmp w8, w0, uxtb
1165; OUTLINE_ATOMICS-NEXT:    csel w10, w8, w0, hi
1166; OUTLINE_ATOMICS-NEXT:    stlxrb w11, w10, [x9]
1167; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB36_1
1168; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
1169; OUTLINE_ATOMICS-NEXT:    mov w0, w8
1170; OUTLINE_ATOMICS-NEXT:    ret
1171   %old = atomicrmw umax i8* @var8, i8 %offset acq_rel
1172; CHECK-NOT: dmb
1173; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1174; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1175
1176; CHECK: .LBB{{[0-9]+}}_1:
1177; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
1178  ; w0 below is a reasonable guess but could change: it certainly comes into the
1179  ;  function there.
1180
1181; CHECK-NEXT: cmp w[[OLD]], w0, uxtb
1182; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
1183
1184
1185; CHECK-NEXT: stlxrb [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
1186; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
1187; CHECK-NOT: dmb
1188
1189; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
1190   ret i8 %old
1191}
1192
1193define i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
1194; CHECK-LABEL: test_atomic_load_umax_i16:
1195; OUTLINE_ATOMICS-LABEL: test_atomic_load_umax_i16:
1196; OUTLINE_ATOMICS:       // %bb.0:
1197; OUTLINE_ATOMICS-NEXT:    adrp x9, var16
1198; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
1199; OUTLINE_ATOMICS-NEXT:  .LBB37_1: // %atomicrmw.start
1200; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
1201; OUTLINE_ATOMICS-NEXT:    ldxrh w8, [x9]
1202; OUTLINE_ATOMICS-NEXT:    cmp w8, w0, uxth
1203; OUTLINE_ATOMICS-NEXT:    csel w10, w8, w0, hi
1204; OUTLINE_ATOMICS-NEXT:    stxrh w11, w10, [x9]
1205; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB37_1
1206; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
1207; OUTLINE_ATOMICS-NEXT:    mov w0, w8
1208; OUTLINE_ATOMICS-NEXT:    ret
1209   %old = atomicrmw umax i16* @var16, i16 %offset monotonic
1210; CHECK-NOT: dmb
1211; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1212; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1213
1214; CHECK: .LBB{{[0-9]+}}_1:
1215; CHECK: ldxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
1216  ; w0 below is a reasonable guess but could change: it certainly comes into the
1217  ;  function there.
1218
1219; CHECK-NEXT: cmp w[[OLD]], w0, uxth
1220; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
1221
1222
1223; CHECK-NEXT: stxrh [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
1224; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
1225; CHECK-NOT: dmb
1226
1227; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
1228   ret i16 %old
1229}
1230
1231define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
1232; CHECK-LABEL: test_atomic_load_umax_i32:
1233; OUTLINE_ATOMICS-LABEL: test_atomic_load_umax_i32:
1234; OUTLINE_ATOMICS:       // %bb.0:
1235; OUTLINE_ATOMICS-NEXT:    adrp x9, var32
1236; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var32
1237; OUTLINE_ATOMICS-NEXT:  .LBB38_1: // %atomicrmw.start
1238; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
1239; OUTLINE_ATOMICS-NEXT:    ldaxr w8, [x9]
1240; OUTLINE_ATOMICS-NEXT:    cmp w8, w0
1241; OUTLINE_ATOMICS-NEXT:    csel w10, w8, w0, hi
1242; OUTLINE_ATOMICS-NEXT:    stlxr w11, w10, [x9]
1243; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB38_1
1244; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
1245; OUTLINE_ATOMICS-NEXT:    mov w0, w8
1246; OUTLINE_ATOMICS-NEXT:    ret
1247   %old = atomicrmw umax i32* @var32, i32 %offset seq_cst
1248; CHECK-NOT: dmb
1249; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1250; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1251
1252; CHECK: .LBB{{[0-9]+}}_1:
1253; CHECK: ldaxr w[[OLD:[0-9]+]], [x[[ADDR]]]
1254  ; w0 below is a reasonable guess but could change: it certainly comes into the
1255  ;  function there.
1256
1257; CHECK-NEXT: cmp w[[OLD]], w0
1258; CHECK-NEXT: csel [[NEW:w[0-9]+]], w[[OLD]], w0, hi
1259
1260
1261; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
1262; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
1263; CHECK-NOT: dmb
1264
1265; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
1266   ret i32 %old
1267}
1268
1269define i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
1270; CHECK-LABEL: test_atomic_load_umax_i64:
1271; OUTLINE_ATOMICS-LABEL: test_atomic_load_umax_i64:
1272; OUTLINE_ATOMICS:       // %bb.0:
1273; OUTLINE_ATOMICS-NEXT:    adrp x9, var64
1274; OUTLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var64
1275; OUTLINE_ATOMICS-NEXT:  .LBB39_1: // %atomicrmw.start
1276; OUTLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
1277; OUTLINE_ATOMICS-NEXT:    ldxr x8, [x9]
1278; OUTLINE_ATOMICS-NEXT:    cmp x8, x0
1279; OUTLINE_ATOMICS-NEXT:    csel x10, x8, x0, hi
1280; OUTLINE_ATOMICS-NEXT:    stlxr w11, x10, [x9]
1281; OUTLINE_ATOMICS-NEXT:    cbnz w11, .LBB39_1
1282; OUTLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
1283; OUTLINE_ATOMICS-NEXT:    mov x0, x8
1284; OUTLINE_ATOMICS-NEXT:    ret
1285   %old = atomicrmw umax i64* @var64, i64 %offset release
1286; CHECK-NOT: dmb
1287; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1288; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1289
1290; CHECK: .LBB{{[0-9]+}}_1:
1291; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
1292  ; x0 below is a reasonable guess but could change: it certainly comes into the
1293  ; function there.
1294
1295; CHECK-NEXT: cmp x[[OLD]], x0
1296; CHECK-NEXT: csel [[NEW:x[0-9]+]], x[[OLD]], x0, hi
1297
1298
1299; CHECK-NEXT: stlxr [[STATUS:w[0-9]+]], [[NEW]], [x[[ADDR]]]
1300; CHECK-NEXT: cbnz [[STATUS]], .LBB{{[0-9]+}}_1
1301; CHECK-NOT: dmb
1302
1303; CHECK: mov x0, x[[OLD]]
1304   ret i64 %old
1305}
1306
1307define i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
1308; CHECK-LABEL: test_atomic_cmpxchg_i8:
1309; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i8:
1310; OUTLINE_ATOMICS:       // %bb.0:
1311; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
1312; OUTLINE_ATOMICS-NEXT:    adrp x2, var8
1313; OUTLINE_ATOMICS-NEXT:    add x2, x2, :lo12:var8
1314; OUTLINE_ATOMICS-NEXT:    bl __aarch64_cas1_acq
1315; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
1316; OUTLINE_ATOMICS-NEXT:    ret
1317   %pair = cmpxchg i8* @var8, i8 %wanted, i8 %new acquire acquire
1318   %old = extractvalue { i8, i1 } %pair, 0
1319
1320; CHECK-NOT: dmb
1321; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1322; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1323
1324; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
1325; CHECK: ldaxrb w[[OLD:[0-9]+]], [x[[ADDR]]]
1326  ; w0 below is a reasonable guess but could change: it certainly comes into the
1327  ;  function there.
1328; CHECK-NEXT: cmp w[[OLD]], w0
1329; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
1330; CHECK: stxrb [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
1331; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
1332; CHECK: [[GET_OUT]]:
1333; CHECK: clrex
1334; CHECK-NOT: dmb
1335
1336; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
1337   ret i8 %old
1338}
1339
1340define i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
1341; CHECK-LABEL: test_atomic_cmpxchg_i16:
1342; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i16:
1343; OUTLINE_ATOMICS:       // %bb.0:
1344; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
1345; OUTLINE_ATOMICS-NEXT:    adrp x2, var16
1346; OUTLINE_ATOMICS-NEXT:    add x2, x2, :lo12:var16
1347; OUTLINE_ATOMICS-NEXT:    bl __aarch64_cas2_acq_rel
1348; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
1349; OUTLINE_ATOMICS-NEXT:    ret
1350   %pair = cmpxchg i16* @var16, i16 %wanted, i16 %new seq_cst seq_cst
1351   %old = extractvalue { i16, i1 } %pair, 0
1352
1353; CHECK-NOT: dmb
1354; CHECK: adrp [[TMPADDR:x[0-9]+]], var16
1355; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16
1356
1357; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
1358; CHECK: ldaxrh w[[OLD:[0-9]+]], [x[[ADDR]]]
1359  ; w0 below is a reasonable guess but could change: it certainly comes into the
1360  ;  function there.
1361; CHECK-NEXT: cmp w[[OLD]], w0
1362; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
1363; CHECK: stlxrh [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
1364; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
1365; CHECK: [[GET_OUT]]:
1366; CHECK: clrex
1367; CHECK-NOT: dmb
1368
1369; CHECK: mov {{[xw]}}0, {{[xw]}}[[OLD]]
1370   ret i16 %old
1371}
1372
1373define i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
1374; CHECK-LABEL: test_atomic_cmpxchg_i32:
1375; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i32:
1376; OUTLINE_ATOMICS:       // %bb.0:
1377; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
1378; OUTLINE_ATOMICS-NEXT:    adrp x2, var32
1379; OUTLINE_ATOMICS-NEXT:    add x2, x2, :lo12:var32
1380; OUTLINE_ATOMICS-NEXT:    bl __aarch64_cas4_rel
1381; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
1382; OUTLINE_ATOMICS-NEXT:    ret
1383   %pair = cmpxchg i32* @var32, i32 %wanted, i32 %new release monotonic
1384   %old = extractvalue { i32, i1 } %pair, 0
1385
1386; CHECK: mov {{[xw]}}[[WANTED:[0-9]+]], {{[xw]}}0
1387
1388; CHECK-NOT: dmb
1389; CHECK: adrp [[TMPADDR:x[0-9]+]], var32
1390; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32
1391
1392; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
1393; CHECK: ldxr w[[OLD:[0-9]+]], [x[[ADDR]]]
1394; CHECK-NEXT: cmp w[[OLD]], w[[WANTED]]
1395; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
1396; CHECK: stlxr [[STATUS:w[0-9]+]], {{w[0-9]+}}, [x[[ADDR]]]
1397; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
1398; CHECK: [[GET_OUT]]:
1399; CHECK: clrex
1400; CHECK-NOT: dmb
1401   ret i32 %old
1402}
1403
1404define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
1405; CHECK-LABEL: test_atomic_cmpxchg_i64:
1406; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i64:
1407; OUTLINE_ATOMICS:       // %bb.0:
1408; OUTLINE_ATOMICS-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
1409; OUTLINE_ATOMICS-NEXT:    adrp x19, var64
1410; OUTLINE_ATOMICS-NEXT:    add x19, x19, :lo12:var64
1411; OUTLINE_ATOMICS-NEXT:    mov x2, x19
1412; OUTLINE_ATOMICS-NEXT:    bl __aarch64_cas8_relax
1413; OUTLINE_ATOMICS-NEXT:    str x0, [x19]
1414; OUTLINE_ATOMICS-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
1415; OUTLINE_ATOMICS-NEXT:    ret
1416   %pair = cmpxchg i64* @var64, i64 %wanted, i64 %new monotonic monotonic
1417   %old = extractvalue { i64, i1 } %pair, 0
1418
1419; CHECK-NOT: dmb
1420; CHECK: adrp [[TMPADDR:x[0-9]+]], var64
1421; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64
1422
1423; CHECK: [[STARTAGAIN:.LBB[0-9]+_[0-9]+]]:
1424; CHECK: ldxr x[[OLD:[0-9]+]], [x[[ADDR]]]
1425  ; w0 below is a reasonable guess but could change: it certainly comes into the
1426  ;  function there.
1427; CHECK-NEXT: cmp x[[OLD]], x0
1428; CHECK-NEXT: b.ne [[GET_OUT:.LBB[0-9]+_[0-9]+]]
1429  ; As above, w1 is a reasonable guess.
1430; CHECK: stxr [[STATUS:w[0-9]+]], x1, [x[[ADDR]]]
1431; CHECK-NEXT: cbnz [[STATUS]], [[STARTAGAIN]]
1432; CHECK: [[GET_OUT]]:
1433; CHECK: clrex
1434; CHECK-NOT: dmb
1435
1436; CHECK: str x[[OLD]],
1437   store i64 %old, i64* @var64
1438   ret void
1439}
1440
1441define i8 @test_atomic_load_monotonic_i8() nounwind {
1442; CHECK-LABEL: test_atomic_load_monotonic_i8:
1443; OUTLINE_ATOMICS-LABEL: test_atomic_load_monotonic_i8:
1444; OUTLINE_ATOMICS:       // %bb.0:
1445; OUTLINE_ATOMICS-NEXT:    adrp x8, var8
1446; OUTLINE_ATOMICS-NEXT:    ldrb w0, [x8, :lo12:var8]
1447; OUTLINE_ATOMICS-NEXT:    ret
1448  %val = load atomic i8, i8* @var8 monotonic, align 1
1449; CHECK-NOT: dmb
1450; CHECK: adrp x[[HIADDR:[0-9]+]], var8
1451; CHECK: ldrb w0, [x[[HIADDR]], {{#?}}:lo12:var8]
1452; CHECK-NOT: dmb
1453
1454  ret i8 %val
1455}
1456
1457define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
1458; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8:
1459; OUTLINE_ATOMICS-LABEL: test_atomic_load_monotonic_regoff_i8:
1460; OUTLINE_ATOMICS:       // %bb.0:
1461; OUTLINE_ATOMICS-NEXT:    ldrb w0, [x0, x1]
1462; OUTLINE_ATOMICS-NEXT:    ret
1463  %addr_int = add i64 %base, %off
1464  %addr = inttoptr i64 %addr_int to i8*
1465
1466  %val = load atomic i8, i8* %addr monotonic, align 1
1467; CHECK-NOT: dmb
1468; CHECK: ldrb w0, [x0, x1]
1469; CHECK-NOT: dmb
1470
1471  ret i8 %val
1472}
1473
1474define i8 @test_atomic_load_acquire_i8() nounwind {
1475; CHECK-LABEL: test_atomic_load_acquire_i8:
1476; OUTLINE_ATOMICS-LABEL: test_atomic_load_acquire_i8:
1477; OUTLINE_ATOMICS:       // %bb.0:
1478; OUTLINE_ATOMICS-NEXT:    adrp x8, var8
1479; OUTLINE_ATOMICS-NEXT:    add x8, x8, :lo12:var8
1480; OUTLINE_ATOMICS-NEXT:    ldarb w0, [x8]
1481; OUTLINE_ATOMICS-NEXT:    ret
1482  %val = load atomic i8, i8* @var8 acquire, align 1
1483; CHECK-NOT: dmb
1484; CHECK: adrp [[TMPADDR:x[0-9]+]], var8
1485; CHECK-NOT: dmb
1486; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8
1487; CHECK-NOT: dmb
1488; CHECK: ldarb w0, [x[[ADDR]]]
1489; CHECK-NOT: dmb
1490  ret i8 %val
1491}
1492
1493define i8 @test_atomic_load_seq_cst_i8() nounwind {
1494; CHECK-LABEL: test_atomic_load_seq_cst_i8:
1495; OUTLINE_ATOMICS-LABEL: test_atomic_load_seq_cst_i8:
1496; OUTLINE_ATOMICS:       // %bb.0:
1497; OUTLINE_ATOMICS-NEXT:    adrp x8, var8
1498; OUTLINE_ATOMICS-NEXT:    add x8, x8, :lo12:var8
1499; OUTLINE_ATOMICS-NEXT:    ldarb w0, [x8]
1500; OUTLINE_ATOMICS-NEXT:    ret
1501  %val = load atomic i8, i8* @var8 seq_cst, align 1
1502; CHECK-NOT: dmb
1503; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1504; CHECK-NOT: dmb
1505; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
1506; CHECK-NOT: dmb
1507; CHECK: ldarb w0, [x[[ADDR]]]
1508; CHECK-NOT: dmb
1509  ret i8 %val
1510}
1511
1512define i16 @test_atomic_load_monotonic_i16() nounwind {
1513; CHECK-LABEL: test_atomic_load_monotonic_i16:
1514; OUTLINE_ATOMICS-LABEL: test_atomic_load_monotonic_i16:
1515; OUTLINE_ATOMICS:       // %bb.0:
1516; OUTLINE_ATOMICS-NEXT:    adrp x8, var16
1517; OUTLINE_ATOMICS-NEXT:    ldrh w0, [x8, :lo12:var16]
1518; OUTLINE_ATOMICS-NEXT:    ret
1519  %val = load atomic i16, i16* @var16 monotonic, align 2
1520; CHECK-NOT: dmb
1521; CHECK: adrp x[[HIADDR:[0-9]+]], var16
1522; CHECK-NOT: dmb
1523; CHECK: ldrh w0, [x[[HIADDR]], {{#?}}:lo12:var16]
1524; CHECK-NOT: dmb
1525
1526  ret i16 %val
1527}
1528
1529define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
1530; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32:
1531; OUTLINE_ATOMICS-LABEL: test_atomic_load_monotonic_regoff_i32:
1532; OUTLINE_ATOMICS:       // %bb.0:
1533; OUTLINE_ATOMICS-NEXT:    ldr w0, [x0, x1]
1534; OUTLINE_ATOMICS-NEXT:    ret
1535  %addr_int = add i64 %base, %off
1536  %addr = inttoptr i64 %addr_int to i32*
1537
1538  %val = load atomic i32, i32* %addr monotonic, align 4
1539; CHECK-NOT: dmb
1540; CHECK: ldr w0, [x0, x1]
1541; CHECK-NOT: dmb
1542
1543  ret i32 %val
1544}
1545
1546define i64 @test_atomic_load_seq_cst_i64() nounwind {
1547; CHECK-LABEL: test_atomic_load_seq_cst_i64:
1548; OUTLINE_ATOMICS-LABEL: test_atomic_load_seq_cst_i64:
1549; OUTLINE_ATOMICS:       // %bb.0:
1550; OUTLINE_ATOMICS-NEXT:    adrp x8, var64
1551; OUTLINE_ATOMICS-NEXT:    add x8, x8, :lo12:var64
1552; OUTLINE_ATOMICS-NEXT:    ldar x0, [x8]
1553; OUTLINE_ATOMICS-NEXT:    ret
1554  %val = load atomic i64, i64* @var64 seq_cst, align 8
1555; CHECK-NOT: dmb
1556; CHECK: adrp [[HIADDR:x[0-9]+]], var64
1557; CHECK-NOT: dmb
1558; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var64
1559; CHECK-NOT: dmb
1560; CHECK: ldar x0, [x[[ADDR]]]
1561; CHECK-NOT: dmb
1562  ret i64 %val
1563}
1564
1565define void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
1566; CHECK-LABEL: test_atomic_store_monotonic_i8:
1567; OUTLINE_ATOMICS-LABEL: test_atomic_store_monotonic_i8:
1568; OUTLINE_ATOMICS:       // %bb.0:
1569; OUTLINE_ATOMICS-NEXT:    adrp x8, var8
1570; OUTLINE_ATOMICS-NEXT:    strb w0, [x8, :lo12:var8]
1571; OUTLINE_ATOMICS-NEXT:    ret
1572  store atomic i8 %val, i8* @var8 monotonic, align 1
1573; CHECK: adrp x[[HIADDR:[0-9]+]], var8
1574; CHECK: strb w0, [x[[HIADDR]], {{#?}}:lo12:var8]
1575
1576  ret void
1577}
1578
1579define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
1580; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8:
1581; OUTLINE_ATOMICS-LABEL: test_atomic_store_monotonic_regoff_i8:
1582; OUTLINE_ATOMICS:       // %bb.0:
1583; OUTLINE_ATOMICS-NEXT:    strb w2, [x0, x1]
1584; OUTLINE_ATOMICS-NEXT:    ret
1585  %addr_int = add i64 %base, %off
1586  %addr = inttoptr i64 %addr_int to i8*
1587
1588  store atomic i8 %val, i8* %addr monotonic, align 1
1589; CHECK: strb w2, [x0, x1]
1590
1591  ret void
1592}
1593define void @test_atomic_store_release_i8(i8 %val) nounwind {
1594; CHECK-LABEL: test_atomic_store_release_i8:
1595; OUTLINE_ATOMICS-LABEL: test_atomic_store_release_i8:
1596; OUTLINE_ATOMICS:       // %bb.0:
1597; OUTLINE_ATOMICS-NEXT:    adrp x8, var8
1598; OUTLINE_ATOMICS-NEXT:    add x8, x8, :lo12:var8
1599; OUTLINE_ATOMICS-NEXT:    stlrb w0, [x8]
1600; OUTLINE_ATOMICS-NEXT:    ret
1601  store atomic i8 %val, i8* @var8 release, align 1
1602; CHECK-NOT: dmb
1603; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1604; CHECK-NOT: dmb
1605; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
1606; CHECK-NOT: dmb
1607; CHECK: stlrb w0, [x[[ADDR]]]
1608; CHECK-NOT: dmb
1609  ret void
1610}
1611
1612define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
1613; CHECK-LABEL: test_atomic_store_seq_cst_i8:
1614; OUTLINE_ATOMICS-LABEL: test_atomic_store_seq_cst_i8:
1615; OUTLINE_ATOMICS:       // %bb.0:
1616; OUTLINE_ATOMICS-NEXT:    adrp x8, var8
1617; OUTLINE_ATOMICS-NEXT:    add x8, x8, :lo12:var8
1618; OUTLINE_ATOMICS-NEXT:    stlrb w0, [x8]
1619; OUTLINE_ATOMICS-NEXT:    ret
1620  store atomic i8 %val, i8* @var8 seq_cst, align 1
1621; CHECK-NOT: dmb
1622; CHECK: adrp [[HIADDR:x[0-9]+]], var8
1623; CHECK-NOT: dmb
1624; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var8
1625; CHECK-NOT: dmb
1626; CHECK: stlrb w0, [x[[ADDR]]]
1627; CHECK-NOT: dmb
1628
1629  ret void
1630}
1631
1632define void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
1633; CHECK-LABEL: test_atomic_store_monotonic_i16:
1634; OUTLINE_ATOMICS-LABEL: test_atomic_store_monotonic_i16:
1635; OUTLINE_ATOMICS:       // %bb.0:
1636; OUTLINE_ATOMICS-NEXT:    adrp x8, var16
1637; OUTLINE_ATOMICS-NEXT:    strh w0, [x8, :lo12:var16]
1638; OUTLINE_ATOMICS-NEXT:    ret
1639  store atomic i16 %val, i16* @var16 monotonic, align 2
1640; CHECK-NOT: dmb
1641; CHECK: adrp x[[HIADDR:[0-9]+]], var16
1642; CHECK-NOT: dmb
1643; CHECK: strh w0, [x[[HIADDR]], {{#?}}:lo12:var16]
1644; CHECK-NOT: dmb
1645  ret void
1646}
1647
1648define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
1649; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32:
1650; OUTLINE_ATOMICS-LABEL: test_atomic_store_monotonic_regoff_i32:
1651; OUTLINE_ATOMICS:       // %bb.0:
1652; OUTLINE_ATOMICS-NEXT:    str w2, [x0, x1]
1653; OUTLINE_ATOMICS-NEXT:    ret
1654  %addr_int = add i64 %base, %off
1655  %addr = inttoptr i64 %addr_int to i32*
1656
1657  store atomic i32 %val, i32* %addr monotonic, align 4
1658; CHECK-NOT: dmb
1659; CHECK: str w2, [x0, x1]
1660; CHECK-NOT: dmb
1661
1662  ret void
1663}
1664
1665define void @test_atomic_store_release_i64(i64 %val) nounwind {
1666; CHECK-LABEL: test_atomic_store_release_i64:
1667; OUTLINE_ATOMICS-LABEL: test_atomic_store_release_i64:
1668; OUTLINE_ATOMICS:       // %bb.0:
1669; OUTLINE_ATOMICS-NEXT:    adrp x8, var64
1670; OUTLINE_ATOMICS-NEXT:    add x8, x8, :lo12:var64
1671; OUTLINE_ATOMICS-NEXT:    stlr x0, [x8]
1672; OUTLINE_ATOMICS-NEXT:    ret
1673  store atomic i64 %val, i64* @var64 release, align 8
1674; CHECK-NOT: dmb
1675; CHECK: adrp [[HIADDR:x[0-9]+]], var64
1676; CHECK-NOT: dmb
1677; CHECK: add x[[ADDR:[0-9]+]], [[HIADDR]], {{#?}}:lo12:var64
1678; CHECK-NOT: dmb
1679; CHECK: stlr x0, [x[[ADDR]]]
1680; CHECK-NOT: dmb
1681  ret void
1682}
1683