1; RUN: llc < %s -march=arm64 -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone | FileCheck %s 2 3@var = global i128 0 4 5define i128 @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) { 6; CHECK-LABEL: val_compare_and_swap: 7; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 8; CHECK: ldaxp [[RESULTLO:x[0-9]+]], [[RESULTHI:x[0-9]+]], [x[[ADDR:[0-9]+]]] 9; CHECK-DAG: eor [[MISMATCH_LO:x[0-9]+]], [[RESULTLO]], x2 10; CHECK-DAG: eor [[MISMATCH_HI:x[0-9]+]], [[RESULTHI]], x3 11; CHECK: orr [[MISMATCH:x[0-9]+]], [[MISMATCH_LO]], [[MISMATCH_HI]] 12; CHECK: cbnz [[MISMATCH]], [[DONE:.LBB[0-9]+_[0-9]+]] 13; CHECK: stxp [[SCRATCH_RES:w[0-9]+]], x4, x5, [x[[ADDR]]] 14; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 15; CHECK: [[DONE]]: 16 %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire 17 %val = extractvalue { i128, i1 } %pair, 0 18 ret i128 %val 19} 20 21define void @fetch_and_nand(i128* %p, i128 %bits) { 22; CHECK-LABEL: fetch_and_nand: 23; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 24; CHECK: ldxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 25; CHECK-DAG: and [[TMP_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 26; CHECK-DAG: and [[TMP_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 27; CHECK-DAG: mvn [[SCRATCH_REGLO:x[0-9]+]], [[TMP_REGLO]] 28; CHECK-DAG: mvn [[SCRATCH_REGHI:x[0-9]+]], [[TMP_REGHI]] 29; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 30; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 31 32; CHECK-DAG: str [[DEST_REGHI]] 33; CHECK-DAG: str [[DEST_REGLO]] 34 %val = atomicrmw nand i128* %p, i128 %bits release 35 store i128 %val, i128* @var, align 16 36 ret void 37} 38 39define void @fetch_and_or(i128* %p, i128 %bits) { 40; CHECK-LABEL: fetch_and_or: 41; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 42; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 43; CHECK-DAG: orr [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 44; CHECK-DAG: orr [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 45; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 46; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 47 48; CHECK-DAG: str [[DEST_REGHI]] 49; CHECK-DAG: str [[DEST_REGLO]] 50 %val = atomicrmw or i128* %p, i128 %bits seq_cst 51 store i128 %val, i128* @var, align 16 52 ret void 53} 54 55define void @fetch_and_add(i128* %p, i128 %bits) { 56; CHECK-LABEL: fetch_and_add: 57; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 58; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 59; CHECK: adds [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 60; CHECK: adcs [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 61; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 62; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 63 64; CHECK-DAG: str [[DEST_REGHI]] 65; CHECK-DAG: str [[DEST_REGLO]] 66 %val = atomicrmw add i128* %p, i128 %bits seq_cst 67 store i128 %val, i128* @var, align 16 68 ret void 69} 70 71define void @fetch_and_sub(i128* %p, i128 %bits) { 72; CHECK-LABEL: fetch_and_sub: 73; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 74; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 75; CHECK: subs [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2 76; CHECK: sbcs [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3 77; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 78; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 79 80; CHECK-DAG: str [[DEST_REGHI]] 81; CHECK-DAG: str [[DEST_REGLO]] 82 %val = atomicrmw sub i128* %p, i128 %bits seq_cst 83 store i128 %val, i128* @var, align 16 84 ret void 85} 86 87define void @fetch_and_min(i128* %p, i128 %bits) { 88; CHECK-LABEL: fetch_and_min: 89; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 90; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 91; CHECK: cmp [[DEST_REGLO]], x2 92; CHECK: cset [[LOCMP:w[0-9]+]], ls 93; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 94; CHECK: cset [[HICMP:w[0-9]+]], le 95; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq 96; CHECK: cmp [[CMP]], #0 97; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne 98; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne 99; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 100; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 101 102; CHECK-DAG: str [[DEST_REGHI]] 103; CHECK-DAG: str [[DEST_REGLO]] 104 %val = atomicrmw min i128* %p, i128 %bits seq_cst 105 store i128 %val, i128* @var, align 16 106 ret void 107} 108 109define void @fetch_and_max(i128* %p, i128 %bits) { 110; CHECK-LABEL: fetch_and_max: 111; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 112; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 113; CHECK: cmp [[DEST_REGLO]], x2 114; CHECK: cset [[LOCMP:w[0-9]+]], hi 115; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 116; CHECK: cset [[HICMP:w[0-9]+]], gt 117; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq 118; CHECK: cmp [[CMP]], #0 119; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne 120; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne 121; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 122; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 123 124; CHECK-DAG: str [[DEST_REGHI]] 125; CHECK-DAG: str [[DEST_REGLO]] 126 %val = atomicrmw max i128* %p, i128 %bits seq_cst 127 store i128 %val, i128* @var, align 16 128 ret void 129} 130 131define void @fetch_and_umin(i128* %p, i128 %bits) { 132; CHECK-LABEL: fetch_and_umin: 133; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 134; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 135; CHECK: cmp [[DEST_REGLO]], x2 136; CHECK: cset [[LOCMP:w[0-9]+]], ls 137; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 138; CHECK: cset [[HICMP:w[0-9]+]], ls 139; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq 140; CHECK: cmp [[CMP]], #0 141; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne 142; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne 143; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 144; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 145 146; CHECK-DAG: str [[DEST_REGHI]] 147; CHECK-DAG: str [[DEST_REGLO]] 148 %val = atomicrmw umin i128* %p, i128 %bits seq_cst 149 store i128 %val, i128* @var, align 16 150 ret void 151} 152 153define void @fetch_and_umax(i128* %p, i128 %bits) { 154; CHECK-LABEL: fetch_and_umax: 155; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 156; CHECK: ldaxp [[DEST_REGLO:x[0-9]+]], [[DEST_REGHI:x[0-9]+]], [x0] 157; CHECK: cmp [[DEST_REGLO]], x2 158; CHECK: cset [[LOCMP:w[0-9]+]], hi 159; CHECK: cmp [[DEST_REGHI:x[0-9]+]], x3 160; CHECK: cset [[HICMP:w[0-9]+]], hi 161; CHECK: csel [[CMP:w[0-9]+]], [[LOCMP]], [[HICMP]], eq 162; CHECK: cmp [[CMP]], #0 163; CHECK-DAG: csel [[SCRATCH_REGHI:x[0-9]+]], [[DEST_REGHI]], x3, ne 164; CHECK-DAG: csel [[SCRATCH_REGLO:x[0-9]+]], [[DEST_REGLO]], x2, ne 165; CHECK: stlxp [[SCRATCH_RES:w[0-9]+]], [[SCRATCH_REGLO]], [[SCRATCH_REGHI]], [x0] 166; CHECK: cbnz [[SCRATCH_RES]], [[LABEL]] 167 168; CHECK-DAG: str [[DEST_REGHI]] 169; CHECK-DAG: str [[DEST_REGLO]] 170 %val = atomicrmw umax i128* %p, i128 %bits seq_cst 171 store i128 %val, i128* @var, align 16 172 ret void 173} 174 175define i128 @atomic_load_seq_cst(i128* %p) { 176; CHECK-LABEL: atomic_load_seq_cst: 177; CHECK-NOT: dmb 178; CHECK-LABEL: ldaxp 179; CHECK-NOT: dmb 180 %r = load atomic i128* %p seq_cst, align 16 181 ret i128 %r 182} 183 184define i128 @atomic_load_relaxed(i128* %p) { 185; CHECK-LABEL: atomic_load_relaxed: 186; CHECK-NOT: dmb 187; CHECK: ldxp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0] 188; CHECK-NOT: dmb 189 %r = load atomic i128* %p monotonic, align 16 190 ret i128 %r 191} 192 193 194define void @atomic_store_seq_cst(i128 %in, i128* %p) { 195; CHECK-LABEL: atomic_store_seq_cst: 196; CHECK-NOT: dmb 197; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 198; CHECK: ldaxp xzr, xzr, [x2] 199; CHECK: stlxp [[SUCCESS:w[0-9]+]], x0, x1, [x2] 200; CHECK: cbnz [[SUCCESS]], [[LABEL]] 201; CHECK-NOT: dmb 202 store atomic i128 %in, i128* %p seq_cst, align 16 203 ret void 204} 205 206define void @atomic_store_release(i128 %in, i128* %p) { 207; CHECK-LABEL: atomic_store_release: 208; CHECK-NOT: dmb 209; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 210; CHECK: ldxp xzr, xzr, [x2] 211; CHECK: stlxp [[SUCCESS:w[0-9]+]], x0, x1, [x2] 212; CHECK: cbnz [[SUCCESS]], [[LABEL]] 213; CHECK-NOT: dmb 214 store atomic i128 %in, i128* %p release, align 16 215 ret void 216} 217 218define void @atomic_store_relaxed(i128 %in, i128* %p) { 219; CHECK-LABEL: atomic_store_relaxed: 220; CHECK-NOT: dmb 221; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]: 222; CHECK: ldxp xzr, xzr, [x2] 223; CHECK: stxp [[SUCCESS:w[0-9]+]], x0, x1, [x2] 224; CHECK: cbnz [[SUCCESS]], [[LABEL]] 225; CHECK-NOT: dmb 226 store atomic i128 %in, i128* %p unordered, align 16 227 ret void 228} 229