1; RUN: llc -O3 -mtriple=thumb-eabi -mcpu=cortex-a9 %s -o - | FileCheck %s -check-prefix=A9 2; RUN: llc -O3 -mtriple=thumb-eabi -mcpu=cortex-a9 -addr-sink-using-gep=1 %s -o - | FileCheck %s -check-prefix=A9 3 4; @simple is the most basic chain of address induction variables. Chaining 5; saves at least one register and avoids complex addressing and setup 6; code. 7; 8; A9: @simple 9; no expensive address computation in the preheader 10; A9: lsl 11; A9-NOT: lsl 12; A9: %loop 13; no complex address modes 14; A9-NOT: lsl 15define i32 @simple(i32* %a, i32* %b, i32 %x) nounwind { 16entry: 17 br label %loop 18loop: 19 %iv = phi i32* [ %a, %entry ], [ %iv4, %loop ] 20 %s = phi i32 [ 0, %entry ], [ %s4, %loop ] 21 %v = load i32, i32* %iv 22 %iv1 = getelementptr inbounds i32, i32* %iv, i32 %x 23 %v1 = load i32, i32* %iv1 24 %iv2 = getelementptr inbounds i32, i32* %iv1, i32 %x 25 %v2 = load i32, i32* %iv2 26 %iv3 = getelementptr inbounds i32, i32* %iv2, i32 %x 27 %v3 = load i32, i32* %iv3 28 %s1 = add i32 %s, %v 29 %s2 = add i32 %s1, %v1 30 %s3 = add i32 %s2, %v2 31 %s4 = add i32 %s3, %v3 32 %iv4 = getelementptr inbounds i32, i32* %iv3, i32 %x 33 %cmp = icmp eq i32* %iv4, %b 34 br i1 %cmp, label %exit, label %loop 35exit: 36 ret i32 %s4 37} 38 39; @user is not currently chained because the IV is live across memory ops. 40; 41; A9: @user 42; stride multiples computed in the preheader 43; A9: lsl 44; A9: lsl 45; A9: %loop 46; complex address modes 47; A9: lsl 48; A9: lsl 49define i32 @user(i32* %a, i32* %b, i32 %x) nounwind { 50entry: 51 br label %loop 52loop: 53 %iv = phi i32* [ %a, %entry ], [ %iv4, %loop ] 54 %s = phi i32 [ 0, %entry ], [ %s4, %loop ] 55 %v = load i32, i32* %iv 56 %iv1 = getelementptr inbounds i32, i32* %iv, i32 %x 57 %v1 = load i32, i32* %iv1 58 %iv2 = getelementptr inbounds i32, i32* %iv1, i32 %x 59 %v2 = load i32, i32* %iv2 60 %iv3 = getelementptr inbounds i32, i32* %iv2, i32 %x 61 %v3 = load i32, i32* %iv3 62 %s1 = add i32 %s, %v 63 %s2 = add i32 %s1, %v1 64 %s3 = add i32 %s2, %v2 65 %s4 = add i32 %s3, %v3 66 %iv4 = getelementptr inbounds i32, i32* %iv3, i32 %x 67 store i32 %s4, i32* %iv 68 %cmp = icmp eq i32* %iv4, %b 69 br i1 %cmp, label %exit, label %loop 70exit: 71 ret i32 %s4 72} 73 74; @extrastride is a slightly more interesting case of a single 75; complete chain with multiple strides. The test case IR is what LSR 76; used to do, and exactly what we don't want to do. LSR's new IV 77; chaining feature should now undo the damage. 78; 79; A9: extrastride: 80; no spills 81; A9-NOT: str 82; only one stride multiple in the preheader 83; A9: lsl 84; A9-NOT: {{str r|lsl}} 85; A9: %for.body{{$}} 86; no complex address modes or reloads 87; A9-NOT: {{ldr .*[sp]|lsl}} 88define void @extrastride(i8* nocapture %main, i32 %main_stride, i32* nocapture %res, i32 %x, i32 %y, i32 %z) nounwind { 89entry: 90 %cmp8 = icmp eq i32 %z, 0 91 br i1 %cmp8, label %for.end, label %for.body.lr.ph 92 93for.body.lr.ph: ; preds = %entry 94 %add.ptr.sum = shl i32 %main_stride, 1 ; s*2 95 %add.ptr1.sum = add i32 %add.ptr.sum, %main_stride ; s*3 96 %add.ptr2.sum = add i32 %x, %main_stride ; s + x 97 %add.ptr4.sum = shl i32 %main_stride, 2 ; s*4 98 %add.ptr3.sum = add i32 %add.ptr2.sum, %add.ptr4.sum ; total IV stride = s*5+x 99 br label %for.body 100 101for.body: ; preds = %for.body.lr.ph, %for.body 102 %main.addr.011 = phi i8* [ %main, %for.body.lr.ph ], [ %add.ptr6, %for.body ] 103 %i.010 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ] 104 %res.addr.09 = phi i32* [ %res, %for.body.lr.ph ], [ %add.ptr7, %for.body ] 105 %0 = bitcast i8* %main.addr.011 to i32* 106 %1 = load i32, i32* %0, align 4 107 %add.ptr = getelementptr inbounds i8, i8* %main.addr.011, i32 %main_stride 108 %2 = bitcast i8* %add.ptr to i32* 109 %3 = load i32, i32* %2, align 4 110 %add.ptr1 = getelementptr inbounds i8, i8* %main.addr.011, i32 %add.ptr.sum 111 %4 = bitcast i8* %add.ptr1 to i32* 112 %5 = load i32, i32* %4, align 4 113 %add.ptr2 = getelementptr inbounds i8, i8* %main.addr.011, i32 %add.ptr1.sum 114 %6 = bitcast i8* %add.ptr2 to i32* 115 %7 = load i32, i32* %6, align 4 116 %add.ptr3 = getelementptr inbounds i8, i8* %main.addr.011, i32 %add.ptr4.sum 117 %8 = bitcast i8* %add.ptr3 to i32* 118 %9 = load i32, i32* %8, align 4 119 %add = add i32 %3, %1 120 %add4 = add i32 %add, %5 121 %add5 = add i32 %add4, %7 122 %add6 = add i32 %add5, %9 123 store i32 %add6, i32* %res.addr.09, align 4 124 %add.ptr6 = getelementptr inbounds i8, i8* %main.addr.011, i32 %add.ptr3.sum 125 %add.ptr7 = getelementptr inbounds i32, i32* %res.addr.09, i32 %y 126 %inc = add i32 %i.010, 1 127 %cmp = icmp eq i32 %inc, %z 128 br i1 %cmp, label %for.end, label %for.body 129 130for.end: ; preds = %for.body, %entry 131 ret void 132} 133 134; @foldedidx is an unrolled variant of this loop: 135; for (unsigned long i = 0; i < len; i += s) { 136; c[i] = a[i] + b[i]; 137; } 138; where 's' can be folded into the addressing mode. 139; Consequently, we should *not* form any chains. 140; 141; A9: foldedidx: 142; A9: ldrb{{(.w)?}} {{r[0-9]|lr}}, [{{r[0-9]|lr}}, #3] 143define void @foldedidx(i8* nocapture %a, i8* nocapture %b, i8* nocapture %c) nounwind ssp { 144entry: 145 br label %for.body 146 147for.body: ; preds = %for.body, %entry 148 %i.07 = phi i32 [ 0, %entry ], [ %inc.3, %for.body ] 149 %arrayidx = getelementptr inbounds i8, i8* %a, i32 %i.07 150 %0 = load i8, i8* %arrayidx, align 1 151 %conv5 = zext i8 %0 to i32 152 %arrayidx1 = getelementptr inbounds i8, i8* %b, i32 %i.07 153 %1 = load i8, i8* %arrayidx1, align 1 154 %conv26 = zext i8 %1 to i32 155 %add = add nsw i32 %conv26, %conv5 156 %conv3 = trunc i32 %add to i8 157 %arrayidx4 = getelementptr inbounds i8, i8* %c, i32 %i.07 158 store i8 %conv3, i8* %arrayidx4, align 1 159 %inc1 = or i32 %i.07, 1 160 %arrayidx.1 = getelementptr inbounds i8, i8* %a, i32 %inc1 161 %2 = load i8, i8* %arrayidx.1, align 1 162 %conv5.1 = zext i8 %2 to i32 163 %arrayidx1.1 = getelementptr inbounds i8, i8* %b, i32 %inc1 164 %3 = load i8, i8* %arrayidx1.1, align 1 165 %conv26.1 = zext i8 %3 to i32 166 %add.1 = add nsw i32 %conv26.1, %conv5.1 167 %conv3.1 = trunc i32 %add.1 to i8 168 %arrayidx4.1 = getelementptr inbounds i8, i8* %c, i32 %inc1 169 store i8 %conv3.1, i8* %arrayidx4.1, align 1 170 %inc.12 = or i32 %i.07, 2 171 %arrayidx.2 = getelementptr inbounds i8, i8* %a, i32 %inc.12 172 %4 = load i8, i8* %arrayidx.2, align 1 173 %conv5.2 = zext i8 %4 to i32 174 %arrayidx1.2 = getelementptr inbounds i8, i8* %b, i32 %inc.12 175 %5 = load i8, i8* %arrayidx1.2, align 1 176 %conv26.2 = zext i8 %5 to i32 177 %add.2 = add nsw i32 %conv26.2, %conv5.2 178 %conv3.2 = trunc i32 %add.2 to i8 179 %arrayidx4.2 = getelementptr inbounds i8, i8* %c, i32 %inc.12 180 store i8 %conv3.2, i8* %arrayidx4.2, align 1 181 %inc.23 = or i32 %i.07, 3 182 %arrayidx.3 = getelementptr inbounds i8, i8* %a, i32 %inc.23 183 %6 = load i8, i8* %arrayidx.3, align 1 184 %conv5.3 = zext i8 %6 to i32 185 %arrayidx1.3 = getelementptr inbounds i8, i8* %b, i32 %inc.23 186 %7 = load i8, i8* %arrayidx1.3, align 1 187 %conv26.3 = zext i8 %7 to i32 188 %add.3 = add nsw i32 %conv26.3, %conv5.3 189 %conv3.3 = trunc i32 %add.3 to i8 190 %arrayidx4.3 = getelementptr inbounds i8, i8* %c, i32 %inc.23 191 store i8 %conv3.3, i8* %arrayidx4.3, align 1 192 %inc.3 = add nsw i32 %i.07, 4 193 %exitcond.3 = icmp eq i32 %inc.3, 400 194 br i1 %exitcond.3, label %for.end, label %for.body 195 196for.end: ; preds = %for.body 197 ret void 198} 199 200; @testNeon is an important example of the nead for ivchains. 201; 202; Currently we have three extra add.w's that keep the store address 203; live past the next increment because ISEL is unfortunately undoing 204; the store chain. ISEL also fails to convert all but one of the stores to 205; post-increment addressing. However, the loads should use 206; post-increment addressing, no add's or add.w's beyond the three 207; mentioned. Most importantly, there should be no spills or reloads! 208; 209; A9: testNeon: 210; A9: %.lr.ph 211; A9-NOT: lsl.w 212; A9-NOT: {{ldr|str|adds|add r}} 213; A9: vst1.8 {{.*}} [r{{[0-9]+}}]! 214; A9-NOT: {{ldr|str|adds|add r}} 215; A9: add.w r 216; A9-NOT: {{ldr|str|adds|add r}} 217; A9: add.w r 218; A9-NOT: {{ldr|str|adds|add r}} 219; A9-NOT: add.w r 220; A9: bne 221define hidden void @testNeon(i8* %ref_data, i32 %ref_stride, i32 %limit, <16 x i8>* nocapture %data) nounwind optsize { 222 %1 = icmp sgt i32 %limit, 0 223 br i1 %1, label %.lr.ph, label %45 224 225.lr.ph: ; preds = %0 226 %2 = shl nsw i32 %ref_stride, 1 227 %3 = mul nsw i32 %ref_stride, 3 228 %4 = shl nsw i32 %ref_stride, 2 229 %5 = mul nsw i32 %ref_stride, 5 230 %6 = mul nsw i32 %ref_stride, 6 231 %7 = mul nsw i32 %ref_stride, 7 232 %8 = shl nsw i32 %ref_stride, 3 233 %9 = sub i32 0, %8 234 %10 = mul i32 %limit, -64 235 br label %11 236 237; <label>:11 ; preds = %11, %.lr.ph 238 %.05 = phi i8* [ %ref_data, %.lr.ph ], [ %42, %11 ] 239 %counter.04 = phi i32 [ 0, %.lr.ph ], [ %44, %11 ] 240 %result.03 = phi <16 x i8> [ zeroinitializer, %.lr.ph ], [ %41, %11 ] 241 %.012 = phi <16 x i8>* [ %data, %.lr.ph ], [ %43, %11 ] 242 %12 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %.05, i32 1) nounwind 243 %13 = getelementptr inbounds i8, i8* %.05, i32 %ref_stride 244 %14 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %13, i32 1) nounwind 245 %15 = shufflevector <1 x i64> %12, <1 x i64> %14, <2 x i32> <i32 0, i32 1> 246 %16 = bitcast <2 x i64> %15 to <16 x i8> 247 %17 = getelementptr inbounds <16 x i8>, <16 x i8>* %.012, i32 1 248 store <16 x i8> %16, <16 x i8>* %.012, align 4 249 %18 = getelementptr inbounds i8, i8* %.05, i32 %2 250 %19 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %18, i32 1) nounwind 251 %20 = getelementptr inbounds i8, i8* %.05, i32 %3 252 %21 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %20, i32 1) nounwind 253 %22 = shufflevector <1 x i64> %19, <1 x i64> %21, <2 x i32> <i32 0, i32 1> 254 %23 = bitcast <2 x i64> %22 to <16 x i8> 255 %24 = getelementptr inbounds <16 x i8>, <16 x i8>* %.012, i32 2 256 store <16 x i8> %23, <16 x i8>* %17, align 4 257 %25 = getelementptr inbounds i8, i8* %.05, i32 %4 258 %26 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %25, i32 1) nounwind 259 %27 = getelementptr inbounds i8, i8* %.05, i32 %5 260 %28 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %27, i32 1) nounwind 261 %29 = shufflevector <1 x i64> %26, <1 x i64> %28, <2 x i32> <i32 0, i32 1> 262 %30 = bitcast <2 x i64> %29 to <16 x i8> 263 %31 = getelementptr inbounds <16 x i8>, <16 x i8>* %.012, i32 3 264 store <16 x i8> %30, <16 x i8>* %24, align 4 265 %32 = getelementptr inbounds i8, i8* %.05, i32 %6 266 %33 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %32, i32 1) nounwind 267 %34 = getelementptr inbounds i8, i8* %.05, i32 %7 268 %35 = tail call <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8* %34, i32 1) nounwind 269 %36 = shufflevector <1 x i64> %33, <1 x i64> %35, <2 x i32> <i32 0, i32 1> 270 %37 = bitcast <2 x i64> %36 to <16 x i8> 271 store <16 x i8> %37, <16 x i8>* %31, align 4 272 %38 = add <16 x i8> %16, %23 273 %39 = add <16 x i8> %38, %30 274 %40 = add <16 x i8> %39, %37 275 %41 = add <16 x i8> %result.03, %40 276 %42 = getelementptr i8, i8* %.05, i32 %9 277 %43 = getelementptr inbounds <16 x i8>, <16 x i8>* %.012, i32 -64 278 %44 = add nsw i32 %counter.04, 1 279 %exitcond = icmp eq i32 %44, %limit 280 br i1 %exitcond, label %._crit_edge, label %11 281 282._crit_edge: ; preds = %11 283 %scevgep = getelementptr <16 x i8>, <16 x i8>* %data, i32 %10 284 br label %45 285 286; <label>:45 ; preds = %._crit_edge, %0 287 %result.0.lcssa = phi <16 x i8> [ %41, %._crit_edge ], [ zeroinitializer, %0 ] 288 %.01.lcssa = phi <16 x i8>* [ %scevgep, %._crit_edge ], [ %data, %0 ] 289 store <16 x i8> %result.0.lcssa, <16 x i8>* %.01.lcssa, align 4 290 ret void 291} 292 293declare <1 x i64> @llvm.arm.neon.vld1.v1i64.p0i8(i8*, i32) nounwind readonly 294 295; Handle chains in which the same offset is used for both loads and 296; stores to the same array. 297; rdar://11410078. 298; 299; A9: @testReuse 300; A9: %for.body 301; A9: vld1.8 {d{{[0-9]+}}}, [[BASE:[r[0-9]+]]], [[INC:r[0-9]]] 302; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]] 303; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]] 304; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]] 305; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]] 306; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]] 307; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]] 308; A9: vld1.8 {d{{[0-9]+}}}, [[BASE]], {{r[0-9]}} 309; A9: vst1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]] 310; A9: vst1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]] 311; A9: vst1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]] 312; A9: vst1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]] 313; A9: vst1.8 {d{{[0-9]+}}}, [[BASE]], [[INC]] 314; A9: vst1.8 {d{{[0-9]+}}}, [[BASE]] 315; A9: bne 316define void @testReuse(i8* %src, i32 %stride) nounwind ssp { 317entry: 318 %mul = shl nsw i32 %stride, 2 319 %idx.neg = sub i32 0, %mul 320 %mul1 = mul nsw i32 %stride, 3 321 %idx.neg2 = sub i32 0, %mul1 322 %mul5 = shl nsw i32 %stride, 1 323 %idx.neg6 = sub i32 0, %mul5 324 %idx.neg10 = sub i32 0, %stride 325 br label %for.body 326 327for.body: ; preds = %for.body, %entry 328 %i.0110 = phi i32 [ 0, %entry ], [ %inc, %for.body ] 329 %src.addr = phi i8* [ %src, %entry ], [ %add.ptr45, %for.body ] 330 %add.ptr = getelementptr inbounds i8, i8* %src.addr, i32 %idx.neg 331 %vld1 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr, i32 1) 332 %add.ptr3 = getelementptr inbounds i8, i8* %src.addr, i32 %idx.neg2 333 %vld2 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr3, i32 1) 334 %add.ptr7 = getelementptr inbounds i8, i8* %src.addr, i32 %idx.neg6 335 %vld3 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr7, i32 1) 336 %add.ptr11 = getelementptr inbounds i8, i8* %src.addr, i32 %idx.neg10 337 %vld4 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr11, i32 1) 338 %vld5 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %src.addr, i32 1) 339 %add.ptr17 = getelementptr inbounds i8, i8* %src.addr, i32 %stride 340 %vld6 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr17, i32 1) 341 %add.ptr20 = getelementptr inbounds i8, i8* %src.addr, i32 %mul5 342 %vld7 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr20, i32 1) 343 %add.ptr23 = getelementptr inbounds i8, i8* %src.addr, i32 %mul1 344 %vld8 = tail call <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8* %add.ptr23, i32 1) 345 %vadd1 = tail call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %vld1, <8 x i8> %vld2) nounwind 346 %vadd2 = tail call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %vld2, <8 x i8> %vld3) nounwind 347 %vadd3 = tail call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %vld3, <8 x i8> %vld4) nounwind 348 %vadd4 = tail call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %vld4, <8 x i8> %vld5) nounwind 349 %vadd5 = tail call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %vld5, <8 x i8> %vld6) nounwind 350 %vadd6 = tail call <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8> %vld6, <8 x i8> %vld7) nounwind 351 tail call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %add.ptr3, <8 x i8> %vadd1, i32 1) 352 tail call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %add.ptr7, <8 x i8> %vadd2, i32 1) 353 tail call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %add.ptr11, <8 x i8> %vadd3, i32 1) 354 tail call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %src.addr, <8 x i8> %vadd4, i32 1) 355 tail call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %add.ptr17, <8 x i8> %vadd5, i32 1) 356 tail call void @llvm.arm.neon.vst1.p0i8.v8i8(i8* %add.ptr20, <8 x i8> %vadd6, i32 1) 357 %inc = add nsw i32 %i.0110, 1 358 %add.ptr45 = getelementptr inbounds i8, i8* %src.addr, i32 8 359 %exitcond = icmp eq i32 %inc, 4 360 br i1 %exitcond, label %for.end, label %for.body 361 362for.end: ; preds = %for.body 363 ret void 364} 365 366declare <8 x i8> @llvm.arm.neon.vld1.v8i8.p0i8(i8*, i32) nounwind readonly 367 368declare void @llvm.arm.neon.vst1.p0i8.v8i8(i8*, <8 x i8>, i32) nounwind 369 370declare <8 x i8> @llvm.arm.neon.vhaddu.v8i8(<8 x i8>, <8 x i8>) nounwind readnone 371