1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=0 < %s | FileCheck %s 3; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=1 < %s | FileCheck %s 4 5; Test for multiple uses of the mgather where the s/zext should not be combined 6 7define <vscale x 2 x i64> @masked_sgather_sext(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask, <vscale x 2 x i8> %vals) { 8; CHECK-LABEL: masked_sgather_sext: 9; CHECK: // %bb.0: 10; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0, z0.d] 11; CHECK-NEXT: ptrue p0.d 12; CHECK-NEXT: sxtb z2.d, p0/m, z0.d 13; CHECK-NEXT: add z0.d, z0.d, z1.d 14; CHECK-NEXT: sxtb z0.d, p0/m, z0.d 15; CHECK-NEXT: mul z0.d, p0/m, z0.d, z2.d 16; CHECK-NEXT: ret 17 %ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets 18 %data = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef) 19 %data.sext = sext <vscale x 2 x i8> %data to <vscale x 2 x i64> 20 %add = add <vscale x 2 x i8> %data, %vals 21 %add.sext = sext <vscale x 2 x i8> %add to <vscale x 2 x i64> 22 %mul = mul <vscale x 2 x i64> %data.sext, %add.sext 23 ret <vscale x 2 x i64> %mul 24} 25 26define <vscale x 2 x i64> @masked_sgather_zext(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask, <vscale x 2 x i8> %vals) { 27; CHECK-LABEL: masked_sgather_zext: 28; CHECK: // %bb.0: 29; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0, z0.d] 30; CHECK-NEXT: ptrue p0.d 31; CHECK-NEXT: add z1.d, z0.d, z1.d 32; CHECK-NEXT: and z0.d, z0.d, #0xff 33; CHECK-NEXT: and z1.d, z1.d, #0xff 34; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d 35; CHECK-NEXT: ret 36 %ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets 37 %data = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef) 38 %data.zext = zext <vscale x 2 x i8> %data to <vscale x 2 x i64> 39 %add = add <vscale x 2 x i8> %data, %vals 40 %add.zext = zext <vscale x 2 x i8> %add to <vscale x 2 x i64> 41 %mul = mul <vscale x 2 x i64> %data.zext, %add.zext 42 ret <vscale x 2 x i64> %mul 43} 44 45; Tests that exercise various type legalisation scenarios for ISD::MGATHER. 46 47; Code generate load of an illegal datatype via promotion. 48define <vscale x 2 x i32> @masked_gather_nxv2i32(<vscale x 2 x i32*> %ptrs, <vscale x 2 x i1> %mask) { 49; CHECK-LABEL: masked_gather_nxv2i32: 50; CHECK-DAG: mov x8, xzr 51; CHECK-DAG: ld1sw { z0.d }, p0/z, [x8, z0.d] 52; CHECK: ret 53 %data = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef) 54 ret <vscale x 2 x i32> %data 55} 56 57; Code generate the worst case scenario when all vector types are illegal. 58define <vscale x 32 x i32> @masked_gather_nxv32i32(i32* %base, <vscale x 32 x i32> %indices, <vscale x 32 x i1> %mask) { 59; CHECK-LABEL: masked_gather_nxv32i32: 60; CHECK-NOT: unpkhi 61; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z0.s, sxtw #2] 62; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z1.s, sxtw #2] 63; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z2.s, sxtw #2] 64; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z3.s, sxtw #2] 65; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z4.s, sxtw #2] 66; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z5.s, sxtw #2] 67; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z6.s, sxtw #2] 68; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z7.s, sxtw #2] 69; CHECK: ret 70 %ptrs = getelementptr i32, i32* %base, <vscale x 32 x i32> %indices 71 %data = call <vscale x 32 x i32> @llvm.masked.gather.nxv32i32(<vscale x 32 x i32*> %ptrs, i32 4, <vscale x 32 x i1> %mask, <vscale x 32 x i32> undef) 72 ret <vscale x 32 x i32> %data 73} 74 75; TODO: Currently, the sign extend gets applied to the values after a 'uzp1' of two 76; registers, so it doesn't get folded away. Same for any other vector-of-pointers 77; style gathers which don't fit in an <vscale x 2 x type*> single register. Better folding 78; is required before we can check those off. 79define <vscale x 4 x i32> @masked_sgather_nxv4i8(<vscale x 4 x i8*> %ptrs, <vscale x 4 x i1> %mask) { 80; CHECK-LABEL: masked_sgather_nxv4i8: 81; CHECK: pfalse p1.b 82; CHECK-NEXT: mov x8, xzr 83; CHECK-NEXT: zip2 p2.s, p0.s, p1.s 84; CHECK-NEXT: zip1 p0.s, p0.s, p1.s 85; CHECK-NEXT: ld1sb { z1.d }, p2/z, [x8, z1.d] 86; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x8, z0.d] 87; CHECK-NEXT: ptrue p0.s 88; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s 89; CHECK-NEXT: sxtb z0.s, p0/m, z0.s 90; CHECK-NEXT: ret 91 %vals = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> undef) 92 %svals = sext <vscale x 4 x i8> %vals to <vscale x 4 x i32> 93 ret <vscale x 4 x i32> %svals 94} 95 96declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>) 97declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>) 98declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>) 99 100declare <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x i8*>, i32, <vscale x 4 x i1>, <vscale x 4 x i8>) 101 102declare <vscale x 16 x i8> @llvm.masked.gather.nxv16i8(<vscale x 16 x i8*>, i32, <vscale x 16 x i1>, <vscale x 16 x i8>) 103declare <vscale x 32 x i32> @llvm.masked.gather.nxv32i32(<vscale x 32 x i32*>, i32, <vscale x 32 x i1>, <vscale x 32 x i32>) 104