1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt -lower-matrix-intrinsics -S < %s | FileCheck %s 3; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s 4 5; Check that we we use flattened vectors for PHI operands and extract the columns afterwards. 6define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B, <9 x double> %C) { 7; CHECK-LABEL: @unsupported_phi( 8; CHECK-NEXT: entry: 9; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] 10 11; CHECK-LABEL: if.then: 12; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <9 x double> [[A:%.*]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2> 13; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5> 14; CHECK-NEXT: [[SPLIT5:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8> 15; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 0 16; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x double> undef, double [[TMP0]], i64 0 17; CHECK-NEXT: [[TMP2:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 0 18; CHECK-NEXT: [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1 19; CHECK-NEXT: [[TMP4:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 0 20; CHECK-NEXT: [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2 21; CHECK-NEXT: [[TMP6:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 1 22; CHECK-NEXT: [[TMP7:%.*]] = insertelement <3 x double> undef, double [[TMP6]], i64 0 23; CHECK-NEXT: [[TMP8:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1 24; CHECK-NEXT: [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1 25; CHECK-NEXT: [[TMP10:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 1 26; CHECK-NEXT: [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2 27; CHECK-NEXT: [[TMP12:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2 28; CHECK-NEXT: [[TMP13:%.*]] = insertelement <3 x double> undef, double [[TMP12]], i64 0 29; CHECK-NEXT: [[TMP14:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2 30; CHECK-NEXT: [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1 31; CHECK-NEXT: [[TMP16:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 2 32; CHECK-NEXT: [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2 33; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> [[TMP11]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5> 34; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef> 35; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <6 x double> [[TMP18]], <6 x double> [[TMP19]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 36; CHECK-NEXT: br label [[IF_END:%.*]] 37 38; CHECK-LABEL: if.else: 39; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <9 x double> [[B:%.*]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2> 40; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5> 41; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8> 42; CHECK-NEXT: [[TMP21:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0 43; CHECK-NEXT: [[TMP22:%.*]] = insertelement <3 x double> undef, double [[TMP21]], i64 0 44; CHECK-NEXT: [[TMP23:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 0 45; CHECK-NEXT: [[TMP24:%.*]] = insertelement <3 x double> [[TMP22]], double [[TMP23]], i64 1 46; CHECK-NEXT: [[TMP25:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 0 47; CHECK-NEXT: [[TMP26:%.*]] = insertelement <3 x double> [[TMP24]], double [[TMP25]], i64 2 48; CHECK-NEXT: [[TMP27:%.*]] = extractelement <3 x double> [[SPLIT]], i64 1 49; CHECK-NEXT: [[TMP28:%.*]] = insertelement <3 x double> undef, double [[TMP27]], i64 0 50; CHECK-NEXT: [[TMP29:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 1 51; CHECK-NEXT: [[TMP30:%.*]] = insertelement <3 x double> [[TMP28]], double [[TMP29]], i64 1 52; CHECK-NEXT: [[TMP31:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 1 53; CHECK-NEXT: [[TMP32:%.*]] = insertelement <3 x double> [[TMP30]], double [[TMP31]], i64 2 54; CHECK-NEXT: [[TMP33:%.*]] = extractelement <3 x double> [[SPLIT]], i64 2 55; CHECK-NEXT: [[TMP34:%.*]] = insertelement <3 x double> undef, double [[TMP33]], i64 0 56; CHECK-NEXT: [[TMP35:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 2 57; CHECK-NEXT: [[TMP36:%.*]] = insertelement <3 x double> [[TMP34]], double [[TMP35]], i64 1 58; CHECK-NEXT: [[TMP37:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2 59; CHECK-NEXT: [[TMP38:%.*]] = insertelement <3 x double> [[TMP36]], double [[TMP37]], i64 2 60; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <3 x double> [[TMP26]], <3 x double> [[TMP32]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5> 61; CHECK-NEXT: [[TMP40:%.*]] = shufflevector <3 x double> [[TMP38]], <3 x double> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef> 62; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <6 x double> [[TMP39]], <6 x double> [[TMP40]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 63; CHECK-NEXT: br label [[IF_END]] 64 65; CHECK-LABEL: if.end: 66; CHECK-NEXT: [[MERGE:%.*]] = phi <9 x double> [ [[TMP20]], [[IF_THEN]] ], [ [[TMP41]], [[IF_ELSE]] ] 67; CHECK-NEXT: [[SPLIT6:%.*]] = shufflevector <9 x double> [[C:%.*]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2> 68; CHECK-NEXT: [[SPLIT7:%.*]] = shufflevector <9 x double> [[C]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5> 69; CHECK-NEXT: [[SPLIT8:%.*]] = shufflevector <9 x double> [[C]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8> 70; CHECK-NEXT: [[SPLIT9:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2> 71; CHECK-NEXT: [[SPLIT10:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5> 72; CHECK-NEXT: [[SPLIT11:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8> 73; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> undef, <1 x i32> zeroinitializer 74; 75entry: 76 br i1 %cond, label %if.then, label %if.else 77 78if.then: ; preds = %entry 79 %A.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %A, i32 3, i32 3) 80 br label %if.end 81 82if.else: ; preds = %entry 83 %B.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %B, i32 3, i32 3) 84 br label %if.end 85 86if.end: ; preds = %if.then, %if.else 87 %merge = phi <9 x double> [ %A.trans, %if.then], [ %B.trans, %if.else ] 88 %res = tail call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %C, <9 x double> %merge, i32 3, i32 3, i32 3) 89 ret <9 x double> %res 90} 91 92; Make sure we use a flattened vector when calling @foo and the use its flat vector result properly. 93define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B) { 94; CHECK-LABEL: @unsupported_call( 95; CHECK-NEXT: [[SPLIT:%.*]] = shufflevector <9 x double> [[A:%.*]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2> 96; CHECK-NEXT: [[SPLIT1:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5> 97; CHECK-NEXT: [[SPLIT2:%.*]] = shufflevector <9 x double> [[A]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8> 98; CHECK-NEXT: [[TMP1:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0 99; CHECK-NEXT: [[TMP2:%.*]] = insertelement <3 x double> undef, double [[TMP1]], i64 0 100; CHECK-NEXT: [[TMP3:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 0 101; CHECK-NEXT: [[TMP4:%.*]] = insertelement <3 x double> [[TMP2]], double [[TMP3]], i64 1 102; CHECK-NEXT: [[TMP5:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 0 103; CHECK-NEXT: [[TMP6:%.*]] = insertelement <3 x double> [[TMP4]], double [[TMP5]], i64 2 104; CHECK-NEXT: [[TMP7:%.*]] = extractelement <3 x double> [[SPLIT]], i64 1 105; CHECK-NEXT: [[TMP8:%.*]] = insertelement <3 x double> undef, double [[TMP7]], i64 0 106; CHECK-NEXT: [[TMP9:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 1 107; CHECK-NEXT: [[TMP10:%.*]] = insertelement <3 x double> [[TMP8]], double [[TMP9]], i64 1 108; CHECK-NEXT: [[TMP11:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 1 109; CHECK-NEXT: [[TMP12:%.*]] = insertelement <3 x double> [[TMP10]], double [[TMP11]], i64 2 110; CHECK-NEXT: [[TMP13:%.*]] = extractelement <3 x double> [[SPLIT]], i64 2 111; CHECK-NEXT: [[TMP14:%.*]] = insertelement <3 x double> undef, double [[TMP13]], i64 0 112; CHECK-NEXT: [[TMP15:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 2 113; CHECK-NEXT: [[TMP16:%.*]] = insertelement <3 x double> [[TMP14]], double [[TMP15]], i64 1 114; CHECK-NEXT: [[TMP17:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2 115; CHECK-NEXT: [[TMP18:%.*]] = insertelement <3 x double> [[TMP16]], double [[TMP17]], i64 2 116; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <3 x double> [[TMP6]], <3 x double> [[TMP12]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5> 117; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <3 x double> [[TMP18]], <3 x double> undef, <6 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef> 118; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <6 x double> [[TMP19]], <6 x double> [[TMP20]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8> 119; CHECK-NEXT: [[A_FOO:%.*]] = call <9 x double> @foo(<9 x double> [[TMP21]]) 120; CHECK-NEXT: [[SPLIT3:%.*]] = shufflevector <9 x double> [[B:%.*]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2> 121; CHECK-NEXT: [[SPLIT4:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5> 122; CHECK-NEXT: [[SPLIT5:%.*]] = shufflevector <9 x double> [[B]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8> 123; CHECK-NEXT: [[SPLIT6:%.*]] = shufflevector <9 x double> [[A_FOO]], <9 x double> undef, <3 x i32> <i32 0, i32 1, i32 2> 124; CHECK-NEXT: [[SPLIT7:%.*]] = shufflevector <9 x double> [[A_FOO]], <9 x double> undef, <3 x i32> <i32 3, i32 4, i32 5> 125; CHECK-NEXT: [[SPLIT8:%.*]] = shufflevector <9 x double> [[A_FOO]], <9 x double> undef, <3 x i32> <i32 6, i32 7, i32 8> 126; 127 %A.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %A, i32 3, i32 3) 128 %A.foo = call <9 x double> @foo(<9 x double> %A.trans) 129 %res = tail call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %B, <9 x double> %A.foo, i32 3, i32 3, i32 3) 130 ret <9 x double> %res 131} 132 133declare <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double>, <9 x double>, i32 immarg, i32 immarg, i32 immarg) 134declare <9 x double> @llvm.matrix.transpose.v9f64(<9 x double>, i32 immarg, i32 immarg) 135declare <9 x double> @foo(<9 x double>) 136