Lines Matching refs:MVT
90 addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass); in AArch64TargetLowering()
91 addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass); in AArch64TargetLowering()
94 addRegisterClass(MVT::f16, &AArch64::FPR16RegClass); in AArch64TargetLowering()
95 addRegisterClass(MVT::f32, &AArch64::FPR32RegClass); in AArch64TargetLowering()
96 addRegisterClass(MVT::f64, &AArch64::FPR64RegClass); in AArch64TargetLowering()
97 addRegisterClass(MVT::f128, &AArch64::FPR128RegClass); in AArch64TargetLowering()
101 addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass); in AArch64TargetLowering()
102 addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass); in AArch64TargetLowering()
104 addDRTypeForNEON(MVT::v2f32); in AArch64TargetLowering()
105 addDRTypeForNEON(MVT::v8i8); in AArch64TargetLowering()
106 addDRTypeForNEON(MVT::v4i16); in AArch64TargetLowering()
107 addDRTypeForNEON(MVT::v2i32); in AArch64TargetLowering()
108 addDRTypeForNEON(MVT::v1i64); in AArch64TargetLowering()
109 addDRTypeForNEON(MVT::v1f64); in AArch64TargetLowering()
110 addDRTypeForNEON(MVT::v4f16); in AArch64TargetLowering()
112 addQRTypeForNEON(MVT::v4f32); in AArch64TargetLowering()
113 addQRTypeForNEON(MVT::v2f64); in AArch64TargetLowering()
114 addQRTypeForNEON(MVT::v16i8); in AArch64TargetLowering()
115 addQRTypeForNEON(MVT::v8i16); in AArch64TargetLowering()
116 addQRTypeForNEON(MVT::v4i32); in AArch64TargetLowering()
117 addQRTypeForNEON(MVT::v2i64); in AArch64TargetLowering()
118 addQRTypeForNEON(MVT::v8f16); in AArch64TargetLowering()
125 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); in AArch64TargetLowering()
126 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); in AArch64TargetLowering()
127 setOperationAction(ISD::SETCC, MVT::i32, Custom); in AArch64TargetLowering()
128 setOperationAction(ISD::SETCC, MVT::i64, Custom); in AArch64TargetLowering()
129 setOperationAction(ISD::SETCC, MVT::f32, Custom); in AArch64TargetLowering()
130 setOperationAction(ISD::SETCC, MVT::f64, Custom); in AArch64TargetLowering()
131 setOperationAction(ISD::BRCOND, MVT::Other, Expand); in AArch64TargetLowering()
132 setOperationAction(ISD::BR_CC, MVT::i32, Custom); in AArch64TargetLowering()
133 setOperationAction(ISD::BR_CC, MVT::i64, Custom); in AArch64TargetLowering()
134 setOperationAction(ISD::BR_CC, MVT::f32, Custom); in AArch64TargetLowering()
135 setOperationAction(ISD::BR_CC, MVT::f64, Custom); in AArch64TargetLowering()
136 setOperationAction(ISD::SELECT, MVT::i32, Custom); in AArch64TargetLowering()
137 setOperationAction(ISD::SELECT, MVT::i64, Custom); in AArch64TargetLowering()
138 setOperationAction(ISD::SELECT, MVT::f32, Custom); in AArch64TargetLowering()
139 setOperationAction(ISD::SELECT, MVT::f64, Custom); in AArch64TargetLowering()
140 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); in AArch64TargetLowering()
141 setOperationAction(ISD::SELECT_CC, MVT::i64, Custom); in AArch64TargetLowering()
142 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); in AArch64TargetLowering()
143 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); in AArch64TargetLowering()
144 setOperationAction(ISD::BR_JT, MVT::Other, Expand); in AArch64TargetLowering()
145 setOperationAction(ISD::JumpTable, MVT::i64, Custom); in AArch64TargetLowering()
147 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
148 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
149 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
151 setOperationAction(ISD::FREM, MVT::f32, Expand); in AArch64TargetLowering()
152 setOperationAction(ISD::FREM, MVT::f64, Expand); in AArch64TargetLowering()
153 setOperationAction(ISD::FREM, MVT::f80, Expand); in AArch64TargetLowering()
157 setOperationAction(ISD::XOR, MVT::i32, Custom); in AArch64TargetLowering()
158 setOperationAction(ISD::XOR, MVT::i64, Custom); in AArch64TargetLowering()
162 setOperationAction(ISD::FABS, MVT::f128, Expand); in AArch64TargetLowering()
163 setOperationAction(ISD::FADD, MVT::f128, Custom); in AArch64TargetLowering()
164 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand); in AArch64TargetLowering()
165 setOperationAction(ISD::FCOS, MVT::f128, Expand); in AArch64TargetLowering()
166 setOperationAction(ISD::FDIV, MVT::f128, Custom); in AArch64TargetLowering()
167 setOperationAction(ISD::FMA, MVT::f128, Expand); in AArch64TargetLowering()
168 setOperationAction(ISD::FMUL, MVT::f128, Custom); in AArch64TargetLowering()
169 setOperationAction(ISD::FNEG, MVT::f128, Expand); in AArch64TargetLowering()
170 setOperationAction(ISD::FPOW, MVT::f128, Expand); in AArch64TargetLowering()
171 setOperationAction(ISD::FREM, MVT::f128, Expand); in AArch64TargetLowering()
172 setOperationAction(ISD::FRINT, MVT::f128, Expand); in AArch64TargetLowering()
173 setOperationAction(ISD::FSIN, MVT::f128, Expand); in AArch64TargetLowering()
174 setOperationAction(ISD::FSINCOS, MVT::f128, Expand); in AArch64TargetLowering()
175 setOperationAction(ISD::FSQRT, MVT::f128, Expand); in AArch64TargetLowering()
176 setOperationAction(ISD::FSUB, MVT::f128, Custom); in AArch64TargetLowering()
177 setOperationAction(ISD::FTRUNC, MVT::f128, Expand); in AArch64TargetLowering()
178 setOperationAction(ISD::SETCC, MVT::f128, Custom); in AArch64TargetLowering()
179 setOperationAction(ISD::BR_CC, MVT::f128, Custom); in AArch64TargetLowering()
180 setOperationAction(ISD::SELECT, MVT::f128, Custom); in AArch64TargetLowering()
181 setOperationAction(ISD::SELECT_CC, MVT::f128, Custom); in AArch64TargetLowering()
182 setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom); in AArch64TargetLowering()
186 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); in AArch64TargetLowering()
187 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); in AArch64TargetLowering()
188 setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom); in AArch64TargetLowering()
189 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); in AArch64TargetLowering()
190 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); in AArch64TargetLowering()
191 setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom); in AArch64TargetLowering()
192 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); in AArch64TargetLowering()
193 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); in AArch64TargetLowering()
194 setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom); in AArch64TargetLowering()
195 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); in AArch64TargetLowering()
196 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); in AArch64TargetLowering()
197 setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom); in AArch64TargetLowering()
198 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom); in AArch64TargetLowering()
199 setOperationAction(ISD::FP_ROUND, MVT::f64, Custom); in AArch64TargetLowering()
202 setOperationAction(ISD::VASTART, MVT::Other, Custom); in AArch64TargetLowering()
203 setOperationAction(ISD::VAARG, MVT::Other, Custom); in AArch64TargetLowering()
204 setOperationAction(ISD::VACOPY, MVT::Other, Custom); in AArch64TargetLowering()
205 setOperationAction(ISD::VAEND, MVT::Other, Expand); in AArch64TargetLowering()
208 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); in AArch64TargetLowering()
209 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); in AArch64TargetLowering()
210 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); in AArch64TargetLowering()
218 setOperationAction(ISD::ConstantPool, MVT::i64, Custom); in AArch64TargetLowering()
221 setOperationAction(ISD::BlockAddress, MVT::i64, Custom); in AArch64TargetLowering()
224 setOperationAction(ISD::ADDC, MVT::i32, Custom); in AArch64TargetLowering()
225 setOperationAction(ISD::ADDE, MVT::i32, Custom); in AArch64TargetLowering()
226 setOperationAction(ISD::SUBC, MVT::i32, Custom); in AArch64TargetLowering()
227 setOperationAction(ISD::SUBE, MVT::i32, Custom); in AArch64TargetLowering()
228 setOperationAction(ISD::ADDC, MVT::i64, Custom); in AArch64TargetLowering()
229 setOperationAction(ISD::ADDE, MVT::i64, Custom); in AArch64TargetLowering()
230 setOperationAction(ISD::SUBC, MVT::i64, Custom); in AArch64TargetLowering()
231 setOperationAction(ISD::SUBE, MVT::i64, Custom); in AArch64TargetLowering()
234 setOperationAction(ISD::ROTL, MVT::i32, Expand); in AArch64TargetLowering()
235 setOperationAction(ISD::ROTL, MVT::i64, Expand); in AArch64TargetLowering()
238 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); in AArch64TargetLowering()
239 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); in AArch64TargetLowering()
244 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand); in AArch64TargetLowering()
245 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand); in AArch64TargetLowering()
246 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand); in AArch64TargetLowering()
247 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand); in AArch64TargetLowering()
249 setOperationAction(ISD::CTPOP, MVT::i32, Custom); in AArch64TargetLowering()
250 setOperationAction(ISD::CTPOP, MVT::i64, Custom); in AArch64TargetLowering()
252 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); in AArch64TargetLowering()
253 setOperationAction(ISD::SDIVREM, MVT::i64, Expand); in AArch64TargetLowering()
254 setOperationAction(ISD::SREM, MVT::i32, Expand); in AArch64TargetLowering()
255 setOperationAction(ISD::SREM, MVT::i64, Expand); in AArch64TargetLowering()
256 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); in AArch64TargetLowering()
257 setOperationAction(ISD::UDIVREM, MVT::i64, Expand); in AArch64TargetLowering()
258 setOperationAction(ISD::UREM, MVT::i32, Expand); in AArch64TargetLowering()
259 setOperationAction(ISD::UREM, MVT::i64, Expand); in AArch64TargetLowering()
262 setOperationAction(ISD::SADDO, MVT::i32, Custom); in AArch64TargetLowering()
263 setOperationAction(ISD::SADDO, MVT::i64, Custom); in AArch64TargetLowering()
264 setOperationAction(ISD::UADDO, MVT::i32, Custom); in AArch64TargetLowering()
265 setOperationAction(ISD::UADDO, MVT::i64, Custom); in AArch64TargetLowering()
266 setOperationAction(ISD::SSUBO, MVT::i32, Custom); in AArch64TargetLowering()
267 setOperationAction(ISD::SSUBO, MVT::i64, Custom); in AArch64TargetLowering()
268 setOperationAction(ISD::USUBO, MVT::i32, Custom); in AArch64TargetLowering()
269 setOperationAction(ISD::USUBO, MVT::i64, Custom); in AArch64TargetLowering()
270 setOperationAction(ISD::SMULO, MVT::i32, Custom); in AArch64TargetLowering()
271 setOperationAction(ISD::SMULO, MVT::i64, Custom); in AArch64TargetLowering()
272 setOperationAction(ISD::UMULO, MVT::i32, Custom); in AArch64TargetLowering()
273 setOperationAction(ISD::UMULO, MVT::i64, Custom); in AArch64TargetLowering()
275 setOperationAction(ISD::FSIN, MVT::f32, Expand); in AArch64TargetLowering()
276 setOperationAction(ISD::FSIN, MVT::f64, Expand); in AArch64TargetLowering()
277 setOperationAction(ISD::FCOS, MVT::f32, Expand); in AArch64TargetLowering()
278 setOperationAction(ISD::FCOS, MVT::f64, Expand); in AArch64TargetLowering()
279 setOperationAction(ISD::FPOW, MVT::f32, Expand); in AArch64TargetLowering()
280 setOperationAction(ISD::FPOW, MVT::f64, Expand); in AArch64TargetLowering()
281 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); in AArch64TargetLowering()
282 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in AArch64TargetLowering()
285 setOperationAction(ISD::SETCC, MVT::f16, Promote); in AArch64TargetLowering()
286 setOperationAction(ISD::BR_CC, MVT::f16, Promote); in AArch64TargetLowering()
287 setOperationAction(ISD::SELECT_CC, MVT::f16, Promote); in AArch64TargetLowering()
288 setOperationAction(ISD::SELECT, MVT::f16, Promote); in AArch64TargetLowering()
289 setOperationAction(ISD::FADD, MVT::f16, Promote); in AArch64TargetLowering()
290 setOperationAction(ISD::FSUB, MVT::f16, Promote); in AArch64TargetLowering()
291 setOperationAction(ISD::FMUL, MVT::f16, Promote); in AArch64TargetLowering()
292 setOperationAction(ISD::FDIV, MVT::f16, Promote); in AArch64TargetLowering()
293 setOperationAction(ISD::FREM, MVT::f16, Promote); in AArch64TargetLowering()
294 setOperationAction(ISD::FMA, MVT::f16, Promote); in AArch64TargetLowering()
295 setOperationAction(ISD::FNEG, MVT::f16, Promote); in AArch64TargetLowering()
296 setOperationAction(ISD::FABS, MVT::f16, Promote); in AArch64TargetLowering()
297 setOperationAction(ISD::FCEIL, MVT::f16, Promote); in AArch64TargetLowering()
298 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote); in AArch64TargetLowering()
299 setOperationAction(ISD::FCOS, MVT::f16, Promote); in AArch64TargetLowering()
300 setOperationAction(ISD::FFLOOR, MVT::f16, Promote); in AArch64TargetLowering()
301 setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote); in AArch64TargetLowering()
302 setOperationAction(ISD::FPOW, MVT::f16, Promote); in AArch64TargetLowering()
303 setOperationAction(ISD::FPOWI, MVT::f16, Promote); in AArch64TargetLowering()
304 setOperationAction(ISD::FRINT, MVT::f16, Promote); in AArch64TargetLowering()
305 setOperationAction(ISD::FSIN, MVT::f16, Promote); in AArch64TargetLowering()
306 setOperationAction(ISD::FSINCOS, MVT::f16, Promote); in AArch64TargetLowering()
307 setOperationAction(ISD::FSQRT, MVT::f16, Promote); in AArch64TargetLowering()
308 setOperationAction(ISD::FEXP, MVT::f16, Promote); in AArch64TargetLowering()
309 setOperationAction(ISD::FEXP2, MVT::f16, Promote); in AArch64TargetLowering()
310 setOperationAction(ISD::FLOG, MVT::f16, Promote); in AArch64TargetLowering()
311 setOperationAction(ISD::FLOG2, MVT::f16, Promote); in AArch64TargetLowering()
312 setOperationAction(ISD::FLOG10, MVT::f16, Promote); in AArch64TargetLowering()
313 setOperationAction(ISD::FROUND, MVT::f16, Promote); in AArch64TargetLowering()
314 setOperationAction(ISD::FTRUNC, MVT::f16, Promote); in AArch64TargetLowering()
315 setOperationAction(ISD::FMINNUM, MVT::f16, Promote); in AArch64TargetLowering()
316 setOperationAction(ISD::FMAXNUM, MVT::f16, Promote); in AArch64TargetLowering()
320 setOperationAction(ISD::FADD, MVT::v4f16, Promote); in AArch64TargetLowering()
321 setOperationAction(ISD::FSUB, MVT::v4f16, Promote); in AArch64TargetLowering()
322 setOperationAction(ISD::FMUL, MVT::v4f16, Promote); in AArch64TargetLowering()
323 setOperationAction(ISD::FDIV, MVT::v4f16, Promote); in AArch64TargetLowering()
324 setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Promote); in AArch64TargetLowering()
325 setOperationAction(ISD::FP_ROUND, MVT::v4f16, Promote); in AArch64TargetLowering()
326 AddPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
327 AddPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
328 AddPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
329 AddPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
330 AddPromotedToType(ISD::FP_EXTEND, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
331 AddPromotedToType(ISD::FP_ROUND, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
336 setOperationAction(ISD::FABS, MVT::v4f16, Expand); in AArch64TargetLowering()
337 setOperationAction(ISD::FCEIL, MVT::v4f16, Expand); in AArch64TargetLowering()
338 setOperationAction(ISD::FCOPYSIGN, MVT::v4f16, Expand); in AArch64TargetLowering()
339 setOperationAction(ISD::FCOS, MVT::v4f16, Expand); in AArch64TargetLowering()
340 setOperationAction(ISD::FFLOOR, MVT::v4f16, Expand); in AArch64TargetLowering()
341 setOperationAction(ISD::FMA, MVT::v4f16, Expand); in AArch64TargetLowering()
342 setOperationAction(ISD::FNEARBYINT, MVT::v4f16, Expand); in AArch64TargetLowering()
343 setOperationAction(ISD::FNEG, MVT::v4f16, Expand); in AArch64TargetLowering()
344 setOperationAction(ISD::FPOW, MVT::v4f16, Expand); in AArch64TargetLowering()
345 setOperationAction(ISD::FPOWI, MVT::v4f16, Expand); in AArch64TargetLowering()
346 setOperationAction(ISD::FREM, MVT::v4f16, Expand); in AArch64TargetLowering()
347 setOperationAction(ISD::FROUND, MVT::v4f16, Expand); in AArch64TargetLowering()
348 setOperationAction(ISD::FRINT, MVT::v4f16, Expand); in AArch64TargetLowering()
349 setOperationAction(ISD::FSIN, MVT::v4f16, Expand); in AArch64TargetLowering()
350 setOperationAction(ISD::FSINCOS, MVT::v4f16, Expand); in AArch64TargetLowering()
351 setOperationAction(ISD::FSQRT, MVT::v4f16, Expand); in AArch64TargetLowering()
352 setOperationAction(ISD::FTRUNC, MVT::v4f16, Expand); in AArch64TargetLowering()
353 setOperationAction(ISD::SETCC, MVT::v4f16, Expand); in AArch64TargetLowering()
354 setOperationAction(ISD::BR_CC, MVT::v4f16, Expand); in AArch64TargetLowering()
355 setOperationAction(ISD::SELECT, MVT::v4f16, Expand); in AArch64TargetLowering()
356 setOperationAction(ISD::SELECT_CC, MVT::v4f16, Expand); in AArch64TargetLowering()
357 setOperationAction(ISD::FEXP, MVT::v4f16, Expand); in AArch64TargetLowering()
358 setOperationAction(ISD::FEXP2, MVT::v4f16, Expand); in AArch64TargetLowering()
359 setOperationAction(ISD::FLOG, MVT::v4f16, Expand); in AArch64TargetLowering()
360 setOperationAction(ISD::FLOG2, MVT::v4f16, Expand); in AArch64TargetLowering()
361 setOperationAction(ISD::FLOG10, MVT::v4f16, Expand); in AArch64TargetLowering()
365 setOperationAction(ISD::FABS, MVT::v8f16, Expand); in AArch64TargetLowering()
366 setOperationAction(ISD::FADD, MVT::v8f16, Expand); in AArch64TargetLowering()
367 setOperationAction(ISD::FCEIL, MVT::v8f16, Expand); in AArch64TargetLowering()
368 setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Expand); in AArch64TargetLowering()
369 setOperationAction(ISD::FCOS, MVT::v8f16, Expand); in AArch64TargetLowering()
370 setOperationAction(ISD::FDIV, MVT::v8f16, Expand); in AArch64TargetLowering()
371 setOperationAction(ISD::FFLOOR, MVT::v8f16, Expand); in AArch64TargetLowering()
372 setOperationAction(ISD::FMA, MVT::v8f16, Expand); in AArch64TargetLowering()
373 setOperationAction(ISD::FMUL, MVT::v8f16, Expand); in AArch64TargetLowering()
374 setOperationAction(ISD::FNEARBYINT, MVT::v8f16, Expand); in AArch64TargetLowering()
375 setOperationAction(ISD::FNEG, MVT::v8f16, Expand); in AArch64TargetLowering()
376 setOperationAction(ISD::FPOW, MVT::v8f16, Expand); in AArch64TargetLowering()
377 setOperationAction(ISD::FPOWI, MVT::v8f16, Expand); in AArch64TargetLowering()
378 setOperationAction(ISD::FREM, MVT::v8f16, Expand); in AArch64TargetLowering()
379 setOperationAction(ISD::FROUND, MVT::v8f16, Expand); in AArch64TargetLowering()
380 setOperationAction(ISD::FRINT, MVT::v8f16, Expand); in AArch64TargetLowering()
381 setOperationAction(ISD::FSIN, MVT::v8f16, Expand); in AArch64TargetLowering()
382 setOperationAction(ISD::FSINCOS, MVT::v8f16, Expand); in AArch64TargetLowering()
383 setOperationAction(ISD::FSQRT, MVT::v8f16, Expand); in AArch64TargetLowering()
384 setOperationAction(ISD::FSUB, MVT::v8f16, Expand); in AArch64TargetLowering()
385 setOperationAction(ISD::FTRUNC, MVT::v8f16, Expand); in AArch64TargetLowering()
386 setOperationAction(ISD::SETCC, MVT::v8f16, Expand); in AArch64TargetLowering()
387 setOperationAction(ISD::BR_CC, MVT::v8f16, Expand); in AArch64TargetLowering()
388 setOperationAction(ISD::SELECT, MVT::v8f16, Expand); in AArch64TargetLowering()
389 setOperationAction(ISD::SELECT_CC, MVT::v8f16, Expand); in AArch64TargetLowering()
390 setOperationAction(ISD::FP_EXTEND, MVT::v8f16, Expand); in AArch64TargetLowering()
391 setOperationAction(ISD::FEXP, MVT::v8f16, Expand); in AArch64TargetLowering()
392 setOperationAction(ISD::FEXP2, MVT::v8f16, Expand); in AArch64TargetLowering()
393 setOperationAction(ISD::FLOG, MVT::v8f16, Expand); in AArch64TargetLowering()
394 setOperationAction(ISD::FLOG2, MVT::v8f16, Expand); in AArch64TargetLowering()
395 setOperationAction(ISD::FLOG10, MVT::v8f16, Expand); in AArch64TargetLowering()
398 for (MVT Ty : {MVT::f32, MVT::f64}) { in AArch64TargetLowering()
407 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); in AArch64TargetLowering()
413 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); in AArch64TargetLowering()
414 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); in AArch64TargetLowering()
416 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in AArch64TargetLowering()
417 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in AArch64TargetLowering()
423 setOperationAction(ISD::ConstantFP, MVT::f32, Legal); in AArch64TargetLowering()
424 setOperationAction(ISD::ConstantFP, MVT::f64, Legal); in AArch64TargetLowering()
429 for (MVT VT : MVT::fp_valuetypes()) { in AArch64TargetLowering()
430 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in AArch64TargetLowering()
431 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); in AArch64TargetLowering()
432 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand); in AArch64TargetLowering()
433 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); in AArch64TargetLowering()
435 for (MVT VT : MVT::integer_valuetypes()) in AArch64TargetLowering()
436 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand); in AArch64TargetLowering()
438 setTruncStoreAction(MVT::f32, MVT::f16, Expand); in AArch64TargetLowering()
439 setTruncStoreAction(MVT::f64, MVT::f32, Expand); in AArch64TargetLowering()
440 setTruncStoreAction(MVT::f64, MVT::f16, Expand); in AArch64TargetLowering()
441 setTruncStoreAction(MVT::f128, MVT::f80, Expand); in AArch64TargetLowering()
442 setTruncStoreAction(MVT::f128, MVT::f64, Expand); in AArch64TargetLowering()
443 setTruncStoreAction(MVT::f128, MVT::f32, Expand); in AArch64TargetLowering()
444 setTruncStoreAction(MVT::f128, MVT::f16, Expand); in AArch64TargetLowering()
446 setOperationAction(ISD::BITCAST, MVT::i16, Custom); in AArch64TargetLowering()
447 setOperationAction(ISD::BITCAST, MVT::f16, Custom); in AArch64TargetLowering()
452 setIndexedLoadAction(im, MVT::i8, Legal); in AArch64TargetLowering()
453 setIndexedLoadAction(im, MVT::i16, Legal); in AArch64TargetLowering()
454 setIndexedLoadAction(im, MVT::i32, Legal); in AArch64TargetLowering()
455 setIndexedLoadAction(im, MVT::i64, Legal); in AArch64TargetLowering()
456 setIndexedLoadAction(im, MVT::f64, Legal); in AArch64TargetLowering()
457 setIndexedLoadAction(im, MVT::f32, Legal); in AArch64TargetLowering()
458 setIndexedStoreAction(im, MVT::i8, Legal); in AArch64TargetLowering()
459 setIndexedStoreAction(im, MVT::i16, Legal); in AArch64TargetLowering()
460 setIndexedStoreAction(im, MVT::i32, Legal); in AArch64TargetLowering()
461 setIndexedStoreAction(im, MVT::i64, Legal); in AArch64TargetLowering()
462 setIndexedStoreAction(im, MVT::f64, Legal); in AArch64TargetLowering()
463 setIndexedStoreAction(im, MVT::f32, Legal); in AArch64TargetLowering()
467 setOperationAction(ISD::TRAP, MVT::Other, Legal); in AArch64TargetLowering()
520 setOperationAction(ISD::FABS, MVT::v1f64, Expand); in AArch64TargetLowering()
521 setOperationAction(ISD::FADD, MVT::v1f64, Expand); in AArch64TargetLowering()
522 setOperationAction(ISD::FCEIL, MVT::v1f64, Expand); in AArch64TargetLowering()
523 setOperationAction(ISD::FCOPYSIGN, MVT::v1f64, Expand); in AArch64TargetLowering()
524 setOperationAction(ISD::FCOS, MVT::v1f64, Expand); in AArch64TargetLowering()
525 setOperationAction(ISD::FDIV, MVT::v1f64, Expand); in AArch64TargetLowering()
526 setOperationAction(ISD::FFLOOR, MVT::v1f64, Expand); in AArch64TargetLowering()
527 setOperationAction(ISD::FMA, MVT::v1f64, Expand); in AArch64TargetLowering()
528 setOperationAction(ISD::FMUL, MVT::v1f64, Expand); in AArch64TargetLowering()
529 setOperationAction(ISD::FNEARBYINT, MVT::v1f64, Expand); in AArch64TargetLowering()
530 setOperationAction(ISD::FNEG, MVT::v1f64, Expand); in AArch64TargetLowering()
531 setOperationAction(ISD::FPOW, MVT::v1f64, Expand); in AArch64TargetLowering()
532 setOperationAction(ISD::FREM, MVT::v1f64, Expand); in AArch64TargetLowering()
533 setOperationAction(ISD::FROUND, MVT::v1f64, Expand); in AArch64TargetLowering()
534 setOperationAction(ISD::FRINT, MVT::v1f64, Expand); in AArch64TargetLowering()
535 setOperationAction(ISD::FSIN, MVT::v1f64, Expand); in AArch64TargetLowering()
536 setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand); in AArch64TargetLowering()
537 setOperationAction(ISD::FSQRT, MVT::v1f64, Expand); in AArch64TargetLowering()
538 setOperationAction(ISD::FSUB, MVT::v1f64, Expand); in AArch64TargetLowering()
539 setOperationAction(ISD::FTRUNC, MVT::v1f64, Expand); in AArch64TargetLowering()
540 setOperationAction(ISD::SETCC, MVT::v1f64, Expand); in AArch64TargetLowering()
541 setOperationAction(ISD::BR_CC, MVT::v1f64, Expand); in AArch64TargetLowering()
542 setOperationAction(ISD::SELECT, MVT::v1f64, Expand); in AArch64TargetLowering()
543 setOperationAction(ISD::SELECT_CC, MVT::v1f64, Expand); in AArch64TargetLowering()
544 setOperationAction(ISD::FP_EXTEND, MVT::v1f64, Expand); in AArch64TargetLowering()
546 setOperationAction(ISD::FP_TO_SINT, MVT::v1i64, Expand); in AArch64TargetLowering()
547 setOperationAction(ISD::FP_TO_UINT, MVT::v1i64, Expand); in AArch64TargetLowering()
548 setOperationAction(ISD::SINT_TO_FP, MVT::v1i64, Expand); in AArch64TargetLowering()
549 setOperationAction(ISD::UINT_TO_FP, MVT::v1i64, Expand); in AArch64TargetLowering()
550 setOperationAction(ISD::FP_ROUND, MVT::v1f64, Expand); in AArch64TargetLowering()
552 setOperationAction(ISD::MUL, MVT::v1i64, Expand); in AArch64TargetLowering()
556 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Promote); in AArch64TargetLowering()
557 setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Promote); in AArch64TargetLowering()
558 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Promote); in AArch64TargetLowering()
559 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Promote); in AArch64TargetLowering()
562 setOperationAction(ISD::SINT_TO_FP, MVT::v8i8, Promote); in AArch64TargetLowering()
563 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Promote); in AArch64TargetLowering()
564 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote); in AArch64TargetLowering()
565 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Promote); in AArch64TargetLowering()
567 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom); in AArch64TargetLowering()
568 setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom); in AArch64TargetLowering()
569 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom); in AArch64TargetLowering()
570 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom); in AArch64TargetLowering()
573 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom); in AArch64TargetLowering()
574 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom); in AArch64TargetLowering()
577 setOperationAction(ISD::MUL, MVT::v2i64, Expand); in AArch64TargetLowering()
579 setOperationAction(ISD::MUL, MVT::v8i16, Custom); in AArch64TargetLowering()
580 setOperationAction(ISD::MUL, MVT::v4i32, Custom); in AArch64TargetLowering()
581 setOperationAction(ISD::MUL, MVT::v2i64, Custom); in AArch64TargetLowering()
583 setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal); in AArch64TargetLowering()
584 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand); in AArch64TargetLowering()
587 for (MVT VT : MVT::vector_valuetypes()) { in AArch64TargetLowering()
597 for (MVT InnerVT : MVT::vector_valuetypes()) { in AArch64TargetLowering()
606 for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64}) { in AArch64TargetLowering()
622 if (VT == MVT::v2f32 || VT == MVT::v4f16) { in addTypeForNEON()
624 AddPromotedToType(ISD::LOAD, VT.getSimpleVT(), MVT::v2i32); in addTypeForNEON()
627 AddPromotedToType(ISD::STORE, VT.getSimpleVT(), MVT::v2i32); in addTypeForNEON()
628 } else if (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16) { in addTypeForNEON()
630 AddPromotedToType(ISD::LOAD, VT.getSimpleVT(), MVT::v2i64); in addTypeForNEON()
633 AddPromotedToType(ISD::STORE, VT.getSimpleVT(), MVT::v2i64); in addTypeForNEON()
637 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) { in addTypeForNEON()
665 for (MVT InnerVT : MVT::all_valuetypes()) in addTypeForNEON()
669 if (VT != MVT::v8i8 && VT != MVT::v16i8) in addTypeForNEON()
690 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) { in addDRTypeForNEON()
692 addTypeForNEON(VT, MVT::v2i32); in addDRTypeForNEON()
695 void AArch64TargetLowering::addQRTypeForNEON(MVT VT) { in addQRTypeForNEON()
697 addTypeForNEON(VT, MVT::v4i32); in addQRTypeForNEON()
702 return MVT::i32; in getSetCCResultType()
751 MVT VT = Op.getOperand(1).getValueType().getSimpleVT(); in computeKnownBitsForTargetNode()
753 if (VT == MVT::v8i8 || VT == MVT::v16i8) { in computeKnownBitsForTargetNode()
757 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) { in computeKnownBitsForTargetNode()
769 MVT AArch64TargetLowering::getScalarShiftAmountTy(EVT LHSTy) const { in getScalarShiftAmountTy()
770 return MVT::i64; in getScalarShiftAmountTy()
1159 return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS) in emitComparison()
1177 if ((VT == MVT::i32 && C != 0x80000000 && in getAArch64Cmp()
1179 (VT == MVT::i64 && C != 0x80000000ULL && in getAArch64Cmp()
1182 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1; in getAArch64Cmp()
1188 if ((VT == MVT::i32 && C != 0 && in getAArch64Cmp()
1190 (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) { in getAArch64Cmp()
1192 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1; in getAArch64Cmp()
1198 if ((VT == MVT::i32 && C != INT32_MAX && in getAArch64Cmp()
1200 (VT == MVT::i64 && C != INT64_MAX && in getAArch64Cmp()
1203 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1; in getAArch64Cmp()
1209 if ((VT == MVT::i32 && C != UINT32_MAX && in getAArch64Cmp()
1211 (VT == MVT::i64 && C != UINT64_MAX && in getAArch64Cmp()
1214 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1; in getAArch64Cmp()
1241 cast<LoadSDNode>(LHS)->getMemoryVT() == MVT::i16 && in getAArch64Cmp()
1247 DAG.getValueType(MVT::i16)); in getAArch64Cmp()
1252 AArch64cc = DAG.getConstant(AArch64CC, MVT::i32); in getAArch64Cmp()
1260 AArch64cc = DAG.getConstant(AArch64CC, MVT::i32); in getAArch64Cmp()
1266 assert((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) && in getAArch64XALUOOp()
1297 if (Op.getValueType() == MVT::i32) { in getAArch64XALUOOp()
1303 LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS); in getAArch64XALUOOp()
1304 RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS); in getAArch64XALUOOp()
1305 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1306 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul, in getAArch64XALUOOp()
1307 DAG.getConstant(0, MVT::i64)); in getAArch64XALUOOp()
1312 Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add); in getAArch64XALUOOp()
1319 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add, in getAArch64XALUOOp()
1320 DAG.getConstant(32, MVT::i64)); in getAArch64XALUOOp()
1321 UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits); in getAArch64XALUOOp()
1322 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value, in getAArch64XALUOOp()
1323 DAG.getConstant(31, MVT::i64)); in getAArch64XALUOOp()
1326 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32); in getAArch64XALUOOp()
1335 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, in getAArch64XALUOOp()
1336 DAG.getConstant(32, MVT::i64)); in getAArch64XALUOOp()
1337 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32); in getAArch64XALUOOp()
1339 DAG.getNode(AArch64ISD::SUBS, DL, VTs, DAG.getConstant(0, MVT::i64), in getAArch64XALUOOp()
1344 assert(Op.getValueType() == MVT::i64 && "Expected an i64 value type"); in getAArch64XALUOOp()
1346 Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1348 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1349 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value, in getAArch64XALUOOp()
1350 DAG.getConstant(63, MVT::i64)); in getAArch64XALUOOp()
1353 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32); in getAArch64XALUOOp()
1357 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1358 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32); in getAArch64XALUOOp()
1360 DAG.getNode(AArch64ISD::SUBS, DL, VTs, DAG.getConstant(0, MVT::i64), in getAArch64XALUOOp()
1368 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32); in getAArch64XALUOOp()
1380 return makeLibCall(DAG, Call, MVT::f128, &Ops[0], Ops.size(), false, in LowerF128Call()
1409 if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64) in LowerXOR()
1450 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDC_ADDE_SUBC_SUBE()
1490 SDValue TVal = DAG.getConstant(1, MVT::i32); in LowerXALUO()
1491 SDValue FVal = DAG.getConstant(0, MVT::i32); in LowerXALUO()
1496 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), MVT::i32); in LowerXALUO()
1497 Overflow = DAG.getNode(AArch64ISD::CSEL, SDLoc(Op), MVT::i32, FVal, TVal, in LowerXALUO()
1500 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in LowerXALUO()
1531 return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0), in LowerPREFETCH()
1532 DAG.getConstant(PrfOp, MVT::i32), Op.getOperand(1)); in LowerPREFETCH()
1537 assert(Op.getValueType() == MVT::f128 && "Unexpected lowering"); in LowerFP_EXTEND()
1547 if (Op.getOperand(0).getValueType() != MVT::f128) { in LowerFP_ROUND()
1580 MVT ExtVT = in LowerVectorFP_TO_INT()
1581 MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()), in LowerVectorFP_TO_INT()
1597 if (Op.getOperand(0).getValueType() == MVT::f16) { in LowerFP_TO_INT()
1601 DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Op.getOperand(0))); in LowerFP_TO_INT()
1604 if (Op.getOperand(0).getValueType() != MVT::f128) { in LowerFP_TO_INT()
1630 MVT CastVT = in LowerVectorINT_TO_FP()
1631 MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()), in LowerVectorINT_TO_FP()
1654 if (Op.getValueType() == MVT::f16) { in LowerINT_TO_FP()
1657 ISD::FP_ROUND, dl, MVT::f16, in LowerINT_TO_FP()
1658 DAG.getNode(Op.getOpcode(), dl, MVT::f32, Op.getOperand(0)), in LowerINT_TO_FP()
1663 if (Op.getOperand(0).getValueType() == MVT::i128) in LowerINT_TO_FP()
1668 if (Op.getValueType() != MVT::f128) in LowerINT_TO_FP()
1699 (ArgVT == MVT::f64) ? "__sincos_stret" : "__sincosf_stret"; in LowerFSINCOS()
1712 if (Op.getValueType() != MVT::f16) in LowerBITCAST()
1715 assert(Op.getOperand(0).getValueType() == MVT::i16); in LowerBITCAST()
1718 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0)); in LowerBITCAST()
1719 Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op); in LowerBITCAST()
1721 DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, MVT::f16, Op, in LowerBITCAST()
1722 DAG.getTargetConstant(AArch64::hsub, MVT::i32)), in LowerBITCAST()
1732 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy; in getExtensionTo64Bits()
1735 case MVT::v2i8: in getExtensionTo64Bits()
1736 case MVT::v2i16: in getExtensionTo64Bits()
1737 return MVT::v2i32; in getExtensionTo64Bits()
1738 case MVT::v4i8: in getExtensionTo64Bits()
1739 return MVT::v4i16; in getExtensionTo64Bits()
1798 MVT TruncVT = MVT::getIntegerVT(EltSize); in skipExtensionForVectorMULL()
1805 Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), MVT::i32)); in skipExtensionForVectorMULL()
1808 MVT::getVectorVT(TruncVT, NumElts), Ops); in skipExtensionForVectorMULL()
1885 if (VT == MVT::v2i64) in LowerMUL()
2073 MVT ValVT = Ins[i].VT; in LowerFormalArguments()
2080 MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : MVT::Other; in LowerFormalArguments()
2082 if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8) in LowerFormalArguments()
2083 ValVT = MVT::i8; in LowerFormalArguments()
2084 else if (ActualMVT == MVT::i16) in LowerFormalArguments()
2085 ValVT = MVT::i16; in LowerFormalArguments()
2122 if (RegVT == MVT::i32) in LowerFormalArguments()
2124 else if (RegVT == MVT::i64) in LowerFormalArguments()
2126 else if (RegVT == MVT::f16) in LowerFormalArguments()
2128 else if (RegVT == MVT::f32) in LowerFormalArguments()
2130 else if (RegVT == MVT::f64 || RegVT.is64BitVector()) in LowerFormalArguments()
2132 else if (RegVT == MVT::f128 || RegVT.is128BitVector()) in LowerFormalArguments()
2181 MVT MemVT = VA.getValVT(); in LowerFormalArguments()
2273 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64); in saveVarArgRegisters()
2301 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128); in saveVarArgRegisters()
2316 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in saveVarArgRegisters()
2343 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i64 && in LowerCallResult()
2520 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in addTokenForArgument()
2585 MVT ArgVT = Outs[i].VT; in LowerCall()
2602 MVT ValVT = Outs[i].VT; in LowerCall()
2606 MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : ValVT; in LowerCall()
2609 if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8) in LowerCall()
2610 ValVT = MVT::i8; in LowerCall()
2611 else if (ActualMVT == MVT::i16) in LowerCall()
2612 ValVT = MVT::i16; in LowerCall()
2688 if (Outs[realArgIdx].ArgVT == MVT::i1) { in LowerCall()
2690 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); in LowerCall()
2691 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg); in LowerCall()
2704 if (realArgIdx == 0 && Flags.isReturned() && Outs[0].VT == MVT::i64) { in LowerCall()
2705 assert(VA.getLocVT() == MVT::i64 && in LowerCall()
2707 assert(!Ins.empty() && Ins[0].VT == MVT::i64 && in LowerCall()
2754 DAG.getConstant(Outs[i].Flags.getByValSize(), MVT::i64); in LowerCall()
2766 if (VA.getValVT() == MVT::i1 || VA.getValVT() == MVT::i8 || in LowerCall()
2767 VA.getValVT() == MVT::i16) in LowerCall()
2778 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
2837 Ops.push_back(DAG.getTargetConstant(FPDiff, MVT::i32)); in LowerCall()
2865 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCall()
2931 if (Outs[i].ArgVT == MVT::i1) { in LowerReturn()
2935 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); in LowerReturn()
2955 return DAG.getNode(AArch64ISD::RET_FLAG, DL, MVT::Other, RetOps); in LowerReturn()
3056 MVT PtrVT = getPointerTy(); in LowerDarwinGlobalTLSAddress()
3067 DAG.getLoad(MVT::i64, DL, Chain, DescAddr, MachinePointerInfo::getGOT(), in LowerDarwinGlobalTLSAddress()
3085 DAG.getNode(AArch64ISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue), in LowerDarwinGlobalTLSAddress()
3086 Chain, FuncTLVGet, DAG.getRegister(AArch64::X0, MVT::i64), in LowerDarwinGlobalTLSAddress()
3114 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerELFTLSDescCallSeq()
3161 HiVar, DAG.getTargetConstant(0, MVT::i32)), in LowerELFGlobalTLSAddress()
3165 LoVar, DAG.getTargetConstant(0, MVT::i32)), in LowerELFGlobalTLSAddress()
3195 GV, DL, MVT::i64, 0, AArch64II::MO_TLS | AArch64II::MO_HI12); in LowerELFGlobalTLSAddress()
3197 GV, DL, MVT::i64, 0, in LowerELFGlobalTLSAddress()
3201 DAG.getTargetConstant(0, MVT::i32)), in LowerELFGlobalTLSAddress()
3204 DAG.getTargetConstant(0, MVT::i32)), in LowerELFGlobalTLSAddress()
3241 if (LHS.getValueType() == MVT::f128) { in LowerBR_CC()
3242 softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl); in LowerBR_CC()
3272 SDValue CCVal = DAG.getConstant(OFCC, MVT::i32); in LowerBR_CC()
3274 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBR_CC()
3280 (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)); in LowerBR_CC()
3296 return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, Test, in LowerBR_CC()
3297 DAG.getConstant(Log2_64(Mask), MVT::i64), Dest); in LowerBR_CC()
3300 return DAG.getNode(AArch64ISD::CBZ, dl, MVT::Other, Chain, LHS, Dest); in LowerBR_CC()
3311 return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, Test, in LowerBR_CC()
3312 DAG.getConstant(Log2_64(Mask), MVT::i64), Dest); in LowerBR_CC()
3315 return DAG.getNode(AArch64ISD::CBNZ, dl, MVT::Other, Chain, LHS, Dest); in LowerBR_CC()
3321 return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, LHS, in LowerBR_CC()
3322 DAG.getConstant(Mask, MVT::i64), Dest); in LowerBR_CC()
3331 return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS, in LowerBR_CC()
3332 DAG.getConstant(Mask, MVT::i64), Dest); in LowerBR_CC()
3337 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBR_CC()
3341 assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); in LowerBR_CC()
3348 SDValue CC1Val = DAG.getConstant(CC1, MVT::i32); in LowerBR_CC()
3350 DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp); in LowerBR_CC()
3352 SDValue CC2Val = DAG.getConstant(CC2, MVT::i32); in LowerBR_CC()
3353 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val, in LowerBR_CC()
3369 if (SrcVT == MVT::f32 && VT == MVT::f64) in LowerFCOPYSIGN()
3371 else if (SrcVT == MVT::f64 && VT == MVT::f32) in LowerFCOPYSIGN()
3383 if (VT == MVT::f32 || VT == MVT::v2f32 || VT == MVT::v4f32) { in LowerFCOPYSIGN()
3384 EltVT = MVT::i32; in LowerFCOPYSIGN()
3385 VecVT = MVT::v4i32; in LowerFCOPYSIGN()
3397 } else if (VT == MVT::f64 || VT == MVT::v2f64) { in LowerFCOPYSIGN()
3398 EltVT = MVT::i64; in LowerFCOPYSIGN()
3399 VecVT = MVT::v2i64; in LowerFCOPYSIGN()
3423 if (VT == MVT::f64 || VT == MVT::v2f64) { in LowerFCOPYSIGN()
3424 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, BuildVec); in LowerFCOPYSIGN()
3425 BuildVec = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, BuildVec); in LowerFCOPYSIGN()
3426 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, BuildVec); in LowerFCOPYSIGN()
3432 if (VT == MVT::f32) in LowerFCOPYSIGN()
3434 else if (VT == MVT::f64) in LowerFCOPYSIGN()
3460 if (VT == MVT::i32) in LowerCTPOP()
3461 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val); in LowerCTPOP()
3462 Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val); in LowerCTPOP()
3464 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val); in LowerCTPOP()
3466 ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, in LowerCTPOP()
3467 DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, MVT::i32), CtPop); in LowerCTPOP()
3469 if (VT == MVT::i64) in LowerCTPOP()
3470 UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV); in LowerCTPOP()
3491 if (LHS.getValueType() == MVT::f128) { in LowerSETCC()
3492 softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl); in LowerSETCC()
3514 assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); in LowerSETCC()
3524 SDValue CC1Val = DAG.getConstant(CC1, MVT::i32); in LowerSETCC()
3537 SDValue CC1Val = DAG.getConstant(CC1, MVT::i32); in LowerSETCC()
3541 SDValue CC2Val = DAG.getConstant(CC2, MVT::i32); in LowerSETCC()
3560 if (CCmp && CResult && Cmp.getValueType() == MVT::f32 && in selectCCOpsAreFMaxCompatible()
3561 Result.getValueType() == MVT::f64) { in selectCCOpsAreFMaxCompatible()
3577 if (LHS.getValueType() == MVT::f128) { in LowerSELECT_CC()
3578 softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl); in LowerSELECT_CC()
3591 (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)); in LowerSELECT_CC()
3640 } else if (TVal.getValueType() == MVT::i32) { in LowerSELECT_CC()
3687 assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); in LowerSELECT_CC()
3736 SDValue CC1Val = DAG.getConstant(CC1, MVT::i32); in LowerSELECT_CC()
3742 SDValue CC2Val = DAG.getConstant(CC2, MVT::i32); in LowerSELECT_CC()
3781 SDValue CCVal = DAG.getConstant(OFCC, MVT::i32); in LowerSELECT()
3960 MemOps.push_back(DAG.getStore(Chain, DL, DAG.getConstant(-GPRSize, MVT::i32), in LowerAAPCS_VASTART()
3967 MemOps.push_back(DAG.getStore(Chain, DL, DAG.getConstant(-FPRSize, MVT::i32), in LowerAAPCS_VASTART()
3971 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerAAPCS_VASTART()
3989 Op.getOperand(2), DAG.getConstant(VaListSize, MVT::i32), in LowerVACOPY()
4027 if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) { in LowerVAARG()
4042 SDValue WideFP = DAG.getLoad(MVT::f64, DL, APStore, VAList, in LowerVAARG()
4122 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, in LowerShiftRightParts()
4123 DAG.getConstant(VTBits, MVT::i64), ShAmt); in LowerShiftRightParts()
4125 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt, in LowerShiftRightParts()
4126 DAG.getConstant(VTBits, MVT::i64)); in LowerShiftRightParts()
4129 SDValue Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, MVT::i64), in LowerShiftRightParts()
4131 SDValue CCVal = DAG.getConstant(AArch64CC::GE, MVT::i32); in LowerShiftRightParts()
4143 DAG.getConstant(VTBits - 1, MVT::i64)) in LowerShiftRightParts()
4166 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, in LowerShiftLeftParts()
4167 DAG.getConstant(VTBits, MVT::i64), ShAmt); in LowerShiftLeftParts()
4169 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt, in LowerShiftLeftParts()
4170 DAG.getConstant(VTBits, MVT::i64)); in LowerShiftLeftParts()
4176 SDValue Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, MVT::i64), in LowerShiftLeftParts()
4178 SDValue CCVal = DAG.getConstant(AArch64CC::GE, MVT::i32); in LowerShiftLeftParts()
4202 if (Imm.isPosZero() && (VT == MVT::f64 || VT == MVT::f32)) in isFPImmLegal()
4205 if (VT == MVT::f64) in isFPImmLegal()
4207 else if (VT == MVT::f32) in isFPImmLegal()
4299 MVT VT) const { in getRegForInlineAsmConstraint()
4307 if (VT == MVT::f32) in getRegForInlineAsmConstraint()
4375 if (Op.getValueType() == MVT::i64) in LowerAsmOperandForConstraint()
4376 Result = DAG.getRegister(AArch64::XZR, MVT::i64); in LowerAsmOperandForConstraint()
4378 Result = DAG.getRegister(AArch64::WZR, MVT::i32); in LowerAsmOperandForConstraint()
4477 Result = DAG.getTargetConstant(CVal, MVT::i64); in LowerAsmOperandForConstraint()
4498 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in WidenVector()
4499 MVT WideTy = MVT::getVectorVT(EltTy, 2 * NarrowSize); in WidenVector()
4503 V64Reg, DAG.getConstant(0, MVT::i32)); in WidenVector()
4518 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in NarrowVector()
4519 MVT NarrowTy = MVT::getVectorVT(EltTy, WideSize / 2); in NarrowVector()
4635 DAG.getConstant(NumSrcElts, MVT::i64)); in ReconstructShuffle()
4641 DAG.getConstant(0, MVT::i64)); in ReconstructShuffle()
4646 DAG.getConstant(0, MVT::i64)); in ReconstructShuffle()
4649 DAG.getConstant(NumSrcElts, MVT::i64)); in ReconstructShuffle()
4653 VEXTSrc2, DAG.getConstant(Imm, MVT::i32)); in ReconstructShuffle()
4988 DAG.getConstant(0, MVT::i64)); in tryFormConcatFromShuffle()
4992 DAG.getConstant(0, MVT::i64)); in tryFormConcatFromShuffle()
5041 if (VT.getVectorElementType() == MVT::i32 || in GeneratePerfectShuffle()
5042 VT.getVectorElementType() == MVT::f32) in GeneratePerfectShuffle()
5045 if (VT.getVectorElementType() == MVT::i16 || in GeneratePerfectShuffle()
5046 VT.getVectorElementType() == MVT::f16) in GeneratePerfectShuffle()
5049 assert(VT.getVectorElementType() == MVT::i8); in GeneratePerfectShuffle()
5057 if (EltTy == MVT::i8) in GeneratePerfectShuffle()
5059 else if (EltTy == MVT::i16 || EltTy == MVT::f16) in GeneratePerfectShuffle()
5061 else if (EltTy == MVT::i32 || EltTy == MVT::f32) in GeneratePerfectShuffle()
5063 else if (EltTy == MVT::i64 || EltTy == MVT::f64) in GeneratePerfectShuffle()
5070 SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, MVT::i64); in GeneratePerfectShuffle()
5078 DAG.getConstant(Imm, MVT::i32)); in GeneratePerfectShuffle()
5115 TBLMask.push_back(DAG.getConstant(Offset, MVT::i32)); in GenerateTBL()
5119 MVT IndexVT = MVT::v8i8; in GenerateTBL()
5122 IndexVT = MVT::v16i8; in GenerateTBL()
5132 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V1Cst); in GenerateTBL()
5135 DAG.getConstant(Intrinsic::aarch64_neon_tbl1, MVT::i32), V1Cst, in GenerateTBL()
5140 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V2Cst); in GenerateTBL()
5143 DAG.getConstant(Intrinsic::aarch64_neon_tbl1, MVT::i32), V1Cst, in GenerateTBL()
5155 DAG.getConstant(Intrinsic::aarch64_neon_tbl2, MVT::i32), V1Cst, V2Cst, in GenerateTBL()
5164 if (EltType == MVT::i8) in getDUPLANEOp()
5166 if (EltType == MVT::i16 || EltType == MVT::f16) in getDUPLANEOp()
5168 if (EltType == MVT::i32 || EltType == MVT::f32) in getDUPLANEOp()
5170 if (EltType == MVT::i64 || EltType == MVT::f64) in getDUPLANEOp()
5224 return DAG.getNode(Opcode, dl, VT, V1, DAG.getConstant(Lane, MVT::i64)); in LowerVECTOR_SHUFFLE()
5241 DAG.getConstant(Imm, MVT::i32)); in LowerVECTOR_SHUFFLE()
5246 DAG.getConstant(Imm, MVT::i32)); in LowerVECTOR_SHUFFLE()
5285 SDValue DstLaneV = DAG.getConstant(Anomaly, MVT::i64); in LowerVECTOR_SHUFFLE()
5293 SDValue SrcLaneV = DAG.getConstant(SrcLane, MVT::i64); in LowerVECTOR_SHUFFLE()
5298 ScalarVT = MVT::i32; in LowerVECTOR_SHUFFLE()
5381 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorAND()
5383 DAG.getConstant(CnstVal, MVT::i32), in LowerVectorAND()
5384 DAG.getConstant(0, MVT::i32)); in LowerVectorAND()
5390 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorAND()
5392 DAG.getConstant(CnstVal, MVT::i32), in LowerVectorAND()
5393 DAG.getConstant(8, MVT::i32)); in LowerVectorAND()
5399 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorAND()
5401 DAG.getConstant(CnstVal, MVT::i32), in LowerVectorAND()
5402 DAG.getConstant(16, MVT::i32)); in LowerVectorAND()
5408 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorAND()
5410 DAG.getConstant(CnstVal, MVT::i32), in LowerVectorAND()
5411 DAG.getConstant(24, MVT::i32)); in LowerVectorAND()
5417 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerVectorAND()
5419 DAG.getConstant(CnstVal, MVT::i32), in LowerVectorAND()
5420 DAG.getConstant(0, MVT::i32)); in LowerVectorAND()
5426 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerVectorAND()
5428 DAG.getConstant(CnstVal, MVT::i32), in LowerVectorAND()
5429 DAG.getConstant(8, MVT::i32)); in LowerVectorAND()
5533 DAG.getConstant(Intrin, MVT::i32), X, Y, Shift.getOperand(1)); in tryLowerToSLI()
5581 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorOR()
5583 DAG.getConstant(CnstVal, MVT::i32), in LowerVectorOR()
5584 DAG.getConstant(0, MVT::i32)); in LowerVectorOR()
5590 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorOR()
5592 DAG.getConstant(CnstVal, MVT::i32), in LowerVectorOR()
5593 DAG.getConstant(8, MVT::i32)); in LowerVectorOR()
5599 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorOR()
5601 DAG.getConstant(CnstVal, MVT::i32), in LowerVectorOR()
5602 DAG.getConstant(16, MVT::i32)); in LowerVectorOR()
5608 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorOR()
5610 DAG.getConstant(CnstVal, MVT::i32), in LowerVectorOR()
5611 DAG.getConstant(24, MVT::i32)); in LowerVectorOR()
5617 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerVectorOR()
5619 DAG.getConstant(CnstVal, MVT::i32), in LowerVectorOR()
5620 DAG.getConstant(0, MVT::i32)); in LowerVectorOR()
5626 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerVectorOR()
5628 DAG.getConstant(CnstVal, MVT::i32), in LowerVectorOR()
5629 DAG.getConstant(8, MVT::i32)); in LowerVectorOR()
5664 Lane = DAG.getConstant(LowBits.getZExtValue(), MVT::i32); in NormalizeBuildVector()
5701 SDValue Mov = DAG.getNode(AArch64ISD::MOVIedit, dl, MVT::v2i64, in LowerBUILD_VECTOR()
5702 DAG.getConstant(CnstVal, MVT::i32)); in LowerBUILD_VECTOR()
5707 SDValue Mov = DAG.getNode(AArch64ISD::MOVIedit, dl, MVT::f64, in LowerBUILD_VECTOR()
5708 DAG.getConstant(CnstVal, MVT::i32)); in LowerBUILD_VECTOR()
5714 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
5716 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5717 DAG.getConstant(0, MVT::i32)); in LowerBUILD_VECTOR()
5723 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
5725 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5726 DAG.getConstant(8, MVT::i32)); in LowerBUILD_VECTOR()
5732 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
5734 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5735 DAG.getConstant(16, MVT::i32)); in LowerBUILD_VECTOR()
5741 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
5743 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5744 DAG.getConstant(24, MVT::i32)); in LowerBUILD_VECTOR()
5750 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerBUILD_VECTOR()
5752 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5753 DAG.getConstant(0, MVT::i32)); in LowerBUILD_VECTOR()
5759 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerBUILD_VECTOR()
5761 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5762 DAG.getConstant(8, MVT::i32)); in LowerBUILD_VECTOR()
5768 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
5770 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5771 DAG.getConstant(264, MVT::i32)); in LowerBUILD_VECTOR()
5777 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
5779 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5780 DAG.getConstant(272, MVT::i32)); in LowerBUILD_VECTOR()
5786 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8; in LowerBUILD_VECTOR()
5788 DAG.getConstant(CnstVal, MVT::i32)); in LowerBUILD_VECTOR()
5795 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4f32 : MVT::v2f32; in LowerBUILD_VECTOR()
5797 DAG.getConstant(CnstVal, MVT::i32)); in LowerBUILD_VECTOR()
5804 SDValue Mov = DAG.getNode(AArch64ISD::FMOV, dl, MVT::v2f64, in LowerBUILD_VECTOR()
5805 DAG.getConstant(CnstVal, MVT::i32)); in LowerBUILD_VECTOR()
5813 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
5815 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5816 DAG.getConstant(0, MVT::i32)); in LowerBUILD_VECTOR()
5822 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
5824 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5825 DAG.getConstant(8, MVT::i32)); in LowerBUILD_VECTOR()
5831 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
5833 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5834 DAG.getConstant(16, MVT::i32)); in LowerBUILD_VECTOR()
5840 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
5842 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5843 DAG.getConstant(24, MVT::i32)); in LowerBUILD_VECTOR()
5849 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerBUILD_VECTOR()
5851 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5852 DAG.getConstant(0, MVT::i32)); in LowerBUILD_VECTOR()
5858 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerBUILD_VECTOR()
5860 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5861 DAG.getConstant(8, MVT::i32)); in LowerBUILD_VECTOR()
5867 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
5869 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5870 DAG.getConstant(264, MVT::i32)); in LowerBUILD_VECTOR()
5876 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
5878 DAG.getConstant(CnstVal, MVT::i32), in LowerBUILD_VECTOR()
5879 DAG.getConstant(272, MVT::i32)); in LowerBUILD_VECTOR()
5963 assert ((EltTy == MVT::f16 || EltTy == MVT::f32 || EltTy == MVT::f64) && in LowerBUILD_VECTOR()
5965 MVT NewType = MVT::getIntegerVT(EltTy.getSizeInBits()); in LowerBUILD_VECTOR()
5985 SDValue LaneIdx = DAG.getConstant(i, MVT::i64); in LowerBUILD_VECTOR()
6027 DAG.getTargetConstant(SubIdx, MVT::i32)); in LowerBUILD_VECTOR()
6035 SDValue LaneIdx = DAG.getConstant(i, MVT::i64); in LowerBUILD_VECTOR()
6057 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerINSERT_VECTOR_ELT()
6058 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 || in LowerINSERT_VECTOR_ELT()
6059 VT == MVT::v8f16) in LowerINSERT_VECTOR_ELT()
6062 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 && in LowerINSERT_VECTOR_ELT()
6063 VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16) in LowerINSERT_VECTOR_ELT()
6091 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerEXTRACT_VECTOR_ELT()
6092 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 || in LowerEXTRACT_VECTOR_ELT()
6093 VT == MVT::v8f16) in LowerEXTRACT_VECTOR_ELT()
6096 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 && in LowerEXTRACT_VECTOR_ELT()
6097 VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16) in LowerEXTRACT_VECTOR_ELT()
6107 if (ExtrTy == MVT::i16 || ExtrTy == MVT::i8) in LowerEXTRACT_VECTOR_ELT()
6108 ExtrTy = MVT::i32; in LowerEXTRACT_VECTOR_ELT()
6259 DAG.getConstant(Cnt, MVT::i32)); in LowerVectorSRA_SRL_SHL()
6261 DAG.getConstant(Intrinsic::aarch64_neon_ushl, MVT::i32), in LowerVectorSRA_SRL_SHL()
6271 DAG.getConstant(Cnt, MVT::i32)); in LowerVectorSRA_SRL_SHL()
6283 DAG.getConstant(Opc, MVT::i32), Op.getOperand(0), NegShift); in LowerVectorSRA_SRL_SHL()
6401 assert(LHS.getValueType().getVectorElementType() == MVT::f32 || in LowerVSETCC()
6402 LHS.getValueType().getVectorElementType() == MVT::f64); in LowerVSETCC()
6455 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); in getTgtMemIntrinsic()
6482 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); in getTgtMemIntrinsic()
6495 Info.memVT = MVT::getVT(PtrTy->getElementType()); in getTgtMemIntrinsic()
6508 Info.memVT = MVT::getVT(PtrTy->getElementType()); in getTgtMemIntrinsic()
6520 Info.memVT = MVT::i128; in getTgtMemIntrinsic()
6532 Info.memVT = MVT::i128; in getTgtMemIntrinsic()
6717 (allowsMisalignedMemoryAccesses(MVT::f128, 0, 1, &Fast) && Fast))) in getOptimalMemOpType()
6718 return MVT::f128; in getOptimalMemOpType()
6722 (allowsMisalignedMemoryAccesses(MVT::i64, 0, 1, &Fast) && Fast))) in getOptimalMemOpType()
6723 return MVT::i64; in getOptimalMemOpType()
6727 (allowsMisalignedMemoryAccesses(MVT::i32, 0, 1, &Fast) && Fast))) in getOptimalMemOpType()
6728 return MVT::i32; in getOptimalMemOpType()
6730 return MVT::Other; in getOptimalMemOpType()
6824 case MVT::f32: in isFMAFasterThanFMulAndFAdd()
6825 case MVT::f64: in isFMAFasterThanFMulAndFAdd()
6850 if (N->getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) && in isDesirableToCommuteWithShift()
6903 DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32), in performIntegerAbsCombine()
6906 DAG.getConstant(AArch64CC::PL, MVT::i32), in performIntegerAbsCombine()
6928 if ((VT != MVT::i32 && VT != MVT::i64) || in BuildSDIVPow2()
6952 DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, MVT::i64)); in BuildSDIVPow2()
6984 DAG.getConstant(VM1.logBase2(), MVT::i64)); in performMulCombine()
6993 DAG.getConstant(VP1.logBase2(), MVT::i64)); in performMulCombine()
7003 DAG.getConstant(VNP1.logBase2(), MVT::i64)); in performMulCombine()
7012 DAG.getConstant(VNM1.logBase2(), MVT::i64)); in performMulCombine()
7077 if (VT != MVT::f32 && VT != MVT::f64) in performIntToFpCombine()
7141 if (VT != MVT::i32 && VT != MVT::i64) in tryCombineToEXTR()
7170 DAG.getConstant(ShiftRHS, MVT::i64)); in tryCombineToEXTR()
7298 SDValue HalfIdx = DAG.getConstant(NumElements, MVT::i64); in performBitcastCombine()
7301 SDValue SubReg = DAG.getTargetConstant(AArch64::dsub, MVT::i32); in performBitcastCombine()
7334 (N00VT == MVT::v2i64 || N00VT == MVT::v4i32) && in performConcatVectorsCombine()
7336 MVT MidVT = (N00VT == MVT::v2i64 ? MVT::v4i32 : MVT::v8i16); in performConcatVectorsCombine()
7359 DAG.getConstant(0, MVT::i64)); in performConcatVectorsCombine()
7374 MVT RHSTy = RHS.getValueType().getSimpleVT(); in performConcatVectorsCombine()
7381 MVT ConcatTy = MVT::getVectorVT(RHSTy.getVectorElementType(), in performConcatVectorsCombine()
7422 if (Vec.getValueType() == MVT::v4i32) in tryCombineFixedPointConvert()
7423 VecResTy = MVT::v4f32; in tryCombineFixedPointConvert()
7424 else if (Vec.getValueType() == MVT::v2i64) in tryCombineFixedPointConvert()
7425 VecResTy = MVT::v2f64; in tryCombineFixedPointConvert()
7467 MVT NarrowTy = N.getSimpleValueType(); in tryExtendDUPToExtractHigh()
7471 MVT ElementTy = NarrowTy.getVectorElementType(); in tryExtendDUPToExtractHigh()
7473 MVT NewDUPVT = MVT::getVectorVT(ElementTy, NumElems * 2); in tryExtendDUPToExtractHigh()
7483 NewDUP, DAG.getConstant(NumElems, MVT::i64)); in tryExtendDUPToExtractHigh()
7600 if (CmpVT != MVT::i32 && CmpVT != MVT::i64) in performSetccAddFolding()
7608 AArch64CC::getInvertedCondCode(InfoAndKind.Info.AArch64.CC), MVT::i32); in performSetccAddFolding()
7638 MVT VT = N->getSimpleValueType(0); in performAddSubLongCombine()
7711 MVT ElemTy = N->getSimpleValueType(0).getScalarType(); in tryCombineShiftImm()
7759 DAG.getConstant(-ShiftAmount, MVT::i32)); in tryCombineShiftImm()
7762 DAG.getConstant(ShiftAmount, MVT::i32)); in tryCombineShiftImm()
7779 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), MVT::i32, in tryCombineCRC32()
7789 DAG.getConstant(0, MVT::i64)); in combineAcrossLanesIntrinsic()
7914 SrcVT = MVT::getVectorVT(MVT::getIntegerVT(SrcEltSize * 2), ElementCount); in performExtendCombine()
7930 DAG.getConstant(0, MVT::i64)); in performExtendCombine()
7932 DAG.getConstant(InNVT.getVectorNumElements(), MVT::i64)); in performExtendCombine()
7991 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, in replaceSplatVectorStore()
7992 DAG.getConstant(Offset, MVT::i64)); in replaceSplatVectorStore()
8028 if (!VT.isVector() || VT.getVectorNumElements() < 2 || VT == MVT::v2i64) in performSTORECombine()
8053 DAG.getConstant(0, MVT::i64)); in performSTORECombine()
8055 DAG.getConstant(NumElts, MVT::i64)); in performSTORECombine()
8060 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, in performSTORECombine()
8061 DAG.getConstant(8, MVT::i64)); in performSTORECombine()
8126 Inc = DAG.getRegister(AArch64::XZR, MVT::i64); in performPostLD1Combine()
8138 EVT Tys[3] = { VT, MVT::i64, MVT::Other }; in performPostLD1Combine()
8250 Inc = DAG.getRegister(AArch64::XZR, MVT::i64); in performNEONPostLDSTCombine()
8267 Tys[n++] = MVT::i64; // Type of write back register in performNEONPostLDSTCombine()
8268 Tys[n] = MVT::Other; // Type of the chain in performNEONPostLDSTCombine()
8300 if ((LoadNode->getMemoryVT() == MVT::i8 && width == 8) in checkValueWidth()
8301 || (LoadNode->getMemoryVT() == MVT::i16 && width == 16)) { in checkValueWidth()
8309 if ((TypeNode->getVT() == MVT::i8 && width == 8) in checkValueWidth()
8310 || (TypeNode->getVT() == MVT::i16 && width == 16)) { in checkValueWidth()
8318 if ((TypeNode->getVT() == MVT::i8 && width == 8) in checkValueWidth()
8319 || (TypeNode->getVT() == MVT::i16 && width == 16)) { in checkValueWidth()
8580 if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64) in performBRCONDCombine()
8596 BR = DAG.getNode(AArch64ISD::CBZ, SDLoc(N), MVT::Other, Chain, LHS, Dest); in performBRCONDCombine()
8598 BR = DAG.getNode(AArch64ISD::CBNZ, SDLoc(N), MVT::Other, Chain, LHS, Dest); in performBRCONDCombine()
8616 CCVT.getVectorElementType() != MVT::i1) in performVSelectCombine()
8644 if (N0.getOpcode() != ISD::SETCC || N0.getValueType() != MVT::i1) in performSelectCombine()
8654 if (SrcVT == MVT::i1) in performSelectCombine()
8773 MVT::Glue) in isUsedByReturnOnly()
8878 if (N->getValueType(0) != MVT::i16 || Op.getValueType() != MVT::f16) in ReplaceBITCASTResults()
8882 DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, MVT::f32, in ReplaceBITCASTResults()
8883 DAG.getUNDEF(MVT::i32), Op, in ReplaceBITCASTResults()
8884 DAG.getTargetConstant(AArch64::hsub, MVT::i32)), in ReplaceBITCASTResults()
8886 Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op); in ReplaceBITCASTResults()
8887 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op)); in ReplaceBITCASTResults()
8900 assert(N->getValueType(0) == MVT::i128 && "unexpected illegal conversion"); in ReplaceNodeResults()
8918 MVT SVT = VT.getSimpleVT(); in getPreferredVectorAction()
8921 if (SVT == MVT::v1i8 || SVT == MVT::v1i16 || SVT == MVT::v1i32 in getPreferredVectorAction()
8922 || SVT == MVT::v1f32) in getPreferredVectorAction()