Lines Matching refs:ISD
112 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); in AArch64TargetLowering()
113 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); in AArch64TargetLowering()
114 setOperationAction(ISD::SETCC, MVT::i32, Custom); in AArch64TargetLowering()
115 setOperationAction(ISD::SETCC, MVT::i64, Custom); in AArch64TargetLowering()
116 setOperationAction(ISD::SETCC, MVT::f32, Custom); in AArch64TargetLowering()
117 setOperationAction(ISD::SETCC, MVT::f64, Custom); in AArch64TargetLowering()
118 setOperationAction(ISD::BRCOND, MVT::Other, Expand); in AArch64TargetLowering()
119 setOperationAction(ISD::BR_CC, MVT::i32, Custom); in AArch64TargetLowering()
120 setOperationAction(ISD::BR_CC, MVT::i64, Custom); in AArch64TargetLowering()
121 setOperationAction(ISD::BR_CC, MVT::f32, Custom); in AArch64TargetLowering()
122 setOperationAction(ISD::BR_CC, MVT::f64, Custom); in AArch64TargetLowering()
123 setOperationAction(ISD::SELECT, MVT::i32, Custom); in AArch64TargetLowering()
124 setOperationAction(ISD::SELECT, MVT::i64, Custom); in AArch64TargetLowering()
125 setOperationAction(ISD::SELECT, MVT::f32, Custom); in AArch64TargetLowering()
126 setOperationAction(ISD::SELECT, MVT::f64, Custom); in AArch64TargetLowering()
127 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); in AArch64TargetLowering()
128 setOperationAction(ISD::SELECT_CC, MVT::i64, Custom); in AArch64TargetLowering()
129 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); in AArch64TargetLowering()
130 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); in AArch64TargetLowering()
131 setOperationAction(ISD::BR_JT, MVT::Other, Expand); in AArch64TargetLowering()
132 setOperationAction(ISD::JumpTable, MVT::i64, Custom); in AArch64TargetLowering()
134 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
135 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
136 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
138 setOperationAction(ISD::FREM, MVT::f32, Expand); in AArch64TargetLowering()
139 setOperationAction(ISD::FREM, MVT::f64, Expand); in AArch64TargetLowering()
140 setOperationAction(ISD::FREM, MVT::f80, Expand); in AArch64TargetLowering()
144 setOperationAction(ISD::XOR, MVT::i32, Custom); in AArch64TargetLowering()
145 setOperationAction(ISD::XOR, MVT::i64, Custom); in AArch64TargetLowering()
149 setOperationAction(ISD::FABS, MVT::f128, Expand); in AArch64TargetLowering()
150 setOperationAction(ISD::FADD, MVT::f128, Custom); in AArch64TargetLowering()
151 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand); in AArch64TargetLowering()
152 setOperationAction(ISD::FCOS, MVT::f128, Expand); in AArch64TargetLowering()
153 setOperationAction(ISD::FDIV, MVT::f128, Custom); in AArch64TargetLowering()
154 setOperationAction(ISD::FMA, MVT::f128, Expand); in AArch64TargetLowering()
155 setOperationAction(ISD::FMUL, MVT::f128, Custom); in AArch64TargetLowering()
156 setOperationAction(ISD::FNEG, MVT::f128, Expand); in AArch64TargetLowering()
157 setOperationAction(ISD::FPOW, MVT::f128, Expand); in AArch64TargetLowering()
158 setOperationAction(ISD::FREM, MVT::f128, Expand); in AArch64TargetLowering()
159 setOperationAction(ISD::FRINT, MVT::f128, Expand); in AArch64TargetLowering()
160 setOperationAction(ISD::FSIN, MVT::f128, Expand); in AArch64TargetLowering()
161 setOperationAction(ISD::FSINCOS, MVT::f128, Expand); in AArch64TargetLowering()
162 setOperationAction(ISD::FSQRT, MVT::f128, Expand); in AArch64TargetLowering()
163 setOperationAction(ISD::FSUB, MVT::f128, Custom); in AArch64TargetLowering()
164 setOperationAction(ISD::FTRUNC, MVT::f128, Expand); in AArch64TargetLowering()
165 setOperationAction(ISD::SETCC, MVT::f128, Custom); in AArch64TargetLowering()
166 setOperationAction(ISD::BR_CC, MVT::f128, Custom); in AArch64TargetLowering()
167 setOperationAction(ISD::SELECT, MVT::f128, Custom); in AArch64TargetLowering()
168 setOperationAction(ISD::SELECT_CC, MVT::f128, Custom); in AArch64TargetLowering()
169 setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom); in AArch64TargetLowering()
173 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); in AArch64TargetLowering()
174 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); in AArch64TargetLowering()
175 setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom); in AArch64TargetLowering()
176 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); in AArch64TargetLowering()
177 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); in AArch64TargetLowering()
178 setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom); in AArch64TargetLowering()
179 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); in AArch64TargetLowering()
180 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); in AArch64TargetLowering()
181 setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom); in AArch64TargetLowering()
182 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); in AArch64TargetLowering()
183 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); in AArch64TargetLowering()
184 setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom); in AArch64TargetLowering()
185 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom); in AArch64TargetLowering()
186 setOperationAction(ISD::FP_ROUND, MVT::f64, Custom); in AArch64TargetLowering()
189 setOperationAction(ISD::VASTART, MVT::Other, Custom); in AArch64TargetLowering()
190 setOperationAction(ISD::VAARG, MVT::Other, Custom); in AArch64TargetLowering()
191 setOperationAction(ISD::VACOPY, MVT::Other, Custom); in AArch64TargetLowering()
192 setOperationAction(ISD::VAEND, MVT::Other, Expand); in AArch64TargetLowering()
195 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); in AArch64TargetLowering()
196 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); in AArch64TargetLowering()
197 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); in AArch64TargetLowering()
200 setOperationAction(ISD::ConstantPool, MVT::i64, Custom); in AArch64TargetLowering()
203 setOperationAction(ISD::BlockAddress, MVT::i64, Custom); in AArch64TargetLowering()
206 setOperationAction(ISD::ADDC, MVT::i32, Custom); in AArch64TargetLowering()
207 setOperationAction(ISD::ADDE, MVT::i32, Custom); in AArch64TargetLowering()
208 setOperationAction(ISD::SUBC, MVT::i32, Custom); in AArch64TargetLowering()
209 setOperationAction(ISD::SUBE, MVT::i32, Custom); in AArch64TargetLowering()
210 setOperationAction(ISD::ADDC, MVT::i64, Custom); in AArch64TargetLowering()
211 setOperationAction(ISD::ADDE, MVT::i64, Custom); in AArch64TargetLowering()
212 setOperationAction(ISD::SUBC, MVT::i64, Custom); in AArch64TargetLowering()
213 setOperationAction(ISD::SUBE, MVT::i64, Custom); in AArch64TargetLowering()
216 setOperationAction(ISD::ROTL, MVT::i32, Expand); in AArch64TargetLowering()
217 setOperationAction(ISD::ROTL, MVT::i64, Expand); in AArch64TargetLowering()
219 setOperationAction(ISD::ROTL, VT, Expand); in AArch64TargetLowering()
220 setOperationAction(ISD::ROTR, VT, Expand); in AArch64TargetLowering()
224 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); in AArch64TargetLowering()
225 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); in AArch64TargetLowering()
230 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand); in AArch64TargetLowering()
231 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand); in AArch64TargetLowering()
232 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand); in AArch64TargetLowering()
233 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand); in AArch64TargetLowering()
235 setOperationAction(ISD::CTPOP, MVT::i32, Custom); in AArch64TargetLowering()
236 setOperationAction(ISD::CTPOP, MVT::i64, Custom); in AArch64TargetLowering()
238 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); in AArch64TargetLowering()
239 setOperationAction(ISD::SDIVREM, MVT::i64, Expand); in AArch64TargetLowering()
241 setOperationAction(ISD::SDIVREM, VT, Expand); in AArch64TargetLowering()
242 setOperationAction(ISD::UDIVREM, VT, Expand); in AArch64TargetLowering()
244 setOperationAction(ISD::SREM, MVT::i32, Expand); in AArch64TargetLowering()
245 setOperationAction(ISD::SREM, MVT::i64, Expand); in AArch64TargetLowering()
246 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); in AArch64TargetLowering()
247 setOperationAction(ISD::UDIVREM, MVT::i64, Expand); in AArch64TargetLowering()
248 setOperationAction(ISD::UREM, MVT::i32, Expand); in AArch64TargetLowering()
249 setOperationAction(ISD::UREM, MVT::i64, Expand); in AArch64TargetLowering()
252 setOperationAction(ISD::SADDO, MVT::i32, Custom); in AArch64TargetLowering()
253 setOperationAction(ISD::SADDO, MVT::i64, Custom); in AArch64TargetLowering()
254 setOperationAction(ISD::UADDO, MVT::i32, Custom); in AArch64TargetLowering()
255 setOperationAction(ISD::UADDO, MVT::i64, Custom); in AArch64TargetLowering()
256 setOperationAction(ISD::SSUBO, MVT::i32, Custom); in AArch64TargetLowering()
257 setOperationAction(ISD::SSUBO, MVT::i64, Custom); in AArch64TargetLowering()
258 setOperationAction(ISD::USUBO, MVT::i32, Custom); in AArch64TargetLowering()
259 setOperationAction(ISD::USUBO, MVT::i64, Custom); in AArch64TargetLowering()
260 setOperationAction(ISD::SMULO, MVT::i32, Custom); in AArch64TargetLowering()
261 setOperationAction(ISD::SMULO, MVT::i64, Custom); in AArch64TargetLowering()
262 setOperationAction(ISD::UMULO, MVT::i32, Custom); in AArch64TargetLowering()
263 setOperationAction(ISD::UMULO, MVT::i64, Custom); in AArch64TargetLowering()
265 setOperationAction(ISD::FSIN, MVT::f32, Expand); in AArch64TargetLowering()
266 setOperationAction(ISD::FSIN, MVT::f64, Expand); in AArch64TargetLowering()
267 setOperationAction(ISD::FCOS, MVT::f32, Expand); in AArch64TargetLowering()
268 setOperationAction(ISD::FCOS, MVT::f64, Expand); in AArch64TargetLowering()
269 setOperationAction(ISD::FPOW, MVT::f32, Expand); in AArch64TargetLowering()
270 setOperationAction(ISD::FPOW, MVT::f64, Expand); in AArch64TargetLowering()
271 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); in AArch64TargetLowering()
272 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in AArch64TargetLowering()
275 setOperationAction(ISD::SETCC, MVT::f16, Promote); in AArch64TargetLowering()
276 setOperationAction(ISD::BR_CC, MVT::f16, Promote); in AArch64TargetLowering()
277 setOperationAction(ISD::SELECT_CC, MVT::f16, Promote); in AArch64TargetLowering()
278 setOperationAction(ISD::SELECT, MVT::f16, Promote); in AArch64TargetLowering()
279 setOperationAction(ISD::FADD, MVT::f16, Promote); in AArch64TargetLowering()
280 setOperationAction(ISD::FSUB, MVT::f16, Promote); in AArch64TargetLowering()
281 setOperationAction(ISD::FMUL, MVT::f16, Promote); in AArch64TargetLowering()
282 setOperationAction(ISD::FDIV, MVT::f16, Promote); in AArch64TargetLowering()
283 setOperationAction(ISD::FREM, MVT::f16, Promote); in AArch64TargetLowering()
284 setOperationAction(ISD::FMA, MVT::f16, Promote); in AArch64TargetLowering()
285 setOperationAction(ISD::FNEG, MVT::f16, Promote); in AArch64TargetLowering()
286 setOperationAction(ISD::FABS, MVT::f16, Promote); in AArch64TargetLowering()
287 setOperationAction(ISD::FCEIL, MVT::f16, Promote); in AArch64TargetLowering()
288 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote); in AArch64TargetLowering()
289 setOperationAction(ISD::FCOS, MVT::f16, Promote); in AArch64TargetLowering()
290 setOperationAction(ISD::FFLOOR, MVT::f16, Promote); in AArch64TargetLowering()
291 setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote); in AArch64TargetLowering()
292 setOperationAction(ISD::FPOW, MVT::f16, Promote); in AArch64TargetLowering()
293 setOperationAction(ISD::FPOWI, MVT::f16, Promote); in AArch64TargetLowering()
294 setOperationAction(ISD::FRINT, MVT::f16, Promote); in AArch64TargetLowering()
295 setOperationAction(ISD::FSIN, MVT::f16, Promote); in AArch64TargetLowering()
296 setOperationAction(ISD::FSINCOS, MVT::f16, Promote); in AArch64TargetLowering()
297 setOperationAction(ISD::FSQRT, MVT::f16, Promote); in AArch64TargetLowering()
298 setOperationAction(ISD::FEXP, MVT::f16, Promote); in AArch64TargetLowering()
299 setOperationAction(ISD::FEXP2, MVT::f16, Promote); in AArch64TargetLowering()
300 setOperationAction(ISD::FLOG, MVT::f16, Promote); in AArch64TargetLowering()
301 setOperationAction(ISD::FLOG2, MVT::f16, Promote); in AArch64TargetLowering()
302 setOperationAction(ISD::FLOG10, MVT::f16, Promote); in AArch64TargetLowering()
303 setOperationAction(ISD::FROUND, MVT::f16, Promote); in AArch64TargetLowering()
304 setOperationAction(ISD::FTRUNC, MVT::f16, Promote); in AArch64TargetLowering()
305 setOperationAction(ISD::FMINNUM, MVT::f16, Promote); in AArch64TargetLowering()
306 setOperationAction(ISD::FMAXNUM, MVT::f16, Promote); in AArch64TargetLowering()
307 setOperationAction(ISD::FMINNAN, MVT::f16, Promote); in AArch64TargetLowering()
308 setOperationAction(ISD::FMAXNAN, MVT::f16, Promote); in AArch64TargetLowering()
312 setOperationAction(ISD::FADD, MVT::v4f16, Promote); in AArch64TargetLowering()
313 setOperationAction(ISD::FSUB, MVT::v4f16, Promote); in AArch64TargetLowering()
314 setOperationAction(ISD::FMUL, MVT::v4f16, Promote); in AArch64TargetLowering()
315 setOperationAction(ISD::FDIV, MVT::v4f16, Promote); in AArch64TargetLowering()
316 setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Promote); in AArch64TargetLowering()
317 setOperationAction(ISD::FP_ROUND, MVT::v4f16, Promote); in AArch64TargetLowering()
318 AddPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
319 AddPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
320 AddPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
321 AddPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
322 AddPromotedToType(ISD::FP_EXTEND, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
323 AddPromotedToType(ISD::FP_ROUND, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
328 setOperationAction(ISD::FABS, MVT::v4f16, Expand); in AArch64TargetLowering()
329 setOperationAction(ISD::FCEIL, MVT::v4f16, Expand); in AArch64TargetLowering()
330 setOperationAction(ISD::FCOPYSIGN, MVT::v4f16, Expand); in AArch64TargetLowering()
331 setOperationAction(ISD::FCOS, MVT::v4f16, Expand); in AArch64TargetLowering()
332 setOperationAction(ISD::FFLOOR, MVT::v4f16, Expand); in AArch64TargetLowering()
333 setOperationAction(ISD::FMA, MVT::v4f16, Expand); in AArch64TargetLowering()
334 setOperationAction(ISD::FNEARBYINT, MVT::v4f16, Expand); in AArch64TargetLowering()
335 setOperationAction(ISD::FNEG, MVT::v4f16, Expand); in AArch64TargetLowering()
336 setOperationAction(ISD::FPOW, MVT::v4f16, Expand); in AArch64TargetLowering()
337 setOperationAction(ISD::FPOWI, MVT::v4f16, Expand); in AArch64TargetLowering()
338 setOperationAction(ISD::FREM, MVT::v4f16, Expand); in AArch64TargetLowering()
339 setOperationAction(ISD::FROUND, MVT::v4f16, Expand); in AArch64TargetLowering()
340 setOperationAction(ISD::FRINT, MVT::v4f16, Expand); in AArch64TargetLowering()
341 setOperationAction(ISD::FSIN, MVT::v4f16, Expand); in AArch64TargetLowering()
342 setOperationAction(ISD::FSINCOS, MVT::v4f16, Expand); in AArch64TargetLowering()
343 setOperationAction(ISD::FSQRT, MVT::v4f16, Expand); in AArch64TargetLowering()
344 setOperationAction(ISD::FTRUNC, MVT::v4f16, Expand); in AArch64TargetLowering()
345 setOperationAction(ISD::SETCC, MVT::v4f16, Expand); in AArch64TargetLowering()
346 setOperationAction(ISD::BR_CC, MVT::v4f16, Expand); in AArch64TargetLowering()
347 setOperationAction(ISD::SELECT, MVT::v4f16, Expand); in AArch64TargetLowering()
348 setOperationAction(ISD::SELECT_CC, MVT::v4f16, Expand); in AArch64TargetLowering()
349 setOperationAction(ISD::FEXP, MVT::v4f16, Expand); in AArch64TargetLowering()
350 setOperationAction(ISD::FEXP2, MVT::v4f16, Expand); in AArch64TargetLowering()
351 setOperationAction(ISD::FLOG, MVT::v4f16, Expand); in AArch64TargetLowering()
352 setOperationAction(ISD::FLOG2, MVT::v4f16, Expand); in AArch64TargetLowering()
353 setOperationAction(ISD::FLOG10, MVT::v4f16, Expand); in AArch64TargetLowering()
357 setOperationAction(ISD::FABS, MVT::v8f16, Expand); in AArch64TargetLowering()
358 setOperationAction(ISD::FADD, MVT::v8f16, Expand); in AArch64TargetLowering()
359 setOperationAction(ISD::FCEIL, MVT::v8f16, Expand); in AArch64TargetLowering()
360 setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Expand); in AArch64TargetLowering()
361 setOperationAction(ISD::FCOS, MVT::v8f16, Expand); in AArch64TargetLowering()
362 setOperationAction(ISD::FDIV, MVT::v8f16, Expand); in AArch64TargetLowering()
363 setOperationAction(ISD::FFLOOR, MVT::v8f16, Expand); in AArch64TargetLowering()
364 setOperationAction(ISD::FMA, MVT::v8f16, Expand); in AArch64TargetLowering()
365 setOperationAction(ISD::FMUL, MVT::v8f16, Expand); in AArch64TargetLowering()
366 setOperationAction(ISD::FNEARBYINT, MVT::v8f16, Expand); in AArch64TargetLowering()
367 setOperationAction(ISD::FNEG, MVT::v8f16, Expand); in AArch64TargetLowering()
368 setOperationAction(ISD::FPOW, MVT::v8f16, Expand); in AArch64TargetLowering()
369 setOperationAction(ISD::FPOWI, MVT::v8f16, Expand); in AArch64TargetLowering()
370 setOperationAction(ISD::FREM, MVT::v8f16, Expand); in AArch64TargetLowering()
371 setOperationAction(ISD::FROUND, MVT::v8f16, Expand); in AArch64TargetLowering()
372 setOperationAction(ISD::FRINT, MVT::v8f16, Expand); in AArch64TargetLowering()
373 setOperationAction(ISD::FSIN, MVT::v8f16, Expand); in AArch64TargetLowering()
374 setOperationAction(ISD::FSINCOS, MVT::v8f16, Expand); in AArch64TargetLowering()
375 setOperationAction(ISD::FSQRT, MVT::v8f16, Expand); in AArch64TargetLowering()
376 setOperationAction(ISD::FSUB, MVT::v8f16, Expand); in AArch64TargetLowering()
377 setOperationAction(ISD::FTRUNC, MVT::v8f16, Expand); in AArch64TargetLowering()
378 setOperationAction(ISD::SETCC, MVT::v8f16, Expand); in AArch64TargetLowering()
379 setOperationAction(ISD::BR_CC, MVT::v8f16, Expand); in AArch64TargetLowering()
380 setOperationAction(ISD::SELECT, MVT::v8f16, Expand); in AArch64TargetLowering()
381 setOperationAction(ISD::SELECT_CC, MVT::v8f16, Expand); in AArch64TargetLowering()
382 setOperationAction(ISD::FP_EXTEND, MVT::v8f16, Expand); in AArch64TargetLowering()
383 setOperationAction(ISD::FEXP, MVT::v8f16, Expand); in AArch64TargetLowering()
384 setOperationAction(ISD::FEXP2, MVT::v8f16, Expand); in AArch64TargetLowering()
385 setOperationAction(ISD::FLOG, MVT::v8f16, Expand); in AArch64TargetLowering()
386 setOperationAction(ISD::FLOG2, MVT::v8f16, Expand); in AArch64TargetLowering()
387 setOperationAction(ISD::FLOG10, MVT::v8f16, Expand); in AArch64TargetLowering()
391 setOperationAction(ISD::FFLOOR, Ty, Legal); in AArch64TargetLowering()
392 setOperationAction(ISD::FNEARBYINT, Ty, Legal); in AArch64TargetLowering()
393 setOperationAction(ISD::FCEIL, Ty, Legal); in AArch64TargetLowering()
394 setOperationAction(ISD::FRINT, Ty, Legal); in AArch64TargetLowering()
395 setOperationAction(ISD::FTRUNC, Ty, Legal); in AArch64TargetLowering()
396 setOperationAction(ISD::FROUND, Ty, Legal); in AArch64TargetLowering()
397 setOperationAction(ISD::FMINNUM, Ty, Legal); in AArch64TargetLowering()
398 setOperationAction(ISD::FMAXNUM, Ty, Legal); in AArch64TargetLowering()
399 setOperationAction(ISD::FMINNAN, Ty, Legal); in AArch64TargetLowering()
400 setOperationAction(ISD::FMAXNAN, Ty, Legal); in AArch64TargetLowering()
403 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); in AArch64TargetLowering()
408 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); in AArch64TargetLowering()
414 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); in AArch64TargetLowering()
415 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); in AArch64TargetLowering()
417 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in AArch64TargetLowering()
418 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in AArch64TargetLowering()
424 setOperationAction(ISD::ConstantFP, MVT::f32, Legal); in AArch64TargetLowering()
425 setOperationAction(ISD::ConstantFP, MVT::f64, Legal); in AArch64TargetLowering()
431 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in AArch64TargetLowering()
432 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); in AArch64TargetLowering()
433 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand); in AArch64TargetLowering()
434 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); in AArch64TargetLowering()
437 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand); in AArch64TargetLowering()
447 setOperationAction(ISD::BITCAST, MVT::i16, Custom); in AArch64TargetLowering()
448 setOperationAction(ISD::BITCAST, MVT::f16, Custom); in AArch64TargetLowering()
451 for (unsigned im = (unsigned)ISD::PRE_INC; in AArch64TargetLowering()
452 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { in AArch64TargetLowering()
470 setOperationAction(ISD::TRAP, MVT::Other, Legal); in AArch64TargetLowering()
473 setTargetDAGCombine(ISD::OR); in AArch64TargetLowering()
477 setTargetDAGCombine(ISD::ADD); in AArch64TargetLowering()
478 setTargetDAGCombine(ISD::SUB); in AArch64TargetLowering()
480 setTargetDAGCombine(ISD::XOR); in AArch64TargetLowering()
481 setTargetDAGCombine(ISD::SINT_TO_FP); in AArch64TargetLowering()
482 setTargetDAGCombine(ISD::UINT_TO_FP); in AArch64TargetLowering()
484 setTargetDAGCombine(ISD::FP_TO_SINT); in AArch64TargetLowering()
485 setTargetDAGCombine(ISD::FP_TO_UINT); in AArch64TargetLowering()
486 setTargetDAGCombine(ISD::FDIV); in AArch64TargetLowering()
488 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); in AArch64TargetLowering()
490 setTargetDAGCombine(ISD::ANY_EXTEND); in AArch64TargetLowering()
491 setTargetDAGCombine(ISD::ZERO_EXTEND); in AArch64TargetLowering()
492 setTargetDAGCombine(ISD::SIGN_EXTEND); in AArch64TargetLowering()
493 setTargetDAGCombine(ISD::BITCAST); in AArch64TargetLowering()
494 setTargetDAGCombine(ISD::CONCAT_VECTORS); in AArch64TargetLowering()
495 setTargetDAGCombine(ISD::STORE); in AArch64TargetLowering()
497 setTargetDAGCombine(ISD::LOAD); in AArch64TargetLowering()
499 setTargetDAGCombine(ISD::MUL); in AArch64TargetLowering()
501 setTargetDAGCombine(ISD::SELECT); in AArch64TargetLowering()
502 setTargetDAGCombine(ISD::VSELECT); in AArch64TargetLowering()
504 setTargetDAGCombine(ISD::INTRINSIC_VOID); in AArch64TargetLowering()
505 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); in AArch64TargetLowering()
506 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); in AArch64TargetLowering()
507 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); in AArch64TargetLowering()
525 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in AArch64TargetLowering()
530 setOperationAction(ISD::FABS, MVT::v1f64, Expand); in AArch64TargetLowering()
531 setOperationAction(ISD::FADD, MVT::v1f64, Expand); in AArch64TargetLowering()
532 setOperationAction(ISD::FCEIL, MVT::v1f64, Expand); in AArch64TargetLowering()
533 setOperationAction(ISD::FCOPYSIGN, MVT::v1f64, Expand); in AArch64TargetLowering()
534 setOperationAction(ISD::FCOS, MVT::v1f64, Expand); in AArch64TargetLowering()
535 setOperationAction(ISD::FDIV, MVT::v1f64, Expand); in AArch64TargetLowering()
536 setOperationAction(ISD::FFLOOR, MVT::v1f64, Expand); in AArch64TargetLowering()
537 setOperationAction(ISD::FMA, MVT::v1f64, Expand); in AArch64TargetLowering()
538 setOperationAction(ISD::FMUL, MVT::v1f64, Expand); in AArch64TargetLowering()
539 setOperationAction(ISD::FNEARBYINT, MVT::v1f64, Expand); in AArch64TargetLowering()
540 setOperationAction(ISD::FNEG, MVT::v1f64, Expand); in AArch64TargetLowering()
541 setOperationAction(ISD::FPOW, MVT::v1f64, Expand); in AArch64TargetLowering()
542 setOperationAction(ISD::FREM, MVT::v1f64, Expand); in AArch64TargetLowering()
543 setOperationAction(ISD::FROUND, MVT::v1f64, Expand); in AArch64TargetLowering()
544 setOperationAction(ISD::FRINT, MVT::v1f64, Expand); in AArch64TargetLowering()
545 setOperationAction(ISD::FSIN, MVT::v1f64, Expand); in AArch64TargetLowering()
546 setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand); in AArch64TargetLowering()
547 setOperationAction(ISD::FSQRT, MVT::v1f64, Expand); in AArch64TargetLowering()
548 setOperationAction(ISD::FSUB, MVT::v1f64, Expand); in AArch64TargetLowering()
549 setOperationAction(ISD::FTRUNC, MVT::v1f64, Expand); in AArch64TargetLowering()
550 setOperationAction(ISD::SETCC, MVT::v1f64, Expand); in AArch64TargetLowering()
551 setOperationAction(ISD::BR_CC, MVT::v1f64, Expand); in AArch64TargetLowering()
552 setOperationAction(ISD::SELECT, MVT::v1f64, Expand); in AArch64TargetLowering()
553 setOperationAction(ISD::SELECT_CC, MVT::v1f64, Expand); in AArch64TargetLowering()
554 setOperationAction(ISD::FP_EXTEND, MVT::v1f64, Expand); in AArch64TargetLowering()
556 setOperationAction(ISD::FP_TO_SINT, MVT::v1i64, Expand); in AArch64TargetLowering()
557 setOperationAction(ISD::FP_TO_UINT, MVT::v1i64, Expand); in AArch64TargetLowering()
558 setOperationAction(ISD::SINT_TO_FP, MVT::v1i64, Expand); in AArch64TargetLowering()
559 setOperationAction(ISD::UINT_TO_FP, MVT::v1i64, Expand); in AArch64TargetLowering()
560 setOperationAction(ISD::FP_ROUND, MVT::v1f64, Expand); in AArch64TargetLowering()
562 setOperationAction(ISD::MUL, MVT::v1i64, Expand); in AArch64TargetLowering()
566 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Promote); in AArch64TargetLowering()
567 setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Promote); in AArch64TargetLowering()
568 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Promote); in AArch64TargetLowering()
569 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Promote); in AArch64TargetLowering()
572 setOperationAction(ISD::SINT_TO_FP, MVT::v8i8, Promote); in AArch64TargetLowering()
573 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Promote); in AArch64TargetLowering()
574 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote); in AArch64TargetLowering()
575 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Promote); in AArch64TargetLowering()
577 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom); in AArch64TargetLowering()
578 setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom); in AArch64TargetLowering()
579 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom); in AArch64TargetLowering()
580 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom); in AArch64TargetLowering()
583 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom); in AArch64TargetLowering()
584 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom); in AArch64TargetLowering()
587 setOperationAction(ISD::MUL, MVT::v2i64, Expand); in AArch64TargetLowering()
589 setOperationAction(ISD::MUL, MVT::v8i16, Custom); in AArch64TargetLowering()
590 setOperationAction(ISD::MUL, MVT::v4i32, Custom); in AArch64TargetLowering()
591 setOperationAction(ISD::MUL, MVT::v2i64, Custom); in AArch64TargetLowering()
593 setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal); in AArch64TargetLowering()
598 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); in AArch64TargetLowering()
600 setOperationAction(ISD::MULHS, VT, Expand); in AArch64TargetLowering()
601 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in AArch64TargetLowering()
602 setOperationAction(ISD::MULHU, VT, Expand); in AArch64TargetLowering()
603 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in AArch64TargetLowering()
605 setOperationAction(ISD::BSWAP, VT, Expand); in AArch64TargetLowering()
609 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
610 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
611 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
617 setOperationAction(ISD::FFLOOR, Ty, Legal); in AArch64TargetLowering()
618 setOperationAction(ISD::FNEARBYINT, Ty, Legal); in AArch64TargetLowering()
619 setOperationAction(ISD::FCEIL, Ty, Legal); in AArch64TargetLowering()
620 setOperationAction(ISD::FRINT, Ty, Legal); in AArch64TargetLowering()
621 setOperationAction(ISD::FTRUNC, Ty, Legal); in AArch64TargetLowering()
622 setOperationAction(ISD::FROUND, Ty, Legal); in AArch64TargetLowering()
633 setOperationAction(ISD::LOAD, VT.getSimpleVT(), Promote); in addTypeForNEON()
634 AddPromotedToType(ISD::LOAD, VT.getSimpleVT(), MVT::v2i32); in addTypeForNEON()
636 setOperationAction(ISD::STORE, VT.getSimpleVT(), Promote); in addTypeForNEON()
637 AddPromotedToType(ISD::STORE, VT.getSimpleVT(), MVT::v2i32); in addTypeForNEON()
639 setOperationAction(ISD::LOAD, VT.getSimpleVT(), Promote); in addTypeForNEON()
640 AddPromotedToType(ISD::LOAD, VT.getSimpleVT(), MVT::v2i64); in addTypeForNEON()
642 setOperationAction(ISD::STORE, VT.getSimpleVT(), Promote); in addTypeForNEON()
643 AddPromotedToType(ISD::STORE, VT.getSimpleVT(), MVT::v2i64); in addTypeForNEON()
648 setOperationAction(ISD::FSIN, VT.getSimpleVT(), Expand); in addTypeForNEON()
649 setOperationAction(ISD::FCOS, VT.getSimpleVT(), Expand); in addTypeForNEON()
650 setOperationAction(ISD::FPOWI, VT.getSimpleVT(), Expand); in addTypeForNEON()
651 setOperationAction(ISD::FPOW, VT.getSimpleVT(), Expand); in addTypeForNEON()
652 setOperationAction(ISD::FLOG, VT.getSimpleVT(), Expand); in addTypeForNEON()
653 setOperationAction(ISD::FLOG2, VT.getSimpleVT(), Expand); in addTypeForNEON()
654 setOperationAction(ISD::FLOG10, VT.getSimpleVT(), Expand); in addTypeForNEON()
655 setOperationAction(ISD::FEXP, VT.getSimpleVT(), Expand); in addTypeForNEON()
656 setOperationAction(ISD::FEXP2, VT.getSimpleVT(), Expand); in addTypeForNEON()
659 setOperationAction(ISD::FCOPYSIGN, VT.getSimpleVT(), Custom); in addTypeForNEON()
662 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT.getSimpleVT(), Custom); in addTypeForNEON()
663 setOperationAction(ISD::INSERT_VECTOR_ELT, VT.getSimpleVT(), Custom); in addTypeForNEON()
664 setOperationAction(ISD::BUILD_VECTOR, VT.getSimpleVT(), Custom); in addTypeForNEON()
665 setOperationAction(ISD::VECTOR_SHUFFLE, VT.getSimpleVT(), Custom); in addTypeForNEON()
666 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT.getSimpleVT(), Custom); in addTypeForNEON()
667 setOperationAction(ISD::SRA, VT.getSimpleVT(), Custom); in addTypeForNEON()
668 setOperationAction(ISD::SRL, VT.getSimpleVT(), Custom); in addTypeForNEON()
669 setOperationAction(ISD::SHL, VT.getSimpleVT(), Custom); in addTypeForNEON()
670 setOperationAction(ISD::AND, VT.getSimpleVT(), Custom); in addTypeForNEON()
671 setOperationAction(ISD::OR, VT.getSimpleVT(), Custom); in addTypeForNEON()
672 setOperationAction(ISD::SETCC, VT.getSimpleVT(), Custom); in addTypeForNEON()
673 setOperationAction(ISD::CONCAT_VECTORS, VT.getSimpleVT(), Legal); in addTypeForNEON()
675 setOperationAction(ISD::SELECT, VT.getSimpleVT(), Expand); in addTypeForNEON()
676 setOperationAction(ISD::SELECT_CC, VT.getSimpleVT(), Expand); in addTypeForNEON()
677 setOperationAction(ISD::VSELECT, VT.getSimpleVT(), Expand); in addTypeForNEON()
679 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT.getSimpleVT(), Expand); in addTypeForNEON()
683 setOperationAction(ISD::CTPOP, VT.getSimpleVT(), Expand); in addTypeForNEON()
685 setOperationAction(ISD::UDIV, VT.getSimpleVT(), Expand); in addTypeForNEON()
686 setOperationAction(ISD::SDIV, VT.getSimpleVT(), Expand); in addTypeForNEON()
687 setOperationAction(ISD::UREM, VT.getSimpleVT(), Expand); in addTypeForNEON()
688 setOperationAction(ISD::SREM, VT.getSimpleVT(), Expand); in addTypeForNEON()
689 setOperationAction(ISD::FREM, VT.getSimpleVT(), Expand); in addTypeForNEON()
691 setOperationAction(ISD::FP_TO_SINT, VT.getSimpleVT(), Custom); in addTypeForNEON()
692 setOperationAction(ISD::FP_TO_UINT, VT.getSimpleVT(), Custom); in addTypeForNEON()
697 for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) in addTypeForNEON()
702 for (unsigned Opcode : {ISD::FMINNAN, ISD::FMAXNAN, in addTypeForNEON()
703 ISD::FMINNUM, ISD::FMAXNUM}) in addTypeForNEON()
707 for (unsigned im = (unsigned)ISD::PRE_INC; in addTypeForNEON()
708 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { in addTypeForNEON()
749 case ISD::INTRINSIC_W_CHAIN: { in computeKnownBitsForTargetNode()
765 case ISD::INTRINSIC_WO_CHAIN: in computeKnownBitsForTargetNode()
766 case ISD::INTRINSIC_VOID: { in computeKnownBitsForTargetNode()
1050 static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) { in changeIntCCToAArch64CC()
1054 case ISD::SETNE: in changeIntCCToAArch64CC()
1056 case ISD::SETEQ: in changeIntCCToAArch64CC()
1058 case ISD::SETGT: in changeIntCCToAArch64CC()
1060 case ISD::SETGE: in changeIntCCToAArch64CC()
1062 case ISD::SETLT: in changeIntCCToAArch64CC()
1064 case ISD::SETLE: in changeIntCCToAArch64CC()
1066 case ISD::SETUGT: in changeIntCCToAArch64CC()
1068 case ISD::SETUGE: in changeIntCCToAArch64CC()
1070 case ISD::SETULT: in changeIntCCToAArch64CC()
1072 case ISD::SETULE: in changeIntCCToAArch64CC()
1078 static void changeFPCCToAArch64CC(ISD::CondCode CC, in changeFPCCToAArch64CC()
1085 case ISD::SETEQ: in changeFPCCToAArch64CC()
1086 case ISD::SETOEQ: in changeFPCCToAArch64CC()
1089 case ISD::SETGT: in changeFPCCToAArch64CC()
1090 case ISD::SETOGT: in changeFPCCToAArch64CC()
1093 case ISD::SETGE: in changeFPCCToAArch64CC()
1094 case ISD::SETOGE: in changeFPCCToAArch64CC()
1097 case ISD::SETOLT: in changeFPCCToAArch64CC()
1100 case ISD::SETOLE: in changeFPCCToAArch64CC()
1103 case ISD::SETONE: in changeFPCCToAArch64CC()
1107 case ISD::SETO: in changeFPCCToAArch64CC()
1110 case ISD::SETUO: in changeFPCCToAArch64CC()
1113 case ISD::SETUEQ: in changeFPCCToAArch64CC()
1117 case ISD::SETUGT: in changeFPCCToAArch64CC()
1120 case ISD::SETUGE: in changeFPCCToAArch64CC()
1123 case ISD::SETLT: in changeFPCCToAArch64CC()
1124 case ISD::SETULT: in changeFPCCToAArch64CC()
1127 case ISD::SETLE: in changeFPCCToAArch64CC()
1128 case ISD::SETULE: in changeFPCCToAArch64CC()
1131 case ISD::SETNE: in changeFPCCToAArch64CC()
1132 case ISD::SETUNE: in changeFPCCToAArch64CC()
1142 static void changeVectorFPCCToAArch64CC(ISD::CondCode CC, in changeVectorFPCCToAArch64CC()
1152 case ISD::SETUO: in changeVectorFPCCToAArch64CC()
1154 case ISD::SETO: in changeVectorFPCCToAArch64CC()
1158 case ISD::SETUEQ: in changeVectorFPCCToAArch64CC()
1159 case ISD::SETULT: in changeVectorFPCCToAArch64CC()
1160 case ISD::SETULE: in changeVectorFPCCToAArch64CC()
1161 case ISD::SETUGT: in changeVectorFPCCToAArch64CC()
1162 case ISD::SETUGE: in changeVectorFPCCToAArch64CC()
1176 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, in emitComparison()
1189 if (RHS.getOpcode() == ISD::SUB && isNullConstant(RHS.getOperand(0)) && in emitComparison()
1190 (CC == ISD::SETEQ || CC == ISD::SETNE)) { in emitComparison()
1203 } else if (LHS.getOpcode() == ISD::AND && isNullConstant(RHS) && in emitComparison()
1260 ISD::CondCode CC, SDValue CCOp, in emitConditionalComparison()
1266 else if (RHS.getOpcode() == ISD::SUB) { in emitConditionalComparison()
1268 if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) { in emitConditionalComparison()
1292 if (Opcode == ISD::SETCC) { in isConjunctionDisjunctionTree()
1299 if (Opcode == ISD::AND || Opcode == ISD::OR) { in isConjunctionDisjunctionTree()
1311 CanPushNegate = (Opcode == ISD::OR) && CanPushNegateL && CanPushNegateR; in isConjunctionDisjunctionTree()
1333 if (Opcode == ISD::SETCC) { in emitConjunctionDisjunctionTree()
1336 ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get(); in emitConjunctionDisjunctionTree()
1377 } else if ((Opcode != ISD::AND && Opcode != ISD::OR) || !Val->hasOneUse()) in emitConjunctionDisjunctionTree()
1380 assert((Opcode == ISD::OR || !PushNegate) in emitConjunctionDisjunctionTree()
1394 bool NegateOperands = Opcode == ISD::OR; in emitConjunctionDisjunctionTree()
1405 bool NeedsNegOutL = LHS->getOpcode() == ISD::OR; in emitConjunctionDisjunctionTree()
1406 bool NeedsNegOutR = RHS->getOpcode() == ISD::OR; in emitConjunctionDisjunctionTree()
1428 if (Opcode == ISD::OR && !PushNegate) in emitConjunctionDisjunctionTree()
1435 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, in getAArch64Cmp()
1445 case ISD::SETLT: in getAArch64Cmp()
1446 case ISD::SETGE: in getAArch64Cmp()
1451 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT; in getAArch64Cmp()
1456 case ISD::SETULT: in getAArch64Cmp()
1457 case ISD::SETUGE: in getAArch64Cmp()
1461 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT; in getAArch64Cmp()
1466 case ISD::SETLE: in getAArch64Cmp()
1467 case ISD::SETGT: in getAArch64Cmp()
1472 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE; in getAArch64Cmp()
1477 case ISD::SETULE: in getAArch64Cmp()
1478 case ISD::SETUGT: in getAArch64Cmp()
1483 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE; in getAArch64Cmp()
1493 if ((CC == ISD::SETEQ || CC == ISD::SETNE) && isa<ConstantSDNode>(RHS)) { in getAArch64Cmp()
1513 cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD && in getAArch64Cmp()
1519 DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS, in getAArch64Cmp()
1530 if ((CC == ISD::SETNE) ^ RHSC->isNullValue()) in getAArch64Cmp()
1556 case ISD::SADDO: in getAArch64XALUOOp()
1560 case ISD::UADDO: in getAArch64XALUOOp()
1564 case ISD::SSUBO: in getAArch64XALUOOp()
1568 case ISD::USUBO: in getAArch64XALUOOp()
1573 case ISD::SMULO: in getAArch64XALUOOp()
1574 case ISD::UMULO: { in getAArch64XALUOOp()
1576 bool IsSigned = Op.getOpcode() == ISD::SMULO; in getAArch64XALUOOp()
1578 unsigned ExtendOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in getAArch64XALUOOp()
1585 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1586 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul, in getAArch64XALUOOp()
1592 Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add); in getAArch64XALUOOp()
1599 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add, in getAArch64XALUOOp()
1601 UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits); in getAArch64XALUOOp()
1602 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value, in getAArch64XALUOOp()
1615 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, in getAArch64XALUOOp()
1627 Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1629 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1630 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value, in getAArch64XALUOOp()
1638 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1670 if (Sel.getOpcode() != ISD::SELECT_CC) in LowerXOR()
1672 if (Sel.getOpcode() != ISD::SELECT_CC) in LowerXOR()
1682 ISD::CondCode CC = cast<CondCodeSDNode>(Sel.getOperand(4))->get(); in LowerXOR()
1705 CC = ISD::getSetCCInverse(CC, true); in LowerXOR()
1714 TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other, in LowerXOR()
1738 case ISD::ADDC: in LowerADDC_ADDE_SUBC_SUBE()
1741 case ISD::SUBC: in LowerADDC_ADDE_SUBC_SUBE()
1744 case ISD::ADDE: in LowerADDC_ADDE_SUBC_SUBE()
1748 case ISD::SUBE: in LowerADDC_ADDE_SUBC_SUBE()
1783 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow); in LowerXALUO()
1859 DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0))); in LowerVectorFP_TO_INT()
1867 return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv); in LowerVectorFP_TO_INT()
1875 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0)); in LowerVectorFP_TO_INT()
1893 DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Op.getOperand(0))); in LowerFP_TO_INT()
1902 if (Op.getOpcode() == ISD::FP_TO_SINT) in LowerFP_TO_INT()
1925 return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl)); in LowerVectorINT_TO_FP()
1930 Op.getOpcode() == ISD::SINT_TO_FP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in LowerVectorINT_TO_FP()
1948 ISD::FP_ROUND, dl, MVT::f16, in LowerINT_TO_FP()
1963 if (Op.getOpcode() == ISD::SINT_TO_FP) in LowerINT_TO_FP()
2010 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0)); in LowerBITCAST()
2011 Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op); in LowerBITCAST()
2056 if (N->getOpcode() != ISD::BUILD_VECTOR) in isExtendedBUILD_VECTOR()
2079 if (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND) in skipExtensionForVectorMULL()
2085 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR"); in skipExtensionForVectorMULL()
2099 return DAG.getNode(ISD::BUILD_VECTOR, dl, in skipExtensionForVectorMULL()
2104 if (N->getOpcode() == ISD::SIGN_EXTEND) in isSignExtended()
2112 if (N->getOpcode() == ISD::ZERO_EXTEND) in isZeroExtended()
2121 if (Opcode == ISD::ADD || Opcode == ISD::SUB) { in isAddSubSExt()
2132 if (Opcode == ISD::ADD || Opcode == ISD::SUB) { in isAddSubZExt()
2205 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1), in LowerMUL()
2207 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1)); in LowerMUL()
2221 return DAG.getNode(ISD::SMAX, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
2224 return DAG.getNode(ISD::UMAX, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
2227 return DAG.getNode(ISD::SMIN, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
2230 return DAG.getNode(ISD::UMIN, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
2241 case ISD::BITCAST: in LowerOperation()
2243 case ISD::GlobalAddress: in LowerOperation()
2245 case ISD::GlobalTLSAddress: in LowerOperation()
2247 case ISD::SETCC: in LowerOperation()
2249 case ISD::BR_CC: in LowerOperation()
2251 case ISD::SELECT: in LowerOperation()
2253 case ISD::SELECT_CC: in LowerOperation()
2255 case ISD::JumpTable: in LowerOperation()
2257 case ISD::ConstantPool: in LowerOperation()
2259 case ISD::BlockAddress: in LowerOperation()
2261 case ISD::VASTART: in LowerOperation()
2263 case ISD::VACOPY: in LowerOperation()
2265 case ISD::VAARG: in LowerOperation()
2267 case ISD::ADDC: in LowerOperation()
2268 case ISD::ADDE: in LowerOperation()
2269 case ISD::SUBC: in LowerOperation()
2270 case ISD::SUBE: in LowerOperation()
2272 case ISD::SADDO: in LowerOperation()
2273 case ISD::UADDO: in LowerOperation()
2274 case ISD::SSUBO: in LowerOperation()
2275 case ISD::USUBO: in LowerOperation()
2276 case ISD::SMULO: in LowerOperation()
2277 case ISD::UMULO: in LowerOperation()
2279 case ISD::FADD: in LowerOperation()
2281 case ISD::FSUB: in LowerOperation()
2283 case ISD::FMUL: in LowerOperation()
2285 case ISD::FDIV: in LowerOperation()
2287 case ISD::FP_ROUND: in LowerOperation()
2289 case ISD::FP_EXTEND: in LowerOperation()
2291 case ISD::FRAMEADDR: in LowerOperation()
2293 case ISD::RETURNADDR: in LowerOperation()
2295 case ISD::INSERT_VECTOR_ELT: in LowerOperation()
2297 case ISD::EXTRACT_VECTOR_ELT: in LowerOperation()
2299 case ISD::BUILD_VECTOR: in LowerOperation()
2301 case ISD::VECTOR_SHUFFLE: in LowerOperation()
2303 case ISD::EXTRACT_SUBVECTOR: in LowerOperation()
2305 case ISD::SRA: in LowerOperation()
2306 case ISD::SRL: in LowerOperation()
2307 case ISD::SHL: in LowerOperation()
2309 case ISD::SHL_PARTS: in LowerOperation()
2311 case ISD::SRL_PARTS: in LowerOperation()
2312 case ISD::SRA_PARTS: in LowerOperation()
2314 case ISD::CTPOP: in LowerOperation()
2316 case ISD::FCOPYSIGN: in LowerOperation()
2318 case ISD::AND: in LowerOperation()
2320 case ISD::OR: in LowerOperation()
2322 case ISD::XOR: in LowerOperation()
2324 case ISD::PREFETCH: in LowerOperation()
2326 case ISD::SINT_TO_FP: in LowerOperation()
2327 case ISD::UINT_TO_FP: in LowerOperation()
2329 case ISD::FP_TO_SINT: in LowerOperation()
2330 case ISD::FP_TO_UINT: in LowerOperation()
2332 case ISD::FSINCOS: in LowerOperation()
2334 case ISD::MUL: in LowerOperation()
2336 case ISD::INTRINSIC_WO_CHAIN: in LowerOperation()
2367 const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc DL, SelectionDAG &DAG, in LowerFormalArguments()
2465 ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue); in LowerFormalArguments()
2495 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD; in LowerFormalArguments()
2505 ExtType = ISD::SEXTLOAD; in LowerFormalArguments()
2508 ExtType = ISD::ZEXTLOAD; in LowerFormalArguments()
2511 ExtType = ISD::EXTLOAD; in LowerFormalArguments()
2597 DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getConstant(8, DL, PtrVT)); in saveVarArgRegisters()
2626 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, in saveVarArgRegisters()
2635 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in saveVarArgRegisters()
2643 const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc DL, SelectionDAG &DAG, in LowerCallResult()
2679 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); in LowerCallResult()
2692 const SmallVectorImpl<ISD::OutputArg> &Outs, in isEligibleForTailCallOptimization()
2694 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const { in isEligibleForTailCallOptimization()
2839 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in addTokenForArgument()
2858 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs; in LowerCall()
2860 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins; in LowerCall()
2905 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; in LowerCall()
2927 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; in LowerCall()
2996 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags; in LowerCall()
3005 Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
3008 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
3013 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); in LowerCall()
3014 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg); in LowerCall()
3016 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
3019 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg); in LowerCall()
3022 Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
3055 PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff); in LowerCall()
3072 DstAddr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff); in LowerCall()
3093 Arg = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Arg); in LowerCall()
3103 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
3221 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const { in CanLowerReturn()
3233 const SmallVectorImpl<ISD::OutputArg> &Outs, in LowerReturn()
3261 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); in LowerReturn()
3262 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); in LowerReturn()
3266 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg); in LowerReturn()
3336 return DAG.getNode(ISD::ADD, DL, PtrVT, GlobalAddr, in LowerGlobalAddress()
3565 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff); in LowerELFGlobalTLSAddress()
3579 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); in LowerBR_CC()
3595 CC = ISD::SETNE; in LowerBR_CC()
3603 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || in LowerBR_CC()
3604 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)) { in LowerBR_CC()
3605 assert((CC == ISD::SETEQ || CC == ISD::SETNE) && in LowerBR_CC()
3616 if (CC == ISD::SETNE) in LowerBR_CC()
3632 if (CC == ISD::SETEQ) { in LowerBR_CC()
3637 if (LHS.getOpcode() == ISD::AND && in LowerBR_CC()
3648 } else if (CC == ISD::SETNE) { in LowerBR_CC()
3653 if (LHS.getOpcode() == ISD::AND && in LowerBR_CC()
3664 } else if (CC == ISD::SETLT && LHS.getOpcode() != ISD::AND) { in LowerBR_CC()
3673 if (RHSC && RHSC->getSExtValue() == -1 && CC == ISD::SETGT && in LowerBR_CC()
3674 LHS.getOpcode() != ISD::AND) { in LowerBR_CC()
3718 In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2); in LowerFCOPYSIGN()
3720 In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL)); in LowerFCOPYSIGN()
3737 VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1); in LowerFCOPYSIGN()
3738 VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2); in LowerFCOPYSIGN()
3755 VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1); in LowerFCOPYSIGN()
3756 VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2); in LowerFCOPYSIGN()
3767 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, BuildVec); in LowerFCOPYSIGN()
3768 BuildVec = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, BuildVec); in LowerFCOPYSIGN()
3769 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, BuildVec); in LowerFCOPYSIGN()
3780 return DAG.getNode(ISD::BITCAST, DL, VT, Sel); in LowerFCOPYSIGN()
3804 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val); in LowerCTPOP()
3805 Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val); in LowerCTPOP()
3807 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val); in LowerCTPOP()
3809 ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, in LowerCTPOP()
3813 UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV); in LowerCTPOP()
3824 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in LowerSETCC()
3848 getAArch64Cmp(LHS, RHS, ISD::getSetCCInverse(CC, true), CCVal, DAG, dl); in LowerSETCC()
3866 changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, false), CC1, CC2); in LowerSETCC()
3889 SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, in LowerSELECT_CC()
3902 CC = ISD::SETNE; in LowerSELECT_CC()
3908 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS); in LowerSELECT_CC()
3909 RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS); in LowerSELECT_CC()
3927 CC = ISD::getSetCCInverse(CC, true); in LowerSELECT_CC()
3931 CC = ISD::getSetCCInverse(CC, true); in LowerSELECT_CC()
3932 } else if (TVal.getOpcode() == ISD::XOR) { in LowerSELECT_CC()
3938 CC = ISD::getSetCCInverse(CC, true); in LowerSELECT_CC()
3940 } else if (TVal.getOpcode() == ISD::SUB) { in LowerSELECT_CC()
3946 CC = ISD::getSetCCInverse(CC, true); in LowerSELECT_CC()
3989 CC = ISD::getSetCCInverse(CC, true); in LowerSELECT_CC()
4032 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); in LowerSELECT_CC()
4052 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || in LowerSELECT()
4053 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)) { in LowerSELECT()
4068 ISD::CondCode CC; in LowerSELECT()
4070 if (CCVal.getOpcode() == ISD::SETCC) { in LowerSELECT()
4077 CC = ISD::SETNE; in LowerSELECT()
4212 DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(8, DL, PtrVT)); in LowerAAPCS_VASTART()
4215 GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop, in LowerAAPCS_VASTART()
4226 VRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerAAPCS_VASTART()
4230 VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop, in LowerAAPCS_VASTART()
4239 DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(24, DL, PtrVT)); in LowerAAPCS_VASTART()
4247 DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(28, DL, PtrVT)); in LowerAAPCS_VASTART()
4253 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerAAPCS_VASTART()
4296 VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerVAARG()
4298 VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList, in LowerVAARG()
4318 SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerVAARG()
4330 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0), in LowerVAARG()
4383 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), in LowerRETURNADDR()
4403 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL; in LowerShiftRightParts()
4405 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS); in LowerShiftRightParts()
4407 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, in LowerShiftRightParts()
4409 SDValue HiBitsForLo = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt); in LowerShiftRightParts()
4414 ISD::SETEQ, dl, DAG); in LowerShiftRightParts()
4420 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt, in LowerShiftRightParts()
4423 SDValue LoBitsForLo = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt); in LowerShiftRightParts()
4425 DAG.getNode(ISD::OR, dl, VT, LoBitsForLo, HiBitsForLo); in LowerShiftRightParts()
4427 Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE, in LowerShiftRightParts()
4438 Opc == ISD::SRA in LowerShiftRightParts()
4462 assert(Op.getOpcode() == ISD::SHL_PARTS); in LowerShiftLeftParts()
4463 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, in LowerShiftLeftParts()
4465 SDValue LoBitsForHi = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt); in LowerShiftLeftParts()
4470 ISD::SETEQ, dl, DAG); in LowerShiftLeftParts()
4476 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt, in LowerShiftLeftParts()
4478 SDValue HiBitsForHi = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt); in LowerShiftLeftParts()
4480 DAG.getNode(ISD::OR, dl, VT, LoBitsForHi, HiBitsForHi); in LowerShiftLeftParts()
4482 SDValue HiForBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt); in LowerShiftLeftParts()
4484 Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE, in LowerShiftLeftParts()
4493 SDValue LoForNormalShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); in LowerShiftLeftParts()
4807 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideTy, DAG.getUNDEF(WideTy), in WidenVector()
4834 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); in ReconstructShuffle()
4865 if (V.getOpcode() == ISD::UNDEF) in ReconstructShuffle()
4867 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) { in ReconstructShuffle()
4924 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
4939 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
4945 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
4950 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
4953 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
4972 Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec); in ReconstructShuffle()
4988 if (Entry.getOpcode() == ISD::UNDEF) in ReconstructShuffle()
5022 return DAG.getNode(ISD::BITCAST, dl, VT, Shuffle); in ReconstructShuffle()
5293 V0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V0, in tryFormConcatFromShuffle()
5297 V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V1, in tryFormConcatFromShuffle()
5300 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1); in tryFormConcatFromShuffle()
5432 SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1); in GenerateTBL()
5433 SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2); in GenerateTBL()
5436 if (V2.getNode()->getOpcode() == ISD::UNDEF) { in GenerateTBL()
5438 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V1Cst); in GenerateTBL()
5440 ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, in GenerateTBL()
5442 DAG.getNode(ISD::BUILD_VECTOR, DL, IndexVT, in GenerateTBL()
5446 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V2Cst); in GenerateTBL()
5448 ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, in GenerateTBL()
5450 DAG.getNode(ISD::BUILD_VECTOR, DL, IndexVT, in GenerateTBL()
5460 ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, in GenerateTBL()
5463 DAG.getNode(ISD::BUILD_VECTOR, DL, IndexVT, in GenerateTBL()
5467 return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle); in GenerateTBL()
5506 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) in LowerVECTOR_SHUFFLE()
5511 if (V1.getOpcode() == ISD::BUILD_VECTOR && in LowerVECTOR_SHUFFLE()
5521 if (V1.getOpcode() == ISD::EXTRACT_SUBVECTOR) { in LowerVECTOR_SHUFFLE()
5524 } else if (V1.getOpcode() == ISD::CONCAT_VECTORS) { in LowerVECTOR_SHUFFLE()
5549 } else if (V2->getOpcode() == ISD::UNDEF && in LowerVECTOR_SHUFFLE()
5608 ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, in LowerVECTOR_SHUFFLE()
5609 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, SrcVec, SrcLaneV), in LowerVECTOR_SHUFFLE()
5778 case ISD::INTRINSIC_WO_CHAIN: { in getIntrinsicID()
5801 if (And.getOpcode() != ISD::AND) in tryLowerToSLI()
5839 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in tryLowerToSLI()
5958 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); in NormalizeBuildVector()
5975 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); in NormalizeBuildVector()
6220 if (V.getOpcode() == ISD::UNDEF) in LowerBUILD_VECTOR()
6245 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); in LowerBUILD_VECTOR()
6251 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in LowerBUILD_VECTOR()
6274 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, NewType, Op.getOperand(i))); in LowerBUILD_VECTOR()
6276 SDValue Val = DAG.getNode(ISD::BUILD_VECTOR, dl, VecVT, Ops); in LowerBUILD_VECTOR()
6279 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
6296 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx); in LowerBUILD_VECTOR()
6331 if (Op0.getOpcode() != ISD::UNDEF && Op0.getOpcode() != ISD::LOAD && in LowerBUILD_VECTOR()
6342 if (V.getOpcode() == ISD::UNDEF) in LowerBUILD_VECTOR()
6345 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); in LowerBUILD_VECTOR()
6356 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && "Unknown opcode!"); in LowerINSERT_VECTOR_ELT()
6381 SDValue Node = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideTy, WideVec, in LowerINSERT_VECTOR_ELT()
6390 assert(Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unknown opcode!"); in LowerEXTRACT_VECTOR_ELT()
6420 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtrTy, WideVec, in LowerEXTRACT_VECTOR_ELT()
6495 while (Op.getOpcode() == ISD::BITCAST) in getVShiftImm()
6546 case ISD::SHL: in LowerVectorSRA_SRL_SHL()
6550 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
6554 case ISD::SRA: in LowerVectorSRA_SRL_SHL()
6555 case ISD::SRL: in LowerVectorSRA_SRL_SHL()
6559 (Op.getOpcode() == ISD::SRA) ? AArch64ISD::VASHR : AArch64ISD::VLSHR; in LowerVectorSRA_SRL_SHL()
6567 unsigned Opc = (Op.getOpcode() == ISD::SRA) ? Intrinsic::aarch64_neon_sshl in LowerVectorSRA_SRL_SHL()
6572 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
6678 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in LowerVSETCC()
6716 Cmp = DAG.getNode(ISD::OR, dl, CmpVT, Cmp, Cmp2); in LowerVSETCC()
6747 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
6768 Info.opc = ISD::INTRINSIC_VOID; in getTgtMemIntrinsic()
6789 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
6802 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
6814 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
6826 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
6881 isOperationLegalOrCustom(ISD::FMA, VT) && in isProfitableToHoist()
6911 if (Val.getOpcode() != ISD::LOAD) in isZExtFree()
7303 if (N->getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) && in isDesirableToCommuteWithShift()
7307 N->getOperand(0).getOpcode() == ISD::SRL && in isDesirableToCommuteWithShift()
7347 if (VT.isInteger() && N->getOpcode() == ISD::XOR && in performIntegerAbsCombine()
7348 N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1 && in performIntegerAbsCombine()
7349 N1.getOpcode() == ISD::SRA && N1.getOperand(0) == N0.getOperand(0)) in performIntegerAbsCombine()
7352 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in performIntegerAbsCombine()
7393 SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETLT, CCVal, DAG, DL); in BuildSDIVPow2()
7394 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in BuildSDIVPow2()
7405 DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64)); in BuildSDIVPow2()
7414 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA); in BuildSDIVPow2()
7437 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in performMulCombine()
7439 return DAG.getNode(ISD::ADD, DL, VT, ShiftedVal, in performMulCombine()
7446 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in performMulCombine()
7448 return DAG.getNode(ISD::SUB, DL, VT, ShiftedVal, in performMulCombine()
7456 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in performMulCombine()
7458 return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), in performMulCombine()
7465 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in performMulCombine()
7468 DAG.getNode(ISD::ADD, DL, VT, ShiftedVal, N->getOperand(0)); in performMulCombine()
7469 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Add); in performMulCombine()
7490 if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND || in performVectorCompareAndMaskUnaryOpCombine()
7491 N->getOperand(0)->getOperand(0)->getOpcode() != ISD::SETCC || in performVectorCompareAndMaskUnaryOpCombine()
7512 SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst); in performVectorCompareAndMaskUnaryOpCombine()
7513 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, in performVectorCompareAndMaskUnaryOpCombine()
7515 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd); in performVectorCompareAndMaskUnaryOpCombine()
7541 if (Subtarget->hasNEON() && ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in performIntToFpCombine()
7555 (N->getOpcode() == ISD::SINT_TO_FP) ? AArch64ISD::SITOF : AArch64ISD::UITOF; in performIntToFpCombine()
7570 if (!Op.getValueType().isVector() || Op.getOpcode() != ISD::FMUL) in performFpToIntCombine()
7612 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT; in performFpToIntCombine()
7616 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResTy, in performFpToIntCombine()
7621 FixConv = DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), FixConv); in performFpToIntCombine()
7636 (Opc != ISD::SINT_TO_FP && Opc != ISD::UINT_TO_FP)) in performFDivCombine()
7678 bool IsSigned = Opc == ISD::SINT_TO_FP; in performFDivCombine()
7680 ConvInput = DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, in performFDivCombine()
7685 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(), in performFDivCombine()
7694 if (N.getOpcode() == ISD::SHL) in findEXTRHalf()
7696 else if (N.getOpcode() == ISD::SRL) in findEXTRHalf()
7720 assert(N->getOpcode() == ISD::OR && "Unexpected root"); in tryCombineToEXTR()
7764 if (N0.getOpcode() != ISD::AND) in tryCombineToBSL()
7768 if (N1.getOpcode() != ISD::AND) in tryCombineToBSL()
7848 if (Op0->getOpcode() != ISD::EXTRACT_SUBVECTOR && in performBitcastCombine()
7853 if (Op0->getOpcode() == ISD::EXTRACT_SUBVECTOR) { in performBitcastCombine()
7863 if (Op0->getOperand(0)->getOpcode() != ISD::BITCAST) in performBitcastCombine()
7880 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Source, HalfIdx); in performBitcastCombine()
7908 N0->getOpcode() == ISD::TRUNCATE && in performConcatVectorsCombine()
7909 N1->getOpcode() == ISD::TRUNCATE) { in performConcatVectorsCombine()
7921 return DAG.getNode(ISD::TRUNCATE, dl, VT, in performConcatVectorsCombine()
7924 DAG.getNode(ISD::BITCAST, dl, MidVT, N00), in performConcatVectorsCombine()
7925 DAG.getNode(ISD::BITCAST, dl, MidVT, N10), Mask)); in performConcatVectorsCombine()
7952 if (N1->getOpcode() != ISD::BITCAST) in performConcatVectorsCombine()
7964 return DAG.getNode(ISD::BITCAST, dl, VT, in performConcatVectorsCombine()
7965 DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatTy, in performConcatVectorsCombine()
7966 DAG.getNode(ISD::BITCAST, dl, RHSTy, N0), in performConcatVectorsCombine()
7988 if (Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in tryCombineFixedPointConvert()
8011 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VecResTy, IID, Vec, Shift); in tryCombineFixedPointConvert()
8012 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResTy, Convert, Lane); in tryCombineFixedPointConvert()
8062 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NarrowTy, in tryExtendDUPToExtractHigh()
8068 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) in isEssentiallyExtractSubvector()
8071 return N.getOpcode() == ISD::BITCAST && in isEssentiallyExtractSubvector()
8072 N.getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR; in isEssentiallyExtractSubvector()
8079 ISD::CondCode CC;
8110 if (Op.getOpcode() == ISD::SETCC) { in isSetCC()
8154 return ((Op.getOpcode() == ISD::ZERO_EXTEND) && in isSetCCOrZExtSetCC()
8165 assert(Op && Op->getOpcode() == ISD::ADD && "Unexpected operation!"); in performSetccAddFolding()
8195 ISD::getSetCCInverse(InfoAndKind.Info.Generic.CC, true), in performSetccAddFolding()
8199 LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT)); in performSetccAddFolding()
8222 if (N->getOpcode() == ISD::ADD) in performAddSubLongCombine()
8230 if ((LHS.getOpcode() != ISD::ZERO_EXTEND && in performAddSubLongCombine()
8231 LHS.getOpcode() != ISD::SIGN_EXTEND) || in performAddSubLongCombine()
8288 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), N->getValueType(0), in tryCombineLongOpWithDup()
8357 if (AndN.getOpcode() != ISD::AND) in tryCombineCRC32()
8364 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), MVT::i32, in tryCombineCRC32()
8371 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0), in combineAcrossLanesIntrinsic()
8402 return DAG.getNode(ISD::FMAXNAN, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
8405 return DAG.getNode(ISD::FMINNAN, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
8408 return DAG.getNode(ISD::FMAXNUM, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
8411 return DAG.getNode(ISD::FMINNUM, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
8441 if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ZERO_EXTEND && in performExtendCombine()
8442 N->getOperand(0).getOpcode() == ISD::INTRINSIC_WO_CHAIN) { in performExtendCombine()
8451 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), in performExtendCombine()
8520 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, Src, in performExtendCombine()
8522 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, Src, in performExtendCombine()
8529 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); in performExtendCombine()
8547 if (StVal.getOpcode() != ISD::INSERT_VECTOR_ELT) in replaceSplatVectorStore()
8560 if (NextInsertElt.getOpcode() != ISD::INSERT_VECTOR_ELT) in replaceSplatVectorStore()
8582 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, in replaceSplatVectorStore()
8643 SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal, in split16BStores()
8645 SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal, in split16BStores()
8651 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, in split16BStores()
8672 if (LD->getOpcode() != ISD::LOAD) in performPostLD1Combine()
8697 if (User->getOpcode() != ISD::ADD in performPostLD1Combine()
8801 if (Op == ISD::FMAXNUM || Op == ISD::FMINNUM) { in tryMatchAcrossLaneShuffleForReduction()
8815 if (Shuffle.getOpcode() != ISD::VECTOR_SHUFFLE) { in tryMatchAcrossLaneShuffleForReduction()
8820 if (Shuffle.getOpcode() != ISD::VECTOR_SHUFFLE) in tryMatchAcrossLaneShuffleForReduction()
8863 case ISD::ADD: in tryMatchAcrossLaneShuffleForReduction()
8866 case ISD::SMAX: in tryMatchAcrossLaneShuffleForReduction()
8869 case ISD::UMAX: in tryMatchAcrossLaneShuffleForReduction()
8872 case ISD::SMIN: in tryMatchAcrossLaneShuffleForReduction()
8875 case ISD::UMIN: in tryMatchAcrossLaneShuffleForReduction()
8878 case ISD::FMAXNUM: in tryMatchAcrossLaneShuffleForReduction()
8882 case ISD::FMINNUM: in tryMatchAcrossLaneShuffleForReduction()
8890 ? DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0), in tryMatchAcrossLaneShuffleForReduction()
8893 ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), in tryMatchAcrossLaneShuffleForReduction()
8931 if (N0.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in performAcrossLaneMinMaxReductionCombine()
8932 IfTrue.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in performAcrossLaneMinMaxReductionCombine()
8933 IfFalse.getOpcode() != ISD::EXTRACT_VECTOR_ELT) in performAcrossLaneMinMaxReductionCombine()
8939 if (SetCC.getOpcode() != ISD::SETCC || !SetCCVT.isVector() || in performAcrossLaneMinMaxReductionCombine()
8946 if (Op != ISD::SMAX && Op != ISD::UMAX && Op != ISD::SMIN && in performAcrossLaneMinMaxReductionCombine()
8947 Op != ISD::UMIN && Op != ISD::FMAXNUM && Op != ISD::FMINNUM) in performAcrossLaneMinMaxReductionCombine()
8958 if (Op == ISD::FMAXNUM || Op == ISD::FMINNUM) { in performAcrossLaneMinMaxReductionCombine()
8977 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get(); in performAcrossLaneMinMaxReductionCombine()
8978 if ((Op == ISD::SMAX && CC != ISD::SETGT && CC != ISD::SETGE) || in performAcrossLaneMinMaxReductionCombine()
8979 (Op == ISD::UMAX && CC != ISD::SETUGT && CC != ISD::SETUGE) || in performAcrossLaneMinMaxReductionCombine()
8980 (Op == ISD::SMIN && CC != ISD::SETLT && CC != ISD::SETLE) || in performAcrossLaneMinMaxReductionCombine()
8981 (Op == ISD::UMIN && CC != ISD::SETULT && CC != ISD::SETULE) || in performAcrossLaneMinMaxReductionCombine()
8982 (Op == ISD::FMAXNUM && CC != ISD::SETOGT && CC != ISD::SETOGE && in performAcrossLaneMinMaxReductionCombine()
8983 CC != ISD::SETUGT && CC != ISD::SETUGE && CC != ISD::SETGT && in performAcrossLaneMinMaxReductionCombine()
8984 CC != ISD::SETGE) || in performAcrossLaneMinMaxReductionCombine()
8985 (Op == ISD::FMINNUM && CC != ISD::SETOLT && CC != ISD::SETOLE && in performAcrossLaneMinMaxReductionCombine()
8986 CC != ISD::SETULT && CC != ISD::SETULE && CC != ISD::SETLT && in performAcrossLaneMinMaxReductionCombine()
8987 CC != ISD::SETLE)) in performAcrossLaneMinMaxReductionCombine()
9027 if (N0->getOpcode() != ISD::ADD) in performAcrossLaneAddReductionCombine()
9046 return tryMatchAcrossLaneShuffleForReduction(N, N0, ISD::ADD, DAG); in performAcrossLaneAddReductionCombine()
9064 if (User->getOpcode() != ISD::ADD || in performNEONPostLDSTCombine()
9184 bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType) { in checkValueWidth()
9185 ExtType = ISD::NON_EXTLOAD; in checkValueWidth()
9189 case ISD::LOAD: { in checkValueWidth()
9198 case ISD::AssertSext: { in checkValueWidth()
9202 ExtType = ISD::SEXTLOAD; in checkValueWidth()
9207 case ISD::AssertZext: { in checkValueWidth()
9211 ExtType = ISD::ZEXTLOAD; in checkValueWidth()
9216 case ISD::Constant: in checkValueWidth()
9217 case ISD::TargetConstant: { in checkValueWidth()
9291 ISD::LoadExtType ExtType, signed AddConstant, in isEquivalentMaskless()
9303 if (ExtType == ISD::SEXTLOAD) in isEquivalentMaskless()
9384 if (AndNode->getOpcode() != ISD::AND) in performCONDCombine()
9400 if (AddValue.getOpcode() != ISD::ADD) in performCONDCombine()
9416 ISD::LoadExtType ExtType; in performCONDCombine()
9480 if (LHS.getOpcode() == ISD::SHL || LHS.getOpcode() == ISD::SRA || in performBRCONDCombine()
9481 LHS.getOpcode() == ISD::SRL) in performBRCONDCombine()
9506 if (N0.getOpcode() != ISD::SETCC || CCVT.getVectorNumElements() != 1 || in performVSelectCombine()
9523 return DAG.getNode(ISD::VSELECT, SDLoc(N), ResVT, SetCC, in performVSelectCombine()
9537 if (N0.getOpcode() != ISD::SETCC) in performSelectCombine()
9577 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(0)); in performSelectCombine()
9579 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(1)); in performSelectCombine()
9580 SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2)); in performSelectCombine()
9585 Mask = DAG.getNode(ISD::BITCAST, DL, in performSelectCombine()
9605 case ISD::ADD: in PerformDAGCombine()
9606 case ISD::SUB: in PerformDAGCombine()
9608 case ISD::XOR: in PerformDAGCombine()
9610 case ISD::MUL: in PerformDAGCombine()
9612 case ISD::SINT_TO_FP: in PerformDAGCombine()
9613 case ISD::UINT_TO_FP: in PerformDAGCombine()
9615 case ISD::FP_TO_SINT: in PerformDAGCombine()
9616 case ISD::FP_TO_UINT: in PerformDAGCombine()
9618 case ISD::FDIV: in PerformDAGCombine()
9620 case ISD::OR: in PerformDAGCombine()
9622 case ISD::INTRINSIC_WO_CHAIN: in PerformDAGCombine()
9624 case ISD::ANY_EXTEND: in PerformDAGCombine()
9625 case ISD::ZERO_EXTEND: in PerformDAGCombine()
9626 case ISD::SIGN_EXTEND: in PerformDAGCombine()
9628 case ISD::BITCAST: in PerformDAGCombine()
9630 case ISD::CONCAT_VECTORS: in PerformDAGCombine()
9632 case ISD::SELECT: { in PerformDAGCombine()
9638 case ISD::VSELECT: in PerformDAGCombine()
9640 case ISD::LOAD: in PerformDAGCombine()
9644 case ISD::STORE: in PerformDAGCombine()
9654 case ISD::INSERT_VECTOR_ELT: in PerformDAGCombine()
9656 case ISD::EXTRACT_VECTOR_ELT: in PerformDAGCombine()
9658 case ISD::INTRINSIC_VOID: in PerformDAGCombine()
9659 case ISD::INTRINSIC_W_CHAIN: in PerformDAGCombine()
9703 if (Copy->getOpcode() == ISD::CopyToReg) { in isUsedByReturnOnly()
9710 } else if (Copy->getOpcode() != ISD::FP_EXTEND) in isUsedByReturnOnly()
9740 ISD::MemIndexedMode &AM, in getIndexedAddressParts()
9743 if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB) in getIndexedAddressParts()
9753 IsInc = (Op->getOpcode() == ISD::ADD); in getIndexedAddressParts()
9762 ISD::MemIndexedMode &AM, in getPreIndexedAddressParts()
9778 AM = IsInc ? ISD::PRE_INC : ISD::PRE_DEC; in getPreIndexedAddressParts()
9784 ISD::MemIndexedMode &AM, SelectionDAG &DAG) const { in getPostIndexedAddressParts()
9803 AM = IsInc ? ISD::POST_INC : ISD::POST_DEC; in getPostIndexedAddressParts()
9820 Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op); in ReplaceBITCASTResults()
9821 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op)); in ReplaceBITCASTResults()
9843 case ISD::BITCAST: in ReplaceNodeResults()
9847 ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::SADDV); in ReplaceNodeResults()
9850 ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::UADDV); in ReplaceNodeResults()
9853 ReplaceReductionResults(N, Results, DAG, ISD::SMIN, AArch64ISD::SMINV); in ReplaceNodeResults()
9856 ReplaceReductionResults(N, Results, DAG, ISD::UMIN, AArch64ISD::UMINV); in ReplaceNodeResults()
9859 ReplaceReductionResults(N, Results, DAG, ISD::SMAX, AArch64ISD::SMAXV); in ReplaceNodeResults()
9862 ReplaceReductionResults(N, Results, DAG, ISD::UMAX, AArch64ISD::UMAXV); in ReplaceNodeResults()
9864 case ISD::FP_TO_UINT: in ReplaceNodeResults()
9865 case ISD::FP_TO_SINT: in ReplaceNodeResults()