Lines Matching +full:d3 +full:- +full:dispatch
1 //===-- ARMISelLowering.cpp - ARM DAG Lowering Implementation -------------===//
8 //===----------------------------------------------------------------------===//
13 //===----------------------------------------------------------------------===//
57 #define DEBUG_TYPE "arm-isel"
64 ARMInterworking("arm-interworking", cl::Hidden,
128 // Promote all bit-wise operations. in addTypeForNEON()
165 RegInfo = Subtarget->getRegisterInfo(); in ARMTargetLowering()
166 Itins = Subtarget->getInstrItineraryData(); in ARMTargetLowering()
170 if (Subtarget->isTargetMachO()) { in ARMTargetLowering()
172 if (Subtarget->isThumb() && Subtarget->hasVFP2() && in ARMTargetLowering()
173 Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) { in ARMTargetLowering()
179 // Single-precision floating-point arithmetic. in ARMTargetLowering()
185 // Double-precision floating-point arithmetic. in ARMTargetLowering()
191 // Single-precision comparisons. in ARMTargetLowering()
201 // Double-precision comparisons. in ARMTargetLowering()
211 // Floating-point to integer conversions. in ARMTargetLowering()
223 // Integer to floating-point conversions. in ARMTargetLowering()
243 if (Subtarget->isTargetWatchABI()) { in ARMTargetLowering()
249 // These libcalls are not available in 32-bit. in ARMTargetLowering()
255 if (Subtarget->isAAPCS_ABI() && in ARMTargetLowering()
256 (Subtarget->isTargetAEABI() || Subtarget->isTargetGNUAEABI() || in ARMTargetLowering()
257 Subtarget->isTargetMuslAEABI() || Subtarget->isTargetAndroid())) { in ARMTargetLowering()
264 // Double-precision floating-point arithmetic helper functions in ARMTargetLowering()
271 // Double-precision floating-point comparison helper functions in ARMTargetLowering()
282 // Single-precision floating-point arithmetic helper functions in ARMTargetLowering()
289 // Single-precision floating-point comparison helper functions in ARMTargetLowering()
300 // Floating-point to integer conversions. in ARMTargetLowering()
317 // Integer to floating-point conversions. in ARMTargetLowering()
379 if (Subtarget->isTargetWindows()) { in ARMTargetLowering()
401 // Use divmod compiler-rt calls for iOS 5.0 and later. in ARMTargetLowering()
402 if (Subtarget->isTargetWatchOS() || in ARMTargetLowering()
403 (Subtarget->isTargetIOS() && in ARMTargetLowering()
404 !Subtarget->getTargetTriple().isOSVersionLT(5, 0))) { in ARMTargetLowering()
409 // The half <-> float conversion functions are always soft-float on in ARMTargetLowering()
410 // non-watchos platforms, but are needed for some targets which use a in ARMTargetLowering()
411 // hard-float calling convention by default. in ARMTargetLowering()
412 if (!Subtarget->isTargetWatchABI()) { in ARMTargetLowering()
413 if (Subtarget->isAAPCS_ABI()) { in ARMTargetLowering()
426 if (Subtarget->isTargetAEABI()) { in ARMTargetLowering()
432 if (Subtarget->isThumb1Only()) in ARMTargetLowering()
436 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() && in ARMTargetLowering()
437 !Subtarget->isThumb1Only()) { in ARMTargetLowering()
464 if (Subtarget->hasNEON()) { in ARMTargetLowering()
552 // Custom handling for some quad-vector types to detect VMULL. in ARMTargetLowering()
576 // types wider than 8-bits. However, custom lowering can leverage the in ARMTargetLowering()
610 if (!Subtarget->hasVFP4()) { in ARMTargetLowering()
645 if (!Subtarget->isThumb1Only()) in ARMTargetLowering()
648 if (Subtarget->isFPOnlySP()) { in ARMTargetLowering()
649 // When targeting a floating-point unit with only single-precision in ARMTargetLowering()
650 // operations, f64 is legal for the few double-precision instructions which in ARMTargetLowering()
651 // are present However, no double-precision operations other than moves, in ARMTargetLowering()
688 computeRegisterProperties(Subtarget->getRegisterInfo()); in ARMTargetLowering()
690 // ARM does not have floating-point extending loads. in ARMTargetLowering()
706 if (!Subtarget->isThumb1Only()) { in ARMTargetLowering()
728 if (Subtarget->isThumb1Only()) { in ARMTargetLowering()
732 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops() in ARMTargetLowering()
733 || (Subtarget->isThumb2() && !Subtarget->hasDSP())) in ARMTargetLowering()
742 if (!Subtarget->isThumb1Only()) { in ARMTargetLowering()
750 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) in ARMTargetLowering()
761 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) in ARMTargetLowering()
767 // implementation-specific ways of obtaining this information. in ARMTargetLowering()
768 if (Subtarget->hasPerfMon()) in ARMTargetLowering()
772 if (!Subtarget->hasV6Ops()) in ARMTargetLowering()
775 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivide() in ARMTargetLowering()
776 : Subtarget->hasDivideInARMMode(); in ARMTargetLowering()
783 if (Subtarget->isTargetWindows() && !Subtarget->hasDivide()) { in ARMTargetLowering()
794 if (Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() || in ARMTargetLowering()
795 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI()) { in ARMTargetLowering()
841 if (Subtarget->getTargetTriple().isWindowsItaniumEnvironment()) in ARMTargetLowering()
849 if (Subtarget->hasAnyDataBarrier() && in ARMTargetLowering()
850 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) { in ARMTargetLowering()
854 if (!Subtarget->isThumb() || !Subtarget->isMClass()) in ARMTargetLowering()
859 if (!Subtarget->hasV8Ops() || getTargetMachine().getOptLevel() == 0) { in ARMTargetLowering()
867 Subtarget->hasAnyDataBarrier() ? Custom : Expand); in ARMTargetLowering()
891 if (!Subtarget->hasV6Ops()) { in ARMTargetLowering()
897 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() && in ARMTargetLowering()
898 !Subtarget->isThumb1Only()) { in ARMTargetLowering()
899 // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR in ARMTargetLowering()
910 if (Subtarget->useSjLjEH()) in ARMTargetLowering()
923 // Thumb-1 cannot currently select ARMISD::SUBE. in ARMTargetLowering()
924 if (!Subtarget->isThumb1Only()) in ARMTargetLowering()
942 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() && in ARMTargetLowering()
943 !Subtarget->isThumb1Only()) { in ARMTargetLowering()
950 if (!Subtarget->hasVFP4()) { in ARMTargetLowering()
956 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) { in ARMTargetLowering()
957 // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded. in ARMTargetLowering()
958 if (!Subtarget->hasFPARMv8() || Subtarget->isFPOnlySP()) { in ARMTargetLowering()
963 // fp16 is a special v7 extension that adds f16 <-> f32 conversions. in ARMTargetLowering()
964 if (!Subtarget->hasFP16()) { in ARMTargetLowering()
971 if (Subtarget->hasSinCos()) { in ARMTargetLowering()
974 if (Subtarget->isTargetWatchABI()) { in ARMTargetLowering()
978 if (Subtarget->isTargetIOS() || Subtarget->isTargetWatchOS()) { in ARMTargetLowering()
986 // FP-ARMv8 implements a lot of rounding-like FP operations. in ARMTargetLowering()
987 if (Subtarget->hasFPARMv8()) { in ARMTargetLowering()
1001 if (!Subtarget->isFPOnlySP()) { in ARMTargetLowering()
1013 if (Subtarget->hasNEON()) { in ARMTargetLowering()
1024 // We have target-specific dag combine patterns for the following nodes: in ARMTargetLowering()
1025 // ARMISD::VMOVRRD - No need to call setTargetDAGCombine in ARMTargetLowering()
1033 if (Subtarget->hasV6Ops()) in ARMTargetLowering()
1038 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() || in ARMTargetLowering()
1039 !Subtarget->hasVFP2()) in ARMTargetLowering()
1044 //// temporary - rewrite interface to use type in ARMTargetLowering()
1047 MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores in ARMTargetLowering()
1049 MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores in ARMTargetLowering()
1056 // Prefer likely predicted branches to selects on out-of-order cores. in ARMTargetLowering()
1057 PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder(); in ARMTargetLowering()
1059 setMinFunctionAlignment(Subtarget->isThumb() ? 1 : 2); in ARMTargetLowering()
1063 return Subtarget->useSoftFloat(); in useSoftFloat()
1067 // nearest super-register that has a non-null superset. For example, DPR_VFP2 is
1068 // a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
1092 // to the VFP2 class (D0-D15). We currently model this constraint prior to in findRepresentativeClass()
1093 // coalescing by double-counting the SP regs. See the FIXME above. in findRepresentativeClass()
1094 if (Subtarget->useNEONForSinglePrecisionFP()) in findRepresentativeClass()
1258 /// getRegClassFor - Return the register class that should be used for the
1264 if (Subtarget->hasNEON()) { in getRegClassFor()
1281 // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1 in shouldAlignPointerArgs()
1282 // cycle faster than 4-byte aligned LDM. in shouldAlignPointerArgs()
1283 PrefAlign = (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? 8 : 4); in shouldAlignPointerArgs()
1295 unsigned NumVals = N->getNumValues(); in getSchedulingPreference()
1300 EVT VT = N->getValueType(i); in getSchedulingPreference()
1307 if (!N->isMachineOpcode()) in getSchedulingPreference()
1312 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in getSchedulingPreference()
1313 const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); in getSchedulingPreference()
1317 if (!Itins->isEmpty() && in getSchedulingPreference()
1318 Itins->getOperandCycle(MCID.getSchedClass(), 0) > 2) in getSchedulingPreference()
1324 //===----------------------------------------------------------------------===//
1326 //===----------------------------------------------------------------------===//
1328 /// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
1345 /// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
1374 //===----------------------------------------------------------------------===//
1376 //===----------------------------------------------------------------------===//
1380 /// getEffectiveCallingConv - Get the effective calling convention, taking into
1399 if (!Subtarget->isAAPCS_ABI()) in getEffectiveCallingConv()
1401 else if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && in getEffectiveCallingConv()
1409 if (!Subtarget->isAAPCS_ABI()) { in getEffectiveCallingConv()
1410 if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && !isVarArg) in getEffectiveCallingConv()
1413 } else if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && !isVarArg) in getEffectiveCallingConv()
1420 /// CCAssignFnForNode - Selects the correct CCAssignFn for the given
1443 /// LowerCallResult - Lower the result values of a call into the
1484 if (!Subtarget->isLittle()) in LowerCallResult()
1501 if (!Subtarget->isLittle()) in LowerCallResult()
1528 /// LowerMemOpCallTo - Store the argument to the stack.
1554 unsigned id = Subtarget->isLittle() ? 0 : 1; in PassF64ArgInRegs()
1558 RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), fmrrd.getValue(1-id))); in PassF64ArgInRegs()
1565 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, fmrrd.getValue(1-id), in PassF64ArgInRegs()
1571 /// LowerCall - Lowering a call into a callseq_start <-
1572 /// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
1593 auto Attr = MF.getFunction()->getFnAttribute("disable-tail-calls"); in LowerCall()
1596 if (!Subtarget->supportsTailCall() || Attr.getValueAsString() == "true") in LowerCall()
1602 isVarArg, isStructRet, MF.getFunction()->hasStructRetAttr(), in LowerCall()
1604 if (!isTailCall && CLI.CS && CLI.CS->isMustTailCall()) in LowerCall()
1734 offset = RegEnd - RegBegin; in LowerCall()
1746 SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl, in LowerCall()
1767 // Build a sequence of copy-to-reg nodes chained together with token chain in LowerCall()
1804 const Module *Mod = MF.getFunction()->getParent(); in LowerCall()
1807 GV = G->getGlobal(); in LowerCall()
1809 !TM.shouldAssumeDSOLocal(*Mod, GV) && Subtarget->isTargetMachO(); in LowerCall()
1811 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass()); in LowerCall()
1816 if (Subtarget->genLongCalls()) { in LowerCall()
1817 assert((!isPositionIndependent() || Subtarget->isTargetWindows()) && in LowerCall()
1818 "long-calls codegen is not position independent!"); in LowerCall()
1824 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerCall()
1836 const char *Sym = S->getSymbol(); in LowerCall()
1839 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerCall()
1853 bool isDef = GV->isStrongDefinitionForLinker(); in LowerCall()
1856 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking); in LowerCall()
1858 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) { in LowerCall()
1859 assert(Subtarget->isTargetMachO() && "WrapperPIC use on non-MachO?"); in LowerCall()
1866 } else if (Subtarget->isTargetCOFF()) { in LowerCall()
1867 assert(Subtarget->isTargetWindows() && in LowerCall()
1869 unsigned TargetFlags = GV->hasDLLImportStorageClass() in LowerCall()
1874 if (GV->hasDLLImportStorageClass()) in LowerCall()
1886 const char *Sym = S->getSymbol(); in LowerCall()
1887 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) { in LowerCall()
1888 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerCall()
1907 if (Subtarget->isThumb()) { in LowerCall()
1908 if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps()) in LowerCall()
1913 if (!isDirect && !Subtarget->hasV5TOps()) in LowerCall()
1915 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() && in LowerCall()
1917 !MF.getFunction()->optForMinSize()) in LowerCall()
1934 // Add a register mask operand representing the call-preserved registers. in LowerCall()
1937 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo(); in LowerCall()
1939 // For 'this' returns, use the R0-preserving mask if applicable in LowerCall()
1940 Mask = ARI->getThisReturnPreservedMask(MF, CallConv); in LowerCall()
1946 Mask = ARI->getCallPreservedMask(MF, CallConv); in LowerCall()
1949 Mask = ARI->getCallPreservedMask(MF, CallConv); in LowerCall()
1960 MF.getFrameInfo()->setHasTailCall(); in LowerCall()
1980 /// HandleByVal - Every parameter *after* a byval parameter is passed
1986 assert((State->getCallOrPrologue() == Prologue || in HandleByVal()
1987 State->getCallOrPrologue() == Call) && in HandleByVal()
1993 unsigned Reg = State->AllocateReg(GPRArgRegs); in HandleByVal()
1998 unsigned Waste = (ARM::R4 - Reg) % AlignInRegs; in HandleByVal()
2000 Reg = State->AllocateReg(GPRArgRegs); in HandleByVal()
2005 unsigned Excess = 4 * (ARM::R4 - Reg); in HandleByVal()
2011 const unsigned NSAAOffset = State->getNextStackOffset(); in HandleByVal()
2013 while (State->AllocateReg(GPRArgRegs)) in HandleByVal()
2021 // the end (first after last) register would be reg + param-size-in-regs, in HandleByVal()
2026 State->addInRegsParamInfo(ByValRegBegin, ByValRegEnd); in HandleByVal()
2030 State->AllocateReg(GPRArgRegs); in HandleByVal()
2035 Size = std::max<int>(Size - Excess, 0); in HandleByVal()
2038 /// MatchingStackOffset - Return true if the given stack call argument is
2048 unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg(); in MatchingStackOffset()
2051 MachineInstr *Def = MRI->getVRegDef(VR); in MatchingStackOffset()
2055 if (!TII->isLoadFromStackSlot(*Def, FI)) in MatchingStackOffset()
2068 SDValue Ptr = Ld->getBasePtr(); in MatchingStackOffset()
2072 FI = FINode->getIndex(); in MatchingStackOffset()
2077 if (!MFI->isFixedObjectIndex(FI)) in MatchingStackOffset()
2079 return Offset == MFI->getObjectOffset(FI) && Bytes == MFI->getObjectSize(FI); in MatchingStackOffset()
2082 /// IsEligibleForTailCallOptimization - Check whether the call is eligible
2097 CallingConv::ID CallerCC = CallerF->getCallingConv(); in IsEligibleForTailCallOptimization()
2099 assert(Subtarget->supportsTailCall()); in IsEligibleForTailCallOptimization()
2109 // Exception-handling functions need a special set of instructions to indicate in IsEligibleForTailCallOptimization()
2110 // a return to the hardware. Tail-calling another function would probably in IsEligibleForTailCallOptimization()
2112 if (CallerF->hasFnAttribute("interrupt")) in IsEligibleForTailCallOptimization()
2120 // Externally-defined functions with weak linkage should not be in IsEligibleForTailCallOptimization()
2121 // tail-called on ARM when the OS does not support dynamic in IsEligibleForTailCallOptimization()
2122 // pre-emption of symbols, as the AAELF spec requires normal calls in IsEligibleForTailCallOptimization()
2125 // situation (as used for tail calls) is implementation-defined, so we in IsEligibleForTailCallOptimization()
2128 const GlobalValue *GV = G->getGlobal(); in IsEligibleForTailCallOptimization()
2130 if (GV->hasExternalWeakLinkage() && in IsEligibleForTailCallOptimization()
2142 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo(); in IsEligibleForTailCallOptimization()
2143 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC); in IsEligibleForTailCallOptimization()
2145 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC); in IsEligibleForTailCallOptimization()
2146 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) in IsEligibleForTailCallOptimization()
2154 if (AFI_Caller->getArgRegsSaveSize()) in IsEligibleForTailCallOptimization()
2171 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in IsEligibleForTailCallOptimization()
2183 // register/stack-slot combinations. The types will not match in IsEligibleForTailCallOptimization()
2228 StringRef IntKind = F->getFnAttribute("interrupt").getValueAsString(); in LowerInterruptReturn()
2263 // CCValAssign - represent the assignment of the return value to a location. in LowerReturn()
2266 // CCState - Info about the registers and stack slots. in LowerReturn()
2277 bool isLittleEndian = Subtarget->isLittle(); in LowerReturn()
2281 AFI->setReturnRegsCount(RVLocs.size()); in LowerReturn()
2325 // Legalize ret f64 -> ret 2 x i32. We always have fmrrd if f64 is in LowerReturn()
2346 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo(); in LowerReturn()
2348 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); in LowerReturn()
2365 // CPUs which aren't M-class use a special sequence to return from in LowerReturn()
2369 // M-class CPUs actually use a normal return sequence with a special in LowerReturn()
2370 // (hardware-provided) value in LR, so the normal code path works. in LowerReturn()
2371 if (DAG.getMachineFunction().getFunction()->hasFnAttribute("interrupt") && in LowerReturn()
2372 !Subtarget->isMClass()) { in LowerReturn()
2373 if (Subtarget->isThumb1Only()) in LowerReturn()
2382 if (N->getNumValues() != 1) in isUsedByReturnOnly()
2384 if (!N->hasNUsesOfValue(1, 0)) in isUsedByReturnOnly()
2388 SDNode *Copy = *N->use_begin(); in isUsedByReturnOnly()
2389 if (Copy->getOpcode() == ISD::CopyToReg) { in isUsedByReturnOnly()
2392 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) in isUsedByReturnOnly()
2394 TCChain = Copy->getOperand(0); in isUsedByReturnOnly()
2395 } else if (Copy->getOpcode() == ARMISD::VMOVRRD) { in isUsedByReturnOnly()
2399 for (SDNode::use_iterator UI = VMov->use_begin(), UE = VMov->use_end(); in isUsedByReturnOnly()
2401 if (UI->getOpcode() != ISD::CopyToReg) in isUsedByReturnOnly()
2408 for (SDNode::use_iterator UI = VMov->use_begin(), UE = VMov->use_end(); in isUsedByReturnOnly()
2410 SDValue UseChain = UI->getOperand(0); in isUsedByReturnOnly()
2418 if (UI->getOperand(UI->getNumOperands()-1).getValueType() == MVT::Glue) in isUsedByReturnOnly()
2424 } else if (Copy->getOpcode() == ISD::BITCAST) { in isUsedByReturnOnly()
2426 if (!Copy->hasOneUse()) in isUsedByReturnOnly()
2428 Copy = *Copy->use_begin(); in isUsedByReturnOnly()
2429 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(1, 0)) in isUsedByReturnOnly()
2433 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) in isUsedByReturnOnly()
2435 TCChain = Copy->getOperand(0); in isUsedByReturnOnly()
2441 for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end(); in isUsedByReturnOnly()
2443 if (UI->getOpcode() != ARMISD::RET_FLAG && in isUsedByReturnOnly()
2444 UI->getOpcode() != ARMISD::INTRET_FLAG) in isUsedByReturnOnly()
2457 if (!Subtarget->supportsTailCall()) in mayBeEmittedAsTailCall()
2461 CI->getParent()->getParent()->getFnAttribute("disable-tail-calls"); in mayBeEmittedAsTailCall()
2462 if (!CI->isTailCall() || Attr.getValueAsString() == "true") in mayBeEmittedAsTailCall()
2472 SDValue WriteValue = Op->getOperand(2); in LowerWRITE_REGISTER()
2476 && "LowerWRITE_REGISTER called for non-i64 type argument."); in LowerWRITE_REGISTER()
2482 SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi }; in LowerWRITE_REGISTER()
2498 if (CP->isMachineConstantPoolEntry()) in LowerConstantPool()
2499 Res = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, in LowerConstantPool()
2500 CP->getAlignment()); in LowerConstantPool()
2502 Res = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, in LowerConstantPool()
2503 CP->getAlignment()); in LowerConstantPool()
2518 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress(); in LowerBlockAddress()
2524 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8; in LowerBlockAddress()
2525 ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerBlockAddress()
2569 assert(Subtarget->isTargetDarwin() && "TLS only supported on Darwin"); in LowerGlobalTLSAddressDarwin()
2587 MFI->setAdjustsStack(true); in LowerGlobalTLSAddressDarwin()
2593 getTargetMachine().getSubtargetImpl(*F.getFunction())->getRegisterInfo(); in LowerGlobalTLSAddressDarwin()
2595 const uint32_t *Mask = ARI->getTLSCallPreservedMask(DAG.getMachineFunction()); in LowerGlobalTLSAddressDarwin()
2611 assert(Subtarget->isTargetWindows() && "Windows specific TLS lowering"); in LowerGlobalTLSAddressWindows()
2656 auto *CPV = ARMConstantPoolConstant::Create(GA->getGlobal(), ARMCP::SECREL); in LowerGlobalTLSAddressWindows()
2673 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8; in LowerToTLSGeneralDynamicModel()
2676 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerToTLSGeneralDynamicModel()
2678 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex, in LowerToTLSGeneralDynamicModel()
2714 const GlobalValue *GV = GA->getGlobal(); in LowerToTLSExecModels()
2725 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerToTLSExecModels()
2727 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8; in LowerToTLSExecModels()
2729 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex, in LowerToTLSExecModels()
2767 if (Subtarget->isTargetDarwin()) in LowerGlobalTLSAddress()
2770 if (Subtarget->isTargetWindows()) in LowerGlobalTLSAddress()
2774 assert(Subtarget->isTargetELF() && "Only ELF implemented here"); in LowerGlobalTLSAddress()
2779 TLSModel::Model model = getTargetMachine().getTLSModel(GA->getGlobal()); in LowerGlobalTLSAddress()
2796 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); in LowerGlobalAddressELF()
2799 bool UseGOT_PREL = !TM.shouldAssumeDSOLocal(*GV->getParent(), GV); in LowerGlobalAddressELF()
2803 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerGlobalAddressELF()
2806 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8; in LowerGlobalAddressELF()
2829 if (Subtarget->useMovt(DAG.getMachineFunction())) { in LowerGlobalAddressELF()
2849 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); in LowerGlobalAddressDarwin()
2851 if (Subtarget->useMovt(DAG.getMachineFunction())) in LowerGlobalAddressDarwin()
2862 if (Subtarget->isGVIndirectSymbol(GV)) in LowerGlobalAddressDarwin()
2871 assert(Subtarget->isTargetWindows() && "non-Windows COFF is not supported"); in LowerGlobalAddressWindows()
2872 assert(Subtarget->useMovt(DAG.getMachineFunction()) && in LowerGlobalAddressWindows()
2875 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal(); in LowerGlobalAddressWindows()
2877 (GV->hasDLLImportStorageClass() ? ARMII::MO_DLLIMPORT : ARMII::MO_NO_FLAG); in LowerGlobalAddressWindows()
2889 if (GV->hasDLLImportStorageClass()) in LowerGlobalAddressWindows()
2922 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); in LowerINTRINSIC_WO_CHAIN()
2938 unsigned ARMPCLabelIndex = AFI->createPICLabelUId(); in LowerINTRINSIC_WO_CHAIN()
2942 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0; in LowerINTRINSIC_WO_CHAIN()
3003 if (!Subtarget->hasDataBarrier()) { in LowerATOMIC_FENCE()
3005 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get in LowerATOMIC_FENCE()
3007 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() && in LowerATOMIC_FENCE()
3014 AtomicOrdering Ord = static_cast<AtomicOrdering>(OrdN->getZExtValue()); in LowerATOMIC_FENCE()
3016 if (Subtarget->isMClass()) { in LowerATOMIC_FENCE()
3017 // Only a full system barrier exists in the M-class architectures. in LowerATOMIC_FENCE()
3019 } else if (Subtarget->preferISHSTBarriers() && in LowerATOMIC_FENCE()
3035 if (!(Subtarget->isThumb2() || in LowerPREFETCH()
3036 (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps()))) in LowerPREFETCH()
3041 unsigned isRead = ~cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue() & 1; in LowerPREFETCH()
3043 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension())) in LowerPREFETCH()
3047 unsigned isData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); in LowerPREFETCH()
3048 if (Subtarget->isThumb()) { in LowerPREFETCH()
3067 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); in LowerVASTART()
3068 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); in LowerVASTART()
3082 if (AFI->isThumb1OnlyFunction()) in GetF64FormalArgument()
3094 int FI = MFI->CreateFixedObject(4, NextVA.getLocMemOffset(), true); in GetF64FormalArgument()
3106 if (!Subtarget->isLittle()) in GetF64FormalArgument()
3111 // The remaining GPRs hold either the beginning of variable-argument
3124 // Currently, two use-cases possible: in StoreByValRegs()
3125 // Case #1. Non-var-args function, and we meet first byval parameter. in StoreByValRegs()
3130 // "store-reg" instructions. in StoreByValRegs()
3131 // Case #2. Var-args function, that doesn't contain byval parameters. in StoreByValRegs()
3148 ArgOffset = -4 * (ARM::R4 - RBegin); in StoreByValRegs()
3151 int FrameIndex = MFI->CreateFixedObject(ArgSize, ArgOffset, false); in StoreByValRegs()
3156 AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass; in StoreByValRegs()
3190 AFI->setVarArgsFrameIndex(FrameIndex); in VarArgStyleRegisters()
3212 Function::const_arg_iterator CurOrigArg = MF.getFunction()->arg_begin(); in LowerFormalArguments()
3218 AFI->setArgRegsSaveSize(0); in LowerFormalArguments()
3245 int lastInsIndex = -1; in LowerFormalArguments()
3246 if (isVarArg && MFI->hasVAStart()) { in LowerFormalArguments()
3252 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin); in LowerFormalArguments()
3253 AFI->setArgRegsSaveSize(TotalArgRegsSaveSize); in LowerFormalArguments()
3260 Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx); in LowerFormalArguments()
3276 int FI = MFI->CreateFixedObject(8, VA.getLocMemOffset(), true); in LowerFormalArguments()
3306 RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass in LowerFormalArguments()
3316 // If this is an 8 or 16-bit value, it is really passed promoted in LowerFormalArguments()
3369 int FI = MFI->CreateFixedObject(VA.getLocVT().getSizeInBits()/8, in LowerFormalArguments()
3385 if (isVarArg && MFI->hasVAStart()) in LowerFormalArguments()
3390 AFI->setArgumentStackSize(CCInfo.getNextStackOffset()); in LowerFormalArguments()
3395 /// isFloatingPointZero - Return true if this is +0.0.
3398 return CFP->getValueAPF().isPosZero(); in isFloatingPointZero()
3404 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal())) in isFloatingPointZero()
3405 return CFP->getValueAPF().isPosZero(); in isFloatingPointZero()
3407 } else if (Op->getOpcode() == ISD::BITCAST && in isFloatingPointZero()
3408 Op->getValueType(0) == MVT::f64) { in isFloatingPointZero()
3411 SDValue BitcastOp = Op->getOperand(0); in isFloatingPointZero()
3412 if (BitcastOp->getOpcode() == ARMISD::VMOVIMM && in isFloatingPointZero()
3413 isNullConstant(BitcastOp->getOperand(0))) in isFloatingPointZero()
3425 unsigned C = RHSC->getZExtValue(); in getARMCmp()
3432 if (C != 0x80000000 && isLegalICmpImmediate(C-1)) { in getARMCmp()
3434 RHS = DAG.getConstant(C - 1, dl, MVT::i32); in getARMCmp()
3439 if (C != 0 && isLegalICmpImmediate(C-1)) { in getARMCmp()
3441 RHS = DAG.getConstant(C - 1, dl, MVT::i32); in getARMCmp()
3481 assert(!Subtarget->isFPOnlySP() || RHS.getValueType() != MVT::f64); in getVFPCmp()
3490 /// duplicateCmp - Glue values can have only one use, so this function
3589 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0))) in LowerSELECT()
3604 // (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond) in LowerSELECT()
3605 // (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond) in LowerSELECT()
3614 unsigned CMOVTrueVal = CMOVTrue->getZExtValue(); in LowerSELECT()
3615 unsigned CMOVFalseVal = CMOVFalse->getZExtValue(); in LowerSELECT()
3639 // undefined bits before doing a full-word comparison with zero. in LowerSELECT()
3700 if (Subtarget->isFPOnlySP() && VT == MVT::f64) { in getCMOV()
3731 // See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating.
3746 // Similar to isLowerSaturate(), but checks for upper-saturating conditions.
3761 // x < -k ? -k : (x > k ? k : x)
3762 // x < -k ? -k : (x < k ? x : k)
3763 // x > -k ? (x > k ? k : x) : -k
3764 // x < k ? (x < -k ? -k : x) : k
3776 ISD::CondCode CC1 = cast<CondCodeSDNode>(Op.getOperand(4))->get(); in isSaturatingConditional()
3786 ISD::CondCode CC2 = cast<CondCodeSDNode>(Op2.getOperand(4))->get(); in isSaturatingConditional()
3801 // We must detect cases where the original operations worked with 16- or in isSaturatingConditional()
3802 // 8-bit values. In such case, V2Tmp != V2 because the comparison operations in isSaturatingConditional()
3803 // must work with sign-extended values but the select operations return in isSaturatingConditional()
3804 // the original non-extended value. in isSaturatingConditional()
3806 if (V2Tmp->getOpcode() == ISD::SIGN_EXTEND_INREG) in isSaturatingConditional()
3807 V2TmpReg = V2Tmp->getOperand(0); in isSaturatingConditional()
3830 // Check that the constant in the lower-bound check is in isSaturatingConditional()
3831 // the opposite of the constant in the upper-bound check in isSaturatingConditional()
3833 int64_t Val1 = cast<ConstantSDNode>(*K1)->getSExtValue(); in isSaturatingConditional()
3834 int64_t Val2 = cast<ConstantSDNode>(*K2)->getSExtValue(); in isSaturatingConditional()
3863 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); in LowerSELECT_CC()
3867 if (Subtarget->isFPOnlySP() && LHS.getValueType() == MVT::f64) { in LowerSELECT_CC()
3890 if (Subtarget->hasFPARMv8() && (TrueVal.getValueType() == MVT::f32 || in LowerSELECT_CC()
3910 if (Subtarget->hasFPARMv8() && (TrueVal.getValueType() == MVT::f32 || in LowerSELECT_CC()
3938 /// canChangeToInt - Given the fp compare operand, return true if it is suitable
3943 if (!N->hasOneUse()) in canChangeToInt()
3946 if (!N->getNumValues()) in canChangeToInt()
3949 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow()) in canChangeToInt()
3951 // vmrs are very slow, e.g. cortex-a8. in canChangeToInt()
3967 Ld->getChain(), Ld->getBasePtr(), Ld->getPointerInfo(), in bitcastf32Toi32()
3968 Ld->isVolatile(), Ld->isNonTemporal(), in bitcastf32Toi32()
3969 Ld->isInvariant(), Ld->getAlignment()); in bitcastf32Toi32()
3985 SDValue Ptr = Ld->getBasePtr(); in expandf64Toi32()
3987 Ld->getChain(), Ptr, in expandf64Toi32()
3988 Ld->getPointerInfo(), in expandf64Toi32()
3989 Ld->isVolatile(), Ld->isNonTemporal(), in expandf64Toi32()
3990 Ld->isInvariant(), Ld->getAlignment()); in expandf64Toi32()
3993 unsigned NewAlign = MinAlign(Ld->getAlignment(), 4); in expandf64Toi32()
3997 Ld->getChain(), NewPtr, in expandf64Toi32()
3998 Ld->getPointerInfo().getWithOffset(4), in expandf64Toi32()
3999 Ld->isVolatile(), Ld->isNonTemporal(), in expandf64Toi32()
4000 Ld->isInvariant(), NewAlign); in expandf64Toi32()
4007 /// OptimizeVFPBrcond - With -enable-unsafe-fp-math, it's legal to optimize some
4012 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); in OptimizeVFPBrcond()
4062 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); in LowerBR_CC()
4068 if (Subtarget->isFPOnlySP() && LHS.getValueType() == MVT::f64) { in LowerBR_CC()
4122 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy); in LowerBR_JT()
4126 if (Subtarget->isThumb2()) { in LowerBR_JT()
4127 // Thumb2 uses a two-level jump. That is, it jumps into the jump table in LowerBR_JT()
4175 if (Subtarget->isFPOnlySP() && Op.getOperand(0).getValueType() == MVT::f64) { in LowerFP_TO_INT()
4227 if (Subtarget->isFPOnlySP() && Op.getValueType() == MVT::f64) { in LowerINT_TO_FP()
4251 bool UseNEON = !InGPR && Subtarget->hasNEON(); in LowerFCOPYSIGN()
4327 MFI->setReturnAddressIsTaken(true); in LowerRETURNADDR()
4334 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); in LowerRETURNADDR()
4343 // Return LR, which contains the return address. Mark it an implicit live-in. in LowerRETURNADDR()
4353 MFI->setFrameAddressIsTaken(true); in LowerFRAMEADDR()
4357 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); in LowerFRAMEADDR()
4360 while (Depth--) in LowerFRAMEADDR()
4387 assert(N->getValueType(0) == MVT::i64 in ExpandREAD_REGISTER()
4388 && "ExpandREAD_REGISTER called for non-i64 type result."); in ExpandREAD_REGISTER()
4392 N->getOperand(0), in ExpandREAD_REGISTER()
4393 N->getOperand(1)); in ExpandREAD_REGISTER()
4409 SDValue Op = BC->getOperand(0); in CombineVMOVDRRCandidateWithVecOp()
4410 EVT DstVT = BC->getValueType(0); in CombineVMOVDRRCandidateWithVecOp()
4431 const APInt &APIntIndex = Index->getAPIntValue(); in CombineVMOVDRRCandidateWithVecOp()
4438 // vMTy bitcast(i64 extractelt vNi64 src, i32 index) -> in CombineVMOVDRRCandidateWithVecOp()
4450 /// ExpandBITCAST - If the target supports VFP, this function is called to
4452 /// use a VMOVDRR or VMOVRRD node. This should not be done when the non-i64
4458 SDValue Op = N->getOperand(0); in ExpandBITCAST()
4463 EVT DstVT = N->getValueType(0); in ExpandBITCAST()
4465 "ExpandBITCAST called for non-i64 type"); in ExpandBITCAST()
4467 // Turn i64->f64 into VMOVDRR. in ExpandBITCAST()
4482 // Turn f64->i64 into VMOVRRD. in ExpandBITCAST()
4500 /// getZeroVector - Returns a vector of specified type with all zero elements.
4515 /// LowerShiftRightParts - Lower SRA_PARTS, which returns two
4519 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); in LowerShiftRightParts()
4551 /// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
4555 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); in LowerShiftLeftParts()
4588 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0 in LowerFLT_ROUNDS_()
4606 EVT VT = N->getValueType(0); in LowerCTTZ()
4608 assert(ST->hasNEON()); in LowerCTTZ()
4610 // Compute the least significant set bit: LSB = X & -X in LowerCTTZ()
4611 SDValue X = N->getOperand(0); in LowerCTTZ()
4618 // Compute with: cttz(x) = ctpop(lsb - 1) in LowerCTTZ()
4626 (N->getOpcode() == ISD::CTTZ_ZERO_UNDEF)) { in LowerCTTZ()
4627 // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0 in LowerCTTZ()
4631 DAG.getTargetConstant(NumBits - 1, dl, ElemTy)); in LowerCTTZ()
4636 // Compute with: cttz(x) = ctpop(lsb - 1) in LowerCTTZ()
4642 // Compute LSB - 1. in LowerCTTZ()
4682 if (!ST->hasV6T2Ops()) in LowerCTTZ()
4685 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0)); in LowerCTTZ()
4689 /// getCTPOP16BitCounts - Returns a v8i8/v16i8 vector containing the bit-count
4690 /// for each 16-bit element from operand, repeated. The basic idea is to
4691 /// leverage vcnt to get the 8-bit counts, gather and add the results.
4694 /// input = [v0 v1 v2 v3 ] (vi 16-bit element)
4695 /// cast: N0 = [w0 w1 w2 w3 w4 w5 w6 w7] (v0 = [w0 w1], wi 8-bit element)
4696 /// vcnt: N1 = [b0 b1 b2 b3 b4 b5 b6 b7] (bi = bit-count of 8-bit element wi)
4700 /// N3=N1+N2 = [k0 k0 k1 k1 k2 k2 k3 k3] (k0 = b0+b1 = bit-count of 16-bit v0,
4701 /// vuzp: = [k0 k1 k2 k3 k0 k1 k2 k3] each ki is 8-bits)
4703 EVT VT = N->getValueType(0); in getCTPOP16BitCounts()
4707 SDValue N0 = DAG.getNode(ISD::BITCAST, DL, VT8Bit, N->getOperand(0)); in getCTPOP16BitCounts()
4714 /// lowerCTPOP16BitElements - Returns a v4i16/v8i16 vector containing the
4715 /// bit-count for each 16-bit element from the operand. We need slightly
4717 /// 64/128-bit registers.
4720 /// input = [v0 v1 v2 v3 ] (vi 16-bit element)
4721 /// v8i8: BitCounts = [k0 k1 k2 k3 k0 k1 k2 k3 ] (ki is the bit-count of vi)
4725 EVT VT = N->getValueType(0); in lowerCTPOP16BitElements()
4740 /// lowerCTPOP32BitElements - Returns a v2i32/v4i32 vector containing the
4741 /// bit-count for each 32-bit element from the operand. The idea here is
4742 /// to split the vector into 16-bit elements, leverage the 16-bit count
4746 /// input = [v0 v1 ] (vi: 32-bit elements)
4747 /// Bitcast = [w0 w1 w2 w3 ] (wi: 16-bit elements, v0 = [w0 w1])
4748 /// Counts16 = [k0 k1 k2 k3 ] (ki: 16-bit elements, bit-count of wi)
4760 EVT VT = N->getValueType(0); in lowerCTPOP32BitElements()
4765 SDValue Bitcast = DAG.getNode(ISD::BITCAST, DL, VT16Bit, N->getOperand(0)); in lowerCTPOP32BitElements()
4784 EVT VT = N->getValueType(0); in LowerCTPOP()
4786 assert(ST->hasNEON() && "Custom ctpop lowering requires NEON."); in LowerCTPOP()
4799 EVT VT = N->getValueType(0); in LowerShift()
4806 assert(ST->hasNEON() && "unexpected vector shift"); in LowerShift()
4809 if (N->getOpcode() == ISD::SHL) in LowerShift()
4813 N->getOperand(0), N->getOperand(1)); in LowerShift()
4815 assert((N->getOpcode() == ISD::SRA || in LowerShift()
4816 N->getOpcode() == ISD::SRL) && "unexpected vector shift opcode"); in LowerShift()
4821 EVT ShiftVT = N->getOperand(1).getValueType(); in LowerShift()
4824 N->getOperand(1)); in LowerShift()
4825 Intrinsic::ID vshiftInt = (N->getOpcode() == ISD::SRA ? in LowerShift()
4830 N->getOperand(0), NegatedCount); in LowerShift()
4835 EVT VT = N->getValueType(0); in Expand64BitShift()
4842 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) && in Expand64BitShift()
4846 if (!isOneConstant(N->getOperand(1))) in Expand64BitShift()
4850 if (ST->isThumb1Only()) return SDValue(); in Expand64BitShift()
4852 // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr. in Expand64BitShift()
4853 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0), in Expand64BitShift()
4855 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0), in Expand64BitShift()
4860 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::SRL_FLAG:ARMISD::SRA_FLAG; in Expand64BitShift()
4881 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); in LowerVSETCC()
4885 // 64-bit comparisons are not legal. We've marked SETCC as non-Custom, in LowerVSETCC()
4886 // but it's possible that our operands are 64-bit but our result is 32-bit. in LowerVSETCC()
4970 // comparison to a specialized compare-against-zero form. in LowerVSETCC()
5026 IntCCToARMCC(cast<CondCodeSDNode>(Cond)->get()), DL, MVT::i32); in LowerSETCCE()
5034 /// isNEONModifiedImm - Check if the specified splat value corresponds to a
5045 // immediate instructions others than VMOV do not support the 8-bit encoding in isNEONModifiedImm()
5047 // 32-bit version. in isNEONModifiedImm()
5055 // Any 1-byte value is OK. Op=0, Cmode=1110. in isNEONModifiedImm()
5063 // NEON's 16-bit VMOV supports splat values where only one byte is nonzero. in isNEONModifiedImm()
5080 // NEON's 32-bit VMOV supports splat values where: in isNEONModifiedImm()
5129 // Note: there are a few 32-bit splat values (specifically: 00ffff00, in isNEONModifiedImm()
5132 // and fall through here to test for a valid 64-bit splat. But, then the in isNEONModifiedImm()
5139 // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff. in isNEONModifiedImm()
5175 if (!ST->hasVFP3()) in LowerConstantFP()
5182 // an SP-only FPU in LowerConstantFP()
5183 if (IsDouble && Subtarget->isFPOnlySP()) in LowerConstantFP()
5187 const APFloat &FPVal = CFP->getValueAPF(); in LowerConstantFP()
5190 if (ImmVal != -1) { in LowerConstantFP()
5191 if (IsDouble || !ST->useNEONForSinglePrecisionFP()) { in LowerConstantFP()
5209 if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP())) in LowerConstantFP()
5320 Imm -= NumElts; in isVEXTMask()
5325 /// isVREVMask - Check if a vector shuffle corresponds to a VREV
5347 if ((unsigned) M[i] != (i - i%BlockElts) + (BlockElts - 1 - i%BlockElts)) in isVREVMask()
5394 // element is undefined, e.g. [-1, 4, 2, 6] will be rejected, because only in isVTRNMask()
5414 /// isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of
5472 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. in isVUZPMask()
5479 /// isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of
5508 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. in isVUZP_v_undef_Mask()
5546 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. in isVZIPMask()
5553 /// isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of
5579 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32. in isVZIP_v_undef_Mask()
5586 /// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
5617 // Look for <15, ..., 3, -1, 1, 0>. in isReverseMask()
5619 if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i)) in isReverseMask()
5633 Val = cast<ConstantSDNode>(N)->getZExtValue(); in IsSingleInstrConstant()
5635 if (ST->isThumb1Only()) { in IsSingleInstrConstant()
5639 if (ARM_AM::getSOImmVal(Val) != -1 || ARM_AM::getSOImmVal(~Val) != -1) in IsSingleInstrConstant()
5656 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) { in LowerBUILD_VECTOR()
5683 if (ImmVal != -1) { in LowerBUILD_VECTOR()
5729 Value = ValueCounts.begin()->first; in LowerBUILD_VECTOR()
5741 // Use VDUP for non-constant splats. For f32 constant splats, reduce to in LowerBUILD_VECTOR()
5751 // constant-index forms. in LowerBUILD_VECTOR()
5753 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in LowerBUILD_VECTOR()
5754 (constIndex = dyn_cast<ConstantSDNode>(Value->getOperand(1)))) { in LowerBUILD_VECTOR()
5759 if (VT != Value->getOperand(0).getValueType()) { in LowerBUILD_VECTOR()
5760 unsigned index = constIndex->getAPIntValue().getLimitedValue() % in LowerBUILD_VECTOR()
5768 Value->getOperand(0), Value->getOperand(1)); in LowerBUILD_VECTOR()
5818 // Vectors with 32- or 64-bit elements can be built by directly assigning in LowerBUILD_VECTOR()
5822 // Do the expansion with floating-point types, since that is what the VFP in LowerBUILD_VECTOR()
5835 // worse. For a vector with one or two non-undef values, that's in LowerBUILD_VECTOR()
5908 unsigned EltNo = cast<ConstantSDNode>(V.getOperand(1))->getZExtValue(); in ReconstructShuffle()
5909 Source->MinElt = std::min(Source->MinElt, EltNo); in ReconstructShuffle()
5910 Source->MaxElt = std::max(Source->MaxElt, EltNo); in ReconstructShuffle()
5960 if (Src.MaxElt - Src.MinElt >= NumSrcElts) { in ReconstructShuffle()
5970 Src.WindowBase = -NumSrcElts; in ReconstructShuffle()
5988 Src.WindowBase = -Src.MinElt; in ReconstructShuffle()
6012 SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1); in ReconstructShuffle()
6020 int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue(); in ReconstructShuffle()
6034 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase; in ReconstructShuffle()
6035 ExtractBase += NumElts * (Src - Sources.begin()); in ReconstructShuffle()
6057 /// isShuffleMaskLegal - Targets can use this to indicate that they only
6099 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
6105 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1); in GeneratePerfectShuffle()
6106 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1); in GeneratePerfectShuffle()
6144 // vrev <4 x i16> -> VREV32 in GeneratePerfectShuffle()
6147 // vrev <4 x i8> -> VREV16 in GeneratePerfectShuffle()
6155 OpLHS, DAG.getConstant(OpNum-OP_VDUP0, dl, MVT::i32)); in GeneratePerfectShuffle()
6161 DAG.getConstant(OpNum - OP_VEXT1 + 1, dl, MVT::i32)); in GeneratePerfectShuffle()
6165 OpLHS, OpRHS).getValue(OpNum-OP_VUZPL); in GeneratePerfectShuffle()
6169 OpLHS, OpRHS).getValue(OpNum-OP_VZIPL); in GeneratePerfectShuffle()
6173 OpLHS, OpRHS).getValue(OpNum-OP_VTRNL); in GeneratePerfectShuffle()
6190 if (V2.getNode()->isUndef()) in LowerVECTOR_SHUFFLEv8i8()
6222 // Convert shuffles that are directly supported on NEON to target-specific in LowerVECTOR_SHUFFLE()
6226 // FIXME: floating-point vectors should be canonicalized to integer vectors in LowerVECTOR_SHUFFLE()
6228 ArrayRef<int> ShuffleMask = SVN->getMask(); in LowerVECTOR_SHUFFLE()
6232 if (SVN->isSplat()) { in LowerVECTOR_SHUFFLE()
6233 int Lane = SVN->getSplatIndex(); in LowerVECTOR_SHUFFLE()
6235 if (Lane == -1) Lane = 0; in LowerVECTOR_SHUFFLE()
6275 if (V2->isUndef() && isSingletonVEXTMask(ShuffleMask, VT, Imm)) { in LowerVECTOR_SHUFFLE()
6298 // -> in LowerVECTOR_SHUFFLE()
6303 // native shuffles produce larger results: the two-result ops. in LowerVECTOR_SHUFFLE()
6307 // -> in LowerVECTOR_SHUFFLE()
6310 if (V1->getOpcode() == ISD::CONCAT_VECTORS && V2->isUndef()) { in LowerVECTOR_SHUFFLE()
6311 SDValue SubV1 = V1->getOperand(0); in LowerVECTOR_SHUFFLE()
6312 SDValue SubV2 = V1->getOperand(1); in LowerVECTOR_SHUFFLE()
6315 // We expect these to have been canonicalized to -1. in LowerVECTOR_SHUFFLE()
6325 "In-place shuffle of concat can only have one result!"); in LowerVECTOR_SHUFFLE()
6335 // the PerfectShuffle-generated table to synthesize it from other shuffles. in LowerVECTOR_SHUFFLE()
6356 // Implement shuffles with 32- or 64-bit elements as ARMISD::BUILD_VECTORs. in LowerVECTOR_SHUFFLE()
6358 // Do the expansion with floating-point types, since that is what the VFP in LowerVECTOR_SHUFFLE()
6371 DAG.getConstant(ShuffleMask[i] & (NumElts-1), in LowerVECTOR_SHUFFLE()
6415 // two 64-bit vectors are concatenated to a 128-bit vector. in LowerCONCAT_VECTORS()
6433 /// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each
6434 /// element has been zero/sign-extended, depending on the isSigned parameter,
6439 EVT VT = N->getValueType(0); in isExtendedBUILD_VECTOR()
6440 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) { in isExtendedBUILD_VECTOR()
6441 SDNode *BVN = N->getOperand(0).getNode(); in isExtendedBUILD_VECTOR()
6442 if (BVN->getValueType(0) != MVT::v4i32 || in isExtendedBUILD_VECTOR()
6443 BVN->getOpcode() != ISD::BUILD_VECTOR) in isExtendedBUILD_VECTOR()
6446 unsigned HiElt = 1 - LoElt; in isExtendedBUILD_VECTOR()
6447 ConstantSDNode *Lo0 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt)); in isExtendedBUILD_VECTOR()
6448 ConstantSDNode *Hi0 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt)); in isExtendedBUILD_VECTOR()
6449 ConstantSDNode *Lo1 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt+2)); in isExtendedBUILD_VECTOR()
6450 ConstantSDNode *Hi1 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt+2)); in isExtendedBUILD_VECTOR()
6454 if (Hi0->getSExtValue() == Lo0->getSExtValue() >> 32 && in isExtendedBUILD_VECTOR()
6455 Hi1->getSExtValue() == Lo1->getSExtValue() >> 32) in isExtendedBUILD_VECTOR()
6458 if (Hi0->isNullValue() && Hi1->isNullValue()) in isExtendedBUILD_VECTOR()
6464 if (N->getOpcode() != ISD::BUILD_VECTOR) in isExtendedBUILD_VECTOR()
6467 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { in isExtendedBUILD_VECTOR()
6468 SDNode *Elt = N->getOperand(i).getNode(); in isExtendedBUILD_VECTOR()
6473 if (!isIntN(HalfSize, C->getSExtValue())) in isExtendedBUILD_VECTOR()
6476 if (!isUIntN(HalfSize, C->getZExtValue())) in isExtendedBUILD_VECTOR()
6487 /// isSignExtended - Check if a node is a vector value that is sign-extended
6488 /// or a constant BUILD_VECTOR with sign-extended elements.
6490 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N)) in isSignExtended()
6497 /// isZeroExtended - Check if a node is a vector value that is zero-extended
6498 /// or a constant BUILD_VECTOR with zero-extended elements.
6500 if (N->getOpcode() == ISD::ZERO_EXTEND || ISD::isZEXTLoad(N)) in isZeroExtended()
6524 /// AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total
6525 /// value size to 64 bits. We need a 64-bit D register as an operand to VMULL.
6532 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than in AddRequiredExtensionForVMULL()
6533 // 64-bits we need to insert a new extension so that it will be 64-bits. in AddRequiredExtensionForVMULL()
6544 /// SkipLoadExtensionForVMULL - return a load of the original vector size that
6550 EVT ExtendedTy = getExtensionTo64Bits(LD->getMemoryVT()); in SkipLoadExtensionForVMULL()
6553 if (ExtendedTy == LD->getMemoryVT()) in SkipLoadExtensionForVMULL()
6554 return DAG.getLoad(LD->getMemoryVT(), SDLoc(LD), LD->getChain(), in SkipLoadExtensionForVMULL()
6555 LD->getBasePtr(), LD->getPointerInfo(), LD->isVolatile(), in SkipLoadExtensionForVMULL()
6556 LD->isNonTemporal(), LD->isInvariant(), in SkipLoadExtensionForVMULL()
6557 LD->getAlignment()); in SkipLoadExtensionForVMULL()
6562 return DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), ExtendedTy, in SkipLoadExtensionForVMULL()
6563 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(), in SkipLoadExtensionForVMULL()
6564 LD->getMemoryVT(), LD->isVolatile(), LD->isInvariant(), in SkipLoadExtensionForVMULL()
6565 LD->isNonTemporal(), LD->getAlignment()); in SkipLoadExtensionForVMULL()
6568 /// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND,
6575 if (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND) in SkipExtensionForVMULL()
6576 return AddRequiredExtensionForVMULL(N->getOperand(0), DAG, in SkipExtensionForVMULL()
6577 N->getOperand(0)->getValueType(0), in SkipExtensionForVMULL()
6578 N->getValueType(0), in SkipExtensionForVMULL()
6579 N->getOpcode()); in SkipExtensionForVMULL()
6586 if (N->getOpcode() == ISD::BITCAST) { in SkipExtensionForVMULL()
6587 SDNode *BVN = N->getOperand(0).getNode(); in SkipExtensionForVMULL()
6588 assert(BVN->getOpcode() == ISD::BUILD_VECTOR && in SkipExtensionForVMULL()
6589 BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR"); in SkipExtensionForVMULL()
6593 {BVN->getOperand(LowElt), BVN->getOperand(LowElt + 2)}); in SkipExtensionForVMULL()
6596 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR"); in SkipExtensionForVMULL()
6597 EVT VT = N->getValueType(0); in SkipExtensionForVMULL()
6604 ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i)); in SkipExtensionForVMULL()
6605 const APInt &CInt = C->getAPIntValue(); in SkipExtensionForVMULL()
6614 unsigned Opcode = N->getOpcode(); in isAddSubSExt()
6616 SDNode *N0 = N->getOperand(0).getNode(); in isAddSubSExt()
6617 SDNode *N1 = N->getOperand(1).getNode(); in isAddSubSExt()
6618 return N0->hasOneUse() && N1->hasOneUse() && in isAddSubSExt()
6625 unsigned Opcode = N->getOpcode(); in isAddSubZExt()
6627 SDNode *N0 = N->getOperand(0).getNode(); in isAddSubZExt()
6628 SDNode *N1 = N->getOperand(1).getNode(); in isAddSubZExt()
6629 return N0->hasOneUse() && N1->hasOneUse() && in isAddSubZExt()
6636 // Multiplications are only custom-lowered for 128-bit vectors so that in LowerMUL()
6640 "unexpected type for custom-lowering ISD::MUL"); in LowerMUL()
6693 // isel lowering to take advantage of no-stall back to back vmul + vmla. in LowerMUL()
6700 SDValue N00 = SkipExtensionForVMULL(N0->getOperand(0).getNode(), DAG); in LowerMUL()
6701 SDValue N01 = SkipExtensionForVMULL(N0->getOperand(1).getNode(), DAG); in LowerMUL()
6703 return DAG.getNode(N0->getOpcode(), DL, VT, in LowerMUL()
6712 // TODO: Should this propagate fast-math-flags? in LowerSDIV_v4i8()
6743 // TODO: Should this propagate fast-math-flags? in LowerSDIV_v4i16()
6783 "unexpected type for custom-lowering ISD::SDIV"); in LowerSDIV()
6816 // TODO: Should this propagate fast-math-flags? in LowerUDIV()
6819 "unexpected type for custom-lowering ISD::UDIV"); in LowerUDIV()
6892 EVT VT = Op.getNode()->getValueType(0); in LowerADDC_ADDE_SUBC_SUBE()
6913 assert(Subtarget->isTargetDarwin()); in LowerFSINCOS()
6931 bool ShouldUseSRet = Subtarget->isAPCS_ABI(); in LowerFSINCOS()
6937 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); in LowerFSINCOS()
6942 Entry.Ty = RetTy->getPointerTo(); in LowerFSINCOS()
7068 if (isStrongerThanMonotonic(cast<AtomicSDNode>(Op)->getOrdering())) in LowerAtomicLoadStore()
7082 // Under Power Management extensions, the cycle-count is: in ReplaceREADCYCLECOUNTER()
7084 SDValue Ops[] = { N->getOperand(0), // Chain in ReplaceREADCYCLECOUNTER()
7118 assert(N->getValueType(0) == MVT::i64 && in ReplaceCMP_SWAP_64Results()
7120 SDValue Ops[] = {N->getOperand(1), in ReplaceCMP_SWAP_64Results()
7121 createGPRPairNode(DAG, N->getOperand(2)), in ReplaceCMP_SWAP_64Results()
7122 createGPRPairNode(DAG, N->getOperand(3)), in ReplaceCMP_SWAP_64Results()
7123 N->getOperand(0)}; in ReplaceCMP_SWAP_64Results()
7130 MemOp[0] = cast<MemSDNode>(N)->getMemOperand(); in ReplaceCMP_SWAP_64Results()
7131 cast<MachineSDNode>(CmpSwap)->setMemRefs(MemOp, MemOp + 1); in ReplaceCMP_SWAP_64Results()
7147 switch (Subtarget->getTargetTriple().getObjectFormat()) { in LowerOperation()
7199 if (Subtarget->isTargetWindows()) in LowerOperation()
7203 if (Subtarget->isTargetWindows()) in LowerOperation()
7221 if (Subtarget->getTargetTriple().isWindowsItaniumEnvironment()) in LowerOperation()
7230 /// ReplaceNodeResults - Replace the results of node with an illegal result
7236 switch (N->getOpcode()) { in ReplaceNodeResults()
7265 assert(Subtarget->isTargetWindows() && "can only expand DIV on Windows"); in ReplaceNodeResults()
7266 return ExpandDIV_Windows(SDValue(N, 0), DAG, N->getOpcode() == ISD::SDIV, in ReplaceNodeResults()
7276 //===----------------------------------------------------------------------===//
7278 //===----------------------------------------------------------------------===//
7280 /// SetupEntryBlockForSjLj - Insert code into the entry block that creates and
7286 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in SetupEntryBlockForSjLj()
7288 MachineFunction *MF = MBB->getParent(); in SetupEntryBlockForSjLj()
7289 MachineRegisterInfo *MRI = &MF->getRegInfo(); in SetupEntryBlockForSjLj()
7290 MachineConstantPool *MCP = MF->getConstantPool(); in SetupEntryBlockForSjLj()
7291 ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>(); in SetupEntryBlockForSjLj()
7292 const Function *F = MF->getFunction(); in SetupEntryBlockForSjLj()
7294 bool isThumb = Subtarget->isThumb(); in SetupEntryBlockForSjLj()
7295 bool isThumb2 = Subtarget->isThumb2(); in SetupEntryBlockForSjLj()
7297 unsigned PCLabelId = AFI->createPICLabelUId(); in SetupEntryBlockForSjLj()
7300 ARMConstantPoolMBB::Create(F->getContext(), DispatchBB, PCLabelId, PCAdj); in SetupEntryBlockForSjLj()
7301 unsigned CPI = MCP->getConstantPoolIndex(CPV, 4); in SetupEntryBlockForSjLj()
7308 MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF), in SetupEntryBlockForSjLj()
7312 MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI), in SetupEntryBlockForSjLj()
7315 // Load the address of the dispatch MBB into the jump buffer. in SetupEntryBlockForSjLj()
7322 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
7323 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::t2LDRpci), NewVReg1) in SetupEntryBlockForSjLj()
7327 unsigned NewVReg2 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
7329 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::t2ORRri), NewVReg2) in SetupEntryBlockForSjLj()
7332 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
7333 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg3) in SetupEntryBlockForSjLj()
7336 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::t2STRi12)) in SetupEntryBlockForSjLj()
7349 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
7350 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tLDRpci), NewVReg1) in SetupEntryBlockForSjLj()
7353 unsigned NewVReg2 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
7354 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg2) in SetupEntryBlockForSjLj()
7358 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
7359 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tMOVi8), NewVReg3) in SetupEntryBlockForSjLj()
7362 unsigned NewVReg4 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
7363 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tORR), NewVReg4) in SetupEntryBlockForSjLj()
7367 unsigned NewVReg5 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
7368 BuildMI(*MBB, MI, dl, TII->get(ARM::tADDframe), NewVReg5) in SetupEntryBlockForSjLj()
7371 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tSTRi)) in SetupEntryBlockForSjLj()
7381 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
7382 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::LDRi12), NewVReg1) in SetupEntryBlockForSjLj()
7386 unsigned NewVReg2 = MRI->createVirtualRegister(TRC); in SetupEntryBlockForSjLj()
7387 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::PICADD), NewVReg2) in SetupEntryBlockForSjLj()
7390 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::STRi12)) in SetupEntryBlockForSjLj()
7400 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in EmitSjLjDispatchBlock()
7402 MachineFunction *MF = MBB->getParent(); in EmitSjLjDispatchBlock()
7403 MachineRegisterInfo *MRI = &MF->getRegInfo(); in EmitSjLjDispatchBlock()
7404 MachineFrameInfo *MFI = MF->getFrameInfo(); in EmitSjLjDispatchBlock()
7405 int FI = MFI->getFunctionContextIndex(); in EmitSjLjDispatchBlock()
7407 const TargetRegisterClass *TRC = Subtarget->isThumb() ? &ARM::tGPRRegClass in EmitSjLjDispatchBlock()
7414 MachineModuleInfo &MMI = MF->getMMI(); in EmitSjLjDispatchBlock()
7415 for (MachineFunction::iterator BB = MF->begin(), E = MF->end(); BB != E; in EmitSjLjDispatchBlock()
7417 if (!BB->isEHPad()) continue; in EmitSjLjDispatchBlock()
7422 II = BB->begin(), IE = BB->end(); II != IE; ++II) { in EmitSjLjDispatchBlock()
7423 if (!II->isEHLabel()) continue; in EmitSjLjDispatchBlock()
7425 MCSymbol *Sym = II->getOperand(0).getMCSymbol(); in EmitSjLjDispatchBlock()
7448 InvokeBBs.insert((*II)->pred_begin(), (*II)->pred_end()); in EmitSjLjDispatchBlock()
7453 "No landing pad destinations for the dispatch jump table!"); in EmitSjLjDispatchBlock()
7457 MF->getOrCreateJumpTableInfo(MachineJumpTableInfo::EK_Inline); in EmitSjLjDispatchBlock()
7458 unsigned MJTI = JTI->createJumpTableIndex(LPadList); in EmitSjLjDispatchBlock()
7460 // Create the MBBs for the dispatch code. in EmitSjLjDispatchBlock()
7462 // Shove the dispatch's address into the return slot in the function context. in EmitSjLjDispatchBlock()
7463 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock(); in EmitSjLjDispatchBlock()
7464 DispatchBB->setIsEHPad(); in EmitSjLjDispatchBlock()
7466 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock(); in EmitSjLjDispatchBlock()
7468 if (Subtarget->isThumb()) in EmitSjLjDispatchBlock()
7471 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP; in EmitSjLjDispatchBlock()
7473 BuildMI(TrapBB, dl, TII->get(trap_opcode)); in EmitSjLjDispatchBlock()
7474 DispatchBB->addSuccessor(TrapBB); in EmitSjLjDispatchBlock()
7476 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock(); in EmitSjLjDispatchBlock()
7477 DispatchBB->addSuccessor(DispContBB); in EmitSjLjDispatchBlock()
7480 MF->insert(MF->end(), DispatchBB); in EmitSjLjDispatchBlock()
7481 MF->insert(MF->end(), DispContBB); in EmitSjLjDispatchBlock()
7482 MF->insert(MF->end(), TrapBB); in EmitSjLjDispatchBlock()
7488 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand( in EmitSjLjDispatchBlock()
7493 MIB = BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup)); in EmitSjLjDispatchBlock()
7496 const ARMBaseRegisterInfo &RI = AII->getRegisterInfo(); in EmitSjLjDispatchBlock()
7504 if (Subtarget->isThumb2()) { in EmitSjLjDispatchBlock()
7505 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7506 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2LDRi12), NewVReg1) in EmitSjLjDispatchBlock()
7512 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPri)) in EmitSjLjDispatchBlock()
7516 unsigned VReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7517 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVi16), VReg1) in EmitSjLjDispatchBlock()
7522 VReg2 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7523 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVTi16), VReg2) in EmitSjLjDispatchBlock()
7528 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPrr)) in EmitSjLjDispatchBlock()
7533 BuildMI(DispatchBB, dl, TII->get(ARM::t2Bcc)) in EmitSjLjDispatchBlock()
7538 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7539 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::t2LEApcrelJT),NewVReg3) in EmitSjLjDispatchBlock()
7542 unsigned NewVReg4 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7545 BuildMI(DispContBB, dl, TII->get(ARM::t2ADDrs), NewVReg4) in EmitSjLjDispatchBlock()
7550 BuildMI(DispContBB, dl, TII->get(ARM::t2BR_JT)) in EmitSjLjDispatchBlock()
7554 } else if (Subtarget->isThumb()) { in EmitSjLjDispatchBlock()
7555 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7556 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tLDRspi), NewVReg1) in EmitSjLjDispatchBlock()
7562 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tCMPi8)) in EmitSjLjDispatchBlock()
7566 MachineConstantPool *ConstantPool = MF->getConstantPool(); in EmitSjLjDispatchBlock()
7567 Type *Int32Ty = Type::getInt32Ty(MF->getFunction()->getContext()); in EmitSjLjDispatchBlock()
7571 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty); in EmitSjLjDispatchBlock()
7573 Align = MF->getDataLayout().getTypeAllocSize(C->getType()); in EmitSjLjDispatchBlock()
7574 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align); in EmitSjLjDispatchBlock()
7576 unsigned VReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7577 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tLDRpci)) in EmitSjLjDispatchBlock()
7580 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tCMPr)) in EmitSjLjDispatchBlock()
7585 BuildMI(DispatchBB, dl, TII->get(ARM::tBcc)) in EmitSjLjDispatchBlock()
7590 unsigned NewVReg2 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7591 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLSLri), NewVReg2) in EmitSjLjDispatchBlock()
7596 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7597 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLEApcrelJT), NewVReg3) in EmitSjLjDispatchBlock()
7600 unsigned NewVReg4 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7601 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg4) in EmitSjLjDispatchBlock()
7606 MachineMemOperand *JTMMOLd = MF->getMachineMemOperand( in EmitSjLjDispatchBlock()
7609 unsigned NewVReg5 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7610 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLDRi), NewVReg5) in EmitSjLjDispatchBlock()
7617 NewVReg6 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7618 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg6) in EmitSjLjDispatchBlock()
7624 BuildMI(DispContBB, dl, TII->get(ARM::tBR_JTr)) in EmitSjLjDispatchBlock()
7628 unsigned NewVReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7629 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::LDRi12), NewVReg1) in EmitSjLjDispatchBlock()
7635 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::CMPri)) in EmitSjLjDispatchBlock()
7638 } else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) { in EmitSjLjDispatchBlock()
7639 unsigned VReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7640 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::MOVi16), VReg1) in EmitSjLjDispatchBlock()
7645 VReg2 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7646 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::MOVTi16), VReg2) in EmitSjLjDispatchBlock()
7651 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr)) in EmitSjLjDispatchBlock()
7655 MachineConstantPool *ConstantPool = MF->getConstantPool(); in EmitSjLjDispatchBlock()
7656 Type *Int32Ty = Type::getInt32Ty(MF->getFunction()->getContext()); in EmitSjLjDispatchBlock()
7660 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty); in EmitSjLjDispatchBlock()
7662 Align = MF->getDataLayout().getTypeAllocSize(C->getType()); in EmitSjLjDispatchBlock()
7663 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align); in EmitSjLjDispatchBlock()
7665 unsigned VReg1 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7666 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::LDRcp)) in EmitSjLjDispatchBlock()
7670 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr)) in EmitSjLjDispatchBlock()
7675 BuildMI(DispatchBB, dl, TII->get(ARM::Bcc)) in EmitSjLjDispatchBlock()
7680 unsigned NewVReg3 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7682 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::MOVsi), NewVReg3) in EmitSjLjDispatchBlock()
7685 unsigned NewVReg4 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7686 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::LEApcrelJT), NewVReg4) in EmitSjLjDispatchBlock()
7689 MachineMemOperand *JTMMOLd = MF->getMachineMemOperand( in EmitSjLjDispatchBlock()
7691 unsigned NewVReg5 = MRI->createVirtualRegister(TRC); in EmitSjLjDispatchBlock()
7693 BuildMI(DispContBB, dl, TII->get(ARM::LDRrs), NewVReg5) in EmitSjLjDispatchBlock()
7700 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTadd)) in EmitSjLjDispatchBlock()
7705 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTr)) in EmitSjLjDispatchBlock()
7717 DispContBB->addSuccessor(CurMBB); in EmitSjLjDispatchBlock()
7726 // with the new dispatch block. in EmitSjLjDispatchBlock()
7727 SmallVector<MachineBasicBlock*, 4> Successors(BB->succ_begin(), in EmitSjLjDispatchBlock()
7728 BB->succ_end()); in EmitSjLjDispatchBlock()
7731 if (SMBB->isEHPad()) { in EmitSjLjDispatchBlock()
7732 BB->removeSuccessor(SMBB); in EmitSjLjDispatchBlock()
7737 BB->addSuccessor(DispatchBB, BranchProbability::getZero()); in EmitSjLjDispatchBlock()
7738 BB->normalizeSuccProbs(); in EmitSjLjDispatchBlock()
7740 // Find the invoke call and mark all of the callee-saved registers as in EmitSjLjDispatchBlock()
7745 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) { in EmitSjLjDispatchBlock()
7746 if (!II->isCall()) continue; in EmitSjLjDispatchBlock()
7750 OI = II->operands_begin(), OE = II->operands_end(); in EmitSjLjDispatchBlock()
7752 if (!OI->isReg()) continue; in EmitSjLjDispatchBlock()
7753 DefRegs[OI->getReg()] = true; in EmitSjLjDispatchBlock()
7760 if (Subtarget->isThumb2() && in EmitSjLjDispatchBlock()
7764 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(Reg)) in EmitSjLjDispatchBlock()
7766 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg)) in EmitSjLjDispatchBlock()
7776 // Mark all former landing pads as non-landing pads. The dispatch is the only in EmitSjLjDispatchBlock()
7780 (*I)->setIsEHPad(false); in EmitSjLjDispatchBlock()
7788 for (MachineBasicBlock::succ_iterator I = MBB->succ_begin(), in OtherSucc()
7789 E = MBB->succ_end(); I != E; ++I) in OtherSucc()
7833 /// Emit a post-increment load operation with given size. The instructions
7842 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) in emitPostLd()
7847 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) in emitPostLd()
7850 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut); in emitPostLd()
7855 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) in emitPostLd()
7859 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data) in emitPostLd()
7865 /// Emit a post-increment store operation with given size. The instructions
7874 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut) in emitPostSt()
7878 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(StOpc)).addReg(Data) in emitPostSt()
7881 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut); in emitPostSt()
7886 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut) in emitPostSt()
7889 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut) in emitPostSt()
7899 // We expand it to a loop if size > Subtarget->getMaxInlineSizeThreshold(). in EmitStructByval()
7901 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in EmitStructByval()
7902 const BasicBlock *LLVM_BB = BB->getBasicBlock(); in EmitStructByval()
7903 MachineFunction::iterator It = ++BB->getIterator(); in EmitStructByval()
7911 MachineFunction *MF = BB->getParent(); in EmitStructByval()
7912 MachineRegisterInfo &MRI = MF->getRegInfo(); in EmitStructByval()
7917 bool IsThumb1 = Subtarget->isThumb1Only(); in EmitStructByval()
7918 bool IsThumb2 = Subtarget->isThumb2(); in EmitStructByval()
7926 if (!MF->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat) && in EmitStructByval()
7927 Subtarget->hasNEON()) { in EmitStructByval()
7947 unsigned LoopSize = SizeVal - BytesLeft; in EmitStructByval()
7949 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) { in EmitStructByval()
7988 // movw varEnd, # --> with thumb2 in EmitStructByval()
7990 // ldrcp varEnd, idx --> without thumb2 in EmitStructByval()
7991 // fallthrough --> loopMBB in EmitStructByval()
8000 // fallthrough --> exitMBB in EmitStructByval()
8002 // epilogue to handle left-over bytes in EmitStructByval()
8005 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitStructByval()
8006 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); in EmitStructByval()
8007 MF->insert(It, loopMBB); in EmitStructByval()
8008 MF->insert(It, exitMBB); in EmitStructByval()
8011 exitMBB->splice(exitMBB->begin(), BB, in EmitStructByval()
8012 std::next(MachineBasicBlock::iterator(MI)), BB->end()); in EmitStructByval()
8013 exitMBB->transferSuccessorsAndUpdatePHIs(BB); in EmitStructByval()
8017 if (Subtarget->useMovt(*MF)) { in EmitStructByval()
8022 TII->get(IsThumb2 ? ARM::t2MOVi16 : ARM::MOVi16), in EmitStructByval()
8027 TII->get(IsThumb2 ? ARM::t2MOVTi16 : ARM::MOVTi16), in EmitStructByval()
8032 MachineConstantPool *ConstantPool = MF->getConstantPool(); in EmitStructByval()
8033 Type *Int32Ty = Type::getInt32Ty(MF->getFunction()->getContext()); in EmitStructByval()
8037 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty); in EmitStructByval()
8039 Align = MF->getDataLayout().getTypeAllocSize(C->getType()); in EmitStructByval()
8040 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align); in EmitStructByval()
8043 AddDefaultPred(BuildMI(*BB, MI, dl, TII->get(ARM::tLDRpci)).addReg( in EmitStructByval()
8046 AddDefaultPred(BuildMI(*BB, MI, dl, TII->get(ARM::LDRcp)).addReg( in EmitStructByval()
8049 BB->addSuccessor(loopMBB); in EmitStructByval()
8064 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), varPhi) in EmitStructByval()
8067 BuildMI(BB, dl, TII->get(ARM::PHI), srcPhi) in EmitStructByval()
8070 BuildMI(BB, dl, TII->get(ARM::PHI), destPhi) in EmitStructByval()
8077 emitPostLd(BB, BB->end(), TII, dl, UnitSize, scratch, srcPhi, srcLoop, in EmitStructByval()
8079 emitPostSt(BB, BB->end(), TII, dl, UnitSize, scratch, destPhi, destLoop, in EmitStructByval()
8085 BuildMI(*BB, BB->end(), dl, TII->get(ARM::tSUBi8), varLoop); in EmitStructByval()
8091 BuildMI(*BB, BB->end(), dl, in EmitStructByval()
8092 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop); in EmitStructByval()
8094 MIB->getOperand(5).setReg(ARM::CPSR); in EmitStructByval()
8095 MIB->getOperand(5).setIsDef(true); in EmitStructByval()
8097 BuildMI(*BB, BB->end(), dl, in EmitStructByval()
8098 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc)) in EmitStructByval()
8102 BB->addSuccessor(loopMBB); in EmitStructByval()
8103 BB->addSuccessor(exitMBB); in EmitStructByval()
8107 auto StartOfExit = exitMBB->begin(); in EmitStructByval()
8133 const TargetInstrInfo &TII = *Subtarget->getInstrInfo(); in EmitLowered__chkstk()
8136 assert(Subtarget->isTargetWindows() && in EmitLowered__chkstk()
8138 assert(Subtarget->isThumb2() && "Windows on ARM requires Thumb-2 mode"); in EmitLowered__chkstk()
8146 // thumb-2 environment, so there is no interworking required. As a result, we in EmitLowered__chkstk()
8154 // branches for Thumb), we can generate the long-call version via in EmitLowered__chkstk()
8155 // -mcmodel=large, alleviating the need for the trampoline which may clobber in EmitLowered__chkstk()
8172 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); in EmitLowered__chkstk()
8201 MachineFunction *MF = MBB->getParent(); in EmitLowered__dbzchk()
8202 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in EmitLowered__dbzchk()
8204 MachineBasicBlock *ContBB = MF->CreateMachineBasicBlock(); in EmitLowered__dbzchk()
8205 MF->insert(++MBB->getIterator(), ContBB); in EmitLowered__dbzchk()
8206 ContBB->splice(ContBB->begin(), MBB, in EmitLowered__dbzchk()
8207 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); in EmitLowered__dbzchk()
8208 ContBB->transferSuccessorsAndUpdatePHIs(MBB); in EmitLowered__dbzchk()
8210 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock(); in EmitLowered__dbzchk()
8211 MF->push_back(TrapBB); in EmitLowered__dbzchk()
8212 BuildMI(TrapBB, DL, TII->get(ARM::t2UDF)).addImm(249); in EmitLowered__dbzchk()
8213 MBB->addSuccessor(TrapBB); in EmitLowered__dbzchk()
8215 BuildMI(*MBB, MI, DL, TII->get(ARM::tCBZ)) in EmitLowered__dbzchk()
8218 AddDefaultPred(BuildMI(*MBB, MI, DL, TII->get(ARM::t2B)).addMBB(ContBB)); in EmitLowered__dbzchk()
8219 MBB->addSuccessor(ContBB); in EmitLowered__dbzchk()
8228 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in EmitInstrWithCustomInserter()
8230 bool isThumb2 = Subtarget->isThumb2(); in EmitInstrWithCustomInserter()
8236 // The Thumb2 pre-indexed stores have the same MI operands, they just in EmitInstrWithCustomInserter()
8240 MI.setDesc(TII->get(ARM::t2STR_PRE)); in EmitInstrWithCustomInserter()
8243 MI.setDesc(TII->get(ARM::t2STRB_PRE)); in EmitInstrWithCustomInserter()
8246 MI.setDesc(TII->get(ARM::t2STRH_PRE)); in EmitInstrWithCustomInserter()
8258 Offset = -Offset; in EmitInstrWithCustomInserter()
8261 BuildMI(*BB, MI, dl, TII->get(NewOpc)) in EmitInstrWithCustomInserter()
8282 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(NewOpc)); in EmitInstrWithCustomInserter()
8291 // diamond control-flow pattern. The incoming instruction knows the in EmitInstrWithCustomInserter()
8294 const BasicBlock *LLVM_BB = BB->getBasicBlock(); in EmitInstrWithCustomInserter()
8295 MachineFunction::iterator It = ++BB->getIterator(); in EmitInstrWithCustomInserter()
8302 // fallthrough --> copy0MBB in EmitInstrWithCustomInserter()
8304 MachineFunction *F = BB->getParent(); in EmitInstrWithCustomInserter()
8305 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); in EmitInstrWithCustomInserter()
8306 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); in EmitInstrWithCustomInserter()
8307 F->insert(It, copy0MBB); in EmitInstrWithCustomInserter()
8308 F->insert(It, sinkMBB); in EmitInstrWithCustomInserter()
8311 sinkMBB->splice(sinkMBB->begin(), BB, in EmitInstrWithCustomInserter()
8312 std::next(MachineBasicBlock::iterator(MI)), BB->end()); in EmitInstrWithCustomInserter()
8313 sinkMBB->transferSuccessorsAndUpdatePHIs(BB); in EmitInstrWithCustomInserter()
8315 BB->addSuccessor(copy0MBB); in EmitInstrWithCustomInserter()
8316 BB->addSuccessor(sinkMBB); in EmitInstrWithCustomInserter()
8318 BuildMI(BB, dl, TII->get(ARM::tBcc)) in EmitInstrWithCustomInserter()
8328 // Update machine-CFG edges in EmitInstrWithCustomInserter()
8329 BB->addSuccessor(sinkMBB); in EmitInstrWithCustomInserter()
8335 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), MI.getOperand(0).getReg()) in EmitInstrWithCustomInserter()
8348 BB->erase(std::next(MachineBasicBlock::iterator(MI)), BB->end()); in EmitInstrWithCustomInserter()
8358 TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri)) in EmitInstrWithCustomInserter()
8360 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri)) in EmitInstrWithCustomInserter()
8367 TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr)) in EmitInstrWithCustomInserter()
8369 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr)) in EmitInstrWithCustomInserter()
8379 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)) in EmitInstrWithCustomInserter()
8382 AddDefaultPred(BuildMI(BB, dl, TII->get(ARM::t2B)).addMBB(exitMBB)); in EmitInstrWithCustomInserter()
8384 BuildMI(BB, dl, TII->get(ARM::B)) .addMBB(exitMBB); in EmitInstrWithCustomInserter()
8404 // diamond control-flow pattern. The incoming instruction knows the in EmitInstrWithCustomInserter()
8415 const BasicBlock *LLVM_BB = BB->getBasicBlock(); in EmitInstrWithCustomInserter()
8416 MachineFunction::iterator BBI = ++BB->getIterator(); in EmitInstrWithCustomInserter()
8417 MachineFunction *Fn = BB->getParent(); in EmitInstrWithCustomInserter()
8418 MachineBasicBlock *RSBBB = Fn->CreateMachineBasicBlock(LLVM_BB); in EmitInstrWithCustomInserter()
8419 MachineBasicBlock *SinkBB = Fn->CreateMachineBasicBlock(LLVM_BB); in EmitInstrWithCustomInserter()
8420 Fn->insert(BBI, RSBBB); in EmitInstrWithCustomInserter()
8421 Fn->insert(BBI, SinkBB); in EmitInstrWithCustomInserter()
8426 bool isThumb2 = Subtarget->isThumb2(); in EmitInstrWithCustomInserter()
8427 MachineRegisterInfo &MRI = Fn->getRegInfo(); in EmitInstrWithCustomInserter()
8434 SinkBB->splice(SinkBB->begin(), BB, in EmitInstrWithCustomInserter()
8435 std::next(MachineBasicBlock::iterator(MI)), BB->end()); in EmitInstrWithCustomInserter()
8436 SinkBB->transferSuccessorsAndUpdatePHIs(BB); in EmitInstrWithCustomInserter()
8438 BB->addSuccessor(RSBBB); in EmitInstrWithCustomInserter()
8439 BB->addSuccessor(SinkBB); in EmitInstrWithCustomInserter()
8442 RSBBB->addSuccessor(SinkBB); in EmitInstrWithCustomInserter()
8446 TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri)) in EmitInstrWithCustomInserter()
8451 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).addMBB(SinkBB) in EmitInstrWithCustomInserter()
8456 // by if-conversion pass in EmitInstrWithCustomInserter()
8457 BuildMI(*RSBBB, RSBBB->begin(), dl, in EmitInstrWithCustomInserter()
8458 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg) in EmitInstrWithCustomInserter()
8464 BuildMI(*SinkBB, SinkBB->begin(), dl, in EmitInstrWithCustomInserter()
8465 TII->get(ARM::PHI), ABSDstReg) in EmitInstrWithCustomInserter()
8486 /// when it is expanded into LDM/STM. This is done as a post-isel lowering
8490 bool isThumb1 = Subtarget->isThumb1Only(); in attachMEMCPYScratchRegs()
8493 MachineFunction *MF = MI.getParent()->getParent(); in attachMEMCPYScratchRegs()
8494 MachineRegisterInfo &MRI = MF->getRegInfo(); in attachMEMCPYScratchRegs()
8498 if (!Node->hasAnyUseOfValue(0)) { in attachMEMCPYScratchRegs()
8501 if (!Node->hasAnyUseOfValue(1)) { in attachMEMCPYScratchRegs()
8526 // e.g. ADCS (..., CPSR<imp-def>) -> ADC (... opt:CPSR<def>). in AdjustInstrPostInstrSelection()
8531 const ARMBaseInstrInfo *TII = Subtarget->getInstrInfo(); in AdjustInstrPostInstrSelection()
8532 MCID = &TII->get(NewOpc); in AdjustInstrPostInstrSelection()
8534 assert(MCID->getNumOperands() == MI.getDesc().getNumOperands() + 1 && in AdjustInstrPostInstrSelection()
8542 unsigned ccOutIdx = MCID->getNumOperands() - 1; in AdjustInstrPostInstrSelection()
8546 if (!MI.hasOptionalDef() || !MCID->OpInfo[ccOutIdx].isOptionalDef()) { in AdjustInstrPostInstrSelection()
8554 for (unsigned i = MCID->getNumOperands(), e = MI.getNumOperands(); i != e; in AdjustInstrPostInstrSelection()
8569 assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag"); in AdjustInstrPostInstrSelection()
8583 //===----------------------------------------------------------------------===//
8585 //===----------------------------------------------------------------------===//
8599 // (select cc -1, y) [AllOnes=1]
8600 // (select cc y, -1) [AllOnes=1]
8608 switch (N->getOpcode()) { in isConditionalZeroOrAllOnes()
8611 CC = N->getOperand(0); in isConditionalZeroOrAllOnes()
8612 SDValue N1 = N->getOperand(1); in isConditionalZeroOrAllOnes()
8613 SDValue N2 = N->getOperand(2); in isConditionalZeroOrAllOnes()
8633 EVT VT = N->getValueType(0); in isConditionalZeroOrAllOnes()
8634 CC = N->getOperand(0); in isConditionalZeroOrAllOnes()
8642 else if (N->getOpcode() == ISD::ZERO_EXTEND) in isConditionalZeroOrAllOnes()
8655 // (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
8656 // (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
8657 // (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) [AllOnes=1]
8658 // (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
8659 // (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
8666 // (add (zext cc), x) -> (select cc (add x, 1), x)
8667 // (add (sext cc), x) -> (select cc (add x, -1), x)
8682 EVT VT = N->getValueType(0); in combineSelectAndUse()
8692 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, in combineSelectAndUse()
8706 SDValue N0 = N->getOperand(0); in combineSelectAndUseCommutative()
8707 SDValue N1 = N->getOperand(1); in combineSelectAndUseCommutative()
8708 if (N0.getNode()->hasOneUse()) in combineSelectAndUseCommutative()
8711 if (N1.getNode()->hasOneUse()) in combineSelectAndUseCommutative()
8717 // AddCombineToVPADDL- For pair-wise add on neon, use the vpaddl instruction
8725 if (DCI.isBeforeLegalize() || !Subtarget->hasNEON() in AddCombineToVPADDL()
8731 EVT VT = N->getValueType(0); in AddCombineToVPADDL()
8742 if (N0->getOperand(0)->getOpcode() != ISD::EXTRACT_VECTOR_ELT) in AddCombineToVPADDL()
8744 SDValue Vec = N0->getOperand(0)->getOperand(0); in AddCombineToVPADDL()
8751 for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i) { in AddCombineToVPADDL()
8752 if (N0->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT in AddCombineToVPADDL()
8753 && N1->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in AddCombineToVPADDL()
8755 SDValue ExtVec0 = N0->getOperand(i); in AddCombineToVPADDL()
8756 SDValue ExtVec1 = N1->getOperand(i); in AddCombineToVPADDL()
8759 if (V != ExtVec0->getOperand(0).getNode() || in AddCombineToVPADDL()
8760 V != ExtVec1->getOperand(0).getNode()) in AddCombineToVPADDL()
8764 ConstantSDNode *C0 = dyn_cast<ConstantSDNode>(ExtVec0->getOperand(1)); in AddCombineToVPADDL()
8765 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(ExtVec1->getOperand(1)); in AddCombineToVPADDL()
8768 if (!C0 || !C1 || C0->getZExtValue() != nextIndex in AddCombineToVPADDL()
8769 || C1->getZExtValue() != nextIndex+1) in AddCombineToVPADDL()
8811 if (V->getOpcode() == ISD::UMUL_LOHI || in findMUL_LOHI()
8812 V->getOpcode() == ISD::SMUL_LOHI) in findMUL_LOHI()
8830 // loAdd -> ADDE | in AddCombineTo64bitMLAL()
8833 // ADDC <- hiAdd in AddCombineTo64bitMLAL()
8835 assert(AddcNode->getOpcode() == ISD::ADDC && "Expect an ADDC"); in AddCombineTo64bitMLAL()
8836 SDValue AddcOp0 = AddcNode->getOperand(0); in AddCombineTo64bitMLAL()
8837 SDValue AddcOp1 = AddcNode->getOperand(1); in AddCombineTo64bitMLAL()
8843 assert(AddcNode->getNumValues() == 2 && in AddCombineTo64bitMLAL()
8844 AddcNode->getValueType(0) == MVT::i32 && in AddCombineTo64bitMLAL()
8848 if (AddcNode->getValueType(1) != MVT::Glue) in AddCombineTo64bitMLAL()
8852 if (AddcOp0->getOpcode() != ISD::UMUL_LOHI && in AddCombineTo64bitMLAL()
8853 AddcOp0->getOpcode() != ISD::SMUL_LOHI && in AddCombineTo64bitMLAL()
8854 AddcOp1->getOpcode() != ISD::UMUL_LOHI && in AddCombineTo64bitMLAL()
8855 AddcOp1->getOpcode() != ISD::SMUL_LOHI) in AddCombineTo64bitMLAL()
8859 SDNode* AddeNode = AddcNode->getGluedUser(); in AddCombineTo64bitMLAL()
8864 if (AddeNode->getOpcode() != ISD::ADDE) in AddCombineTo64bitMLAL()
8867 assert(AddeNode->getNumOperands() == 3 && in AddCombineTo64bitMLAL()
8868 AddeNode->getOperand(2).getValueType() == MVT::Glue && in AddCombineTo64bitMLAL()
8872 SDValue AddeOp0 = AddeNode->getOperand(0); in AddCombineTo64bitMLAL()
8873 SDValue AddeOp1 = AddeNode->getOperand(1); in AddCombineTo64bitMLAL()
8890 unsigned Opc = MULOp->getOpcode(); in AddCombineTo64bitMLAL()
8928 Ops.push_back(LoMul->getOperand(0)); in AddCombineTo64bitMLAL()
8929 Ops.push_back(LoMul->getOperand(1)); in AddCombineTo64bitMLAL()
8957 if (!Subtarget->hasV6Ops()) in AddCombineTo64bitUMAAL()
8961 if (AddcNode->getOperand(0).getOpcode() == ISD::ADDC) in AddCombineTo64bitUMAAL()
8962 PrevAddc = AddcNode->getOperand(0).getNode(); in AddCombineTo64bitUMAAL()
8963 else if (AddcNode->getOperand(1).getOpcode() == ISD::ADDC) in AddCombineTo64bitUMAAL()
8964 PrevAddc = AddcNode->getOperand(1).getNode(); in AddCombineTo64bitUMAAL()
8979 if (AddcNode->getOperand(0).getOpcode() == ARMISD::UMLAL) { in AddCombineTo64bitUMAAL()
8980 UmlalNode = AddcNode->getOperand(0).getNode(); in AddCombineTo64bitUMAAL()
8981 AddHi = AddcNode->getOperand(1); in AddCombineTo64bitUMAAL()
8982 } else if (AddcNode->getOperand(1).getOpcode() == ARMISD::UMLAL) { in AddCombineTo64bitUMAAL()
8983 UmlalNode = AddcNode->getOperand(1).getNode(); in AddCombineTo64bitUMAAL()
8984 AddHi = AddcNode->getOperand(0); in AddCombineTo64bitUMAAL()
8991 auto *Zero = dyn_cast<ConstantSDNode>(UmlalNode->getOperand(3)); in AddCombineTo64bitUMAAL()
8993 if (!Zero || Zero->getZExtValue() != 0) in AddCombineTo64bitUMAAL()
8997 if (AddcNode->getValueType(1) != MVT::Glue) in AddCombineTo64bitUMAAL()
9001 SDNode* AddeNode = AddcNode->getGluedUser(); in AddCombineTo64bitUMAAL()
9005 if ((AddeNode->getOperand(0).getNode() == Zero && in AddCombineTo64bitUMAAL()
9006 AddeNode->getOperand(1).getNode() == UmlalNode) || in AddCombineTo64bitUMAAL()
9007 (AddeNode->getOperand(0).getNode() == UmlalNode && in AddCombineTo64bitUMAAL()
9008 AddeNode->getOperand(1).getNode() == Zero)) { in AddCombineTo64bitUMAAL()
9011 SDValue Ops[] = { UmlalNode->getOperand(0), UmlalNode->getOperand(1), in AddCombineTo64bitUMAAL()
9012 UmlalNode->getOperand(2), AddHi }; in AddCombineTo64bitUMAAL()
9026 /// PerformADDCCombine - Target-specific dag combine transform from
9033 if (Subtarget->isThumb1Only()) return SDValue(); in PerformADDCCombine()
9041 /// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
9053 // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c)) in PerformADDCombineWithOperands()
9054 if (N0.getNode()->hasOneUse()) in PerformADDCombineWithOperands()
9060 /// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
9065 SDValue N0 = N->getOperand(0); in PerformADDCombine()
9066 SDValue N1 = N->getOperand(1); in PerformADDCombine()
9076 /// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
9080 SDValue N0 = N->getOperand(0); in PerformSUBCombine()
9081 SDValue N1 = N->getOperand(1); in PerformSUBCombine()
9083 // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c)) in PerformSUBCombine()
9084 if (N1.getNode()->hasOneUse()) in PerformSUBCombine()
9094 /// vmul d3, d0, d2
9095 /// vmla d3, d1, d2
9097 /// vadd d3, d0, d1
9098 /// vmul d3, d3, d2
9101 // vmul d3, d0, d2
9102 // vmla d3, d1, d2
9105 // vmul d3, d2, d2
9109 if (!Subtarget->hasVMLxForwarding()) in PerformVMULCombine()
9113 SDValue N0 = N->getOperand(0); in PerformVMULCombine()
9114 SDValue N1 = N->getOperand(1); in PerformVMULCombine()
9128 EVT VT = N->getValueType(0); in PerformVMULCombine()
9130 SDValue N00 = N0->getOperand(0); in PerformVMULCombine()
9131 SDValue N01 = N0->getOperand(1); in PerformVMULCombine()
9142 if (Subtarget->isThumb1Only()) in PerformMULCombine()
9148 EVT VT = N->getValueType(0); in PerformMULCombine()
9154 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); in PerformMULCombine()
9158 int64_t MulAmt = C->getSExtValue(); in PerformMULCombine()
9161 ShiftAmt = ShiftAmt & (32 - 1); in PerformMULCombine()
9162 SDValue V = N->getOperand(0); in PerformMULCombine()
9169 if (isPowerOf2_32(MulAmt - 1)) { in PerformMULCombine()
9175 DAG.getConstant(Log2_32(MulAmt - 1), DL, in PerformMULCombine()
9178 // (mul x, 2^N - 1) => (sub (shl x, N), x) in PerformMULCombine()
9188 uint64_t MulAmtAbs = -MulAmt; in PerformMULCombine()
9190 // (mul x, -(2^N - 1)) => (sub x, (shl x, N)) in PerformMULCombine()
9197 } else if (isPowerOf2_32(MulAmtAbs - 1)) { in PerformMULCombine()
9198 // (mul x, -(2^N + 1)) => - (add (shl x, N), x) in PerformMULCombine()
9203 DAG.getConstant(Log2_32(MulAmtAbs - 1), DL, in PerformMULCombine()
9225 // Attempt to use immediate-form VBIC in PerformANDCombine()
9226 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1)); in PerformANDCombine()
9228 EVT VT = N->getValueType(0); in PerformANDCombine()
9238 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) { in PerformANDCombine()
9247 DAG.getNode(ISD::BITCAST, dl, VbicVT, N->getOperand(0)); in PerformANDCombine()
9254 if (!Subtarget->isThumb1Only()) { in PerformANDCombine()
9255 // fold (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) in PerformANDCombine()
9263 /// PerformORCombine - Target-specific dag combine xforms for ISD::OR
9267 // Attempt to use immediate-form VORR in PerformORCombine()
9268 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1)); in PerformORCombine()
9270 EVT VT = N->getValueType(0); in PerformORCombine()
9279 if (BVN && Subtarget->hasNEON() && in PerformORCombine()
9280 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) { in PerformORCombine()
9289 DAG.getNode(ISD::BITCAST, dl, VorrVT, N->getOperand(0)); in PerformORCombine()
9296 if (!Subtarget->isThumb1Only()) { in PerformORCombine()
9297 // fold (or (select cc, 0, c), x) -> (select cc, x, (or, x, c)) in PerformORCombine()
9305 SDValue N0 = N->getOperand(0); in PerformORCombine()
9308 SDValue N1 = N->getOperand(1); in PerformORCombine()
9311 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() && in PerformORCombine()
9318 BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(1)); in PerformORCombine()
9319 BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(1)); in PerformORCombine()
9321 if (BVN0 && BVN0->isConstantSplat(SplatBits0, SplatUndef, SplatBitSize, in PerformORCombine()
9323 if (BVN1 && BVN1->isConstantSplat(SplatBits1, SplatUndef, SplatBitSize, in PerformORCombine()
9334 N0->getOperand(1), in PerformORCombine()
9335 N0->getOperand(0), in PerformORCombine()
9336 N1->getOperand(0)); in PerformORCombine()
9347 if (Subtarget->isThumb1Only() || !Subtarget->hasV6T2Ops()) in PerformORCombine()
9373 unsigned Mask = MaskC->getZExtValue(); in PerformORCombine()
9380 unsigned Val = N1C->getZExtValue(); in PerformORCombine()
9400 unsigned Mask2 = N11C->getZExtValue(); in PerformORCombine()
9408 if (Subtarget->hasT2ExtractPack() && in PerformORCombine()
9424 if (Subtarget->hasT2ExtractPack() && in PerformORCombine()
9439 if (DAG.MaskedValueIsZero(N1, MaskC->getAPIntValue()) && in PerformORCombine()
9445 unsigned ShAmtC = cast<ConstantSDNode>(ShAmt)->getZExtValue(); in PerformORCombine()
9463 EVT VT = N->getValueType(0); in PerformXORCombine()
9469 if (!Subtarget->isThumb1Only()) { in PerformXORCombine()
9470 // fold (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c)) in PerformXORCombine()
9478 // ParseBFI - given a BFI instruction in N, extract the "from" value (Rn) and return it,
9482 assert(N->getOpcode() == ARMISD::BFI); in ParseBFI()
9484 SDValue From = N->getOperand(1); in ParseBFI()
9485 ToMask = ~cast<ConstantSDNode>(N->getOperand(2))->getAPIntValue(); in ParseBFI()
9490 if (From->getOpcode() == ISD::SRL && in ParseBFI()
9491 isa<ConstantSDNode>(From->getOperand(1))) { in ParseBFI()
9492 APInt Shift = cast<ConstantSDNode>(From->getOperand(1))->getAPIntValue(); in ParseBFI()
9495 From = From->getOperand(0); in ParseBFI()
9506 unsigned FirstActiveBitInB = B.getBitWidth() - B.countLeadingZeros() - 1; in BitsProperlyConcatenate()
9507 return LastActiveBitInA - 1 == FirstActiveBitInB; in BitsProperlyConcatenate()
9515 SDValue To = N->getOperand(0); in FindBFIToCombineWith()
9534 // Conflicting bits - bail out because going further is unsafe. in FindBFIToCombineWith()
9556 SDValue N1 = N->getOperand(1); in PerformBFICombine()
9558 // (bfi A, (and B, Mask1), Mask2) -> (bfi A, B, Mask2) iff in PerformBFICombine()
9563 unsigned InvMask = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue(); in PerformBFICombine()
9565 unsigned Width = (32 - countLeadingZeros(~InvMask)) - LSB; in PerformBFICombine()
9569 unsigned Mask = (1u << Width) - 1; in PerformBFICombine()
9570 unsigned Mask2 = N11C->getZExtValue(); in PerformBFICombine()
9572 return DCI.DAG.getNode(ARMISD::BFI, SDLoc(N), N->getValueType(0), in PerformBFICombine()
9573 N->getOperand(0), N1.getOperand(0), in PerformBFICombine()
9574 N->getOperand(2)); in PerformBFICombine()
9575 } else if (N->getOperand(0).getOpcode() == ARMISD::BFI) { in PerformBFICombine()
9598 EVT VT = N->getValueType(0); in PerformBFICombine()
9605 return DCI.DAG.getNode(ARMISD::BFI, dl, VT, N->getOperand(0), From1, in PerformBFICombine()
9611 /// PerformVMOVRRDCombine - Target-specific dag combine xforms for
9616 // vmovrrd(vmovdrr x, y) -> x,y in PerformVMOVRRDCombine()
9617 SDValue InDouble = N->getOperand(0); in PerformVMOVRRDCombine()
9618 if (InDouble.getOpcode() == ARMISD::VMOVDRR && !Subtarget->isFPOnlySP()) in PerformVMOVRRDCombine()
9621 // vmovrrd(load f64) -> (load i32), (load i32) in PerformVMOVRRDCombine()
9623 if (ISD::isNormalLoad(InNode) && InNode->hasOneUse() && in PerformVMOVRRDCombine()
9624 InNode->getValueType(0) == MVT::f64 && in PerformVMOVRRDCombine()
9625 InNode->getOperand(1).getOpcode() == ISD::FrameIndex && in PerformVMOVRRDCombine()
9626 !cast<LoadSDNode>(InNode)->isVolatile()) { in PerformVMOVRRDCombine()
9627 // TODO: Should this be done for non-FrameIndex operands? in PerformVMOVRRDCombine()
9632 SDValue BasePtr = LD->getBasePtr(); in PerformVMOVRRDCombine()
9633 SDValue NewLD1 = DAG.getLoad(MVT::i32, DL, LD->getChain(), BasePtr, in PerformVMOVRRDCombine()
9634 LD->getPointerInfo(), LD->isVolatile(), in PerformVMOVRRDCombine()
9635 LD->isNonTemporal(), LD->isInvariant(), in PerformVMOVRRDCombine()
9636 LD->getAlignment()); in PerformVMOVRRDCombine()
9641 LD->getPointerInfo(), LD->isVolatile(), in PerformVMOVRRDCombine()
9642 LD->isNonTemporal(), LD->isInvariant(), in PerformVMOVRRDCombine()
9643 std::min(4U, LD->getAlignment() / 2)); in PerformVMOVRRDCombine()
9655 /// PerformVMOVDRRCombine - Target-specific dag combine xforms for
9658 // N=vmovrrd(X); vmovdrr(N:0, N:1) -> bit_convert(X) in PerformVMOVDRRCombine()
9659 SDValue Op0 = N->getOperand(0); in PerformVMOVDRRCombine()
9660 SDValue Op1 = N->getOperand(1); in PerformVMOVDRRCombine()
9669 N->getValueType(0), Op0.getOperand(0)); in PerformVMOVDRRCombine()
9673 /// hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node
9674 /// are normal, non-volatile loads. If so, it is profitable to bitcast an
9678 unsigned NumElts = N->getValueType(0).getVectorNumElements(); in hasNormalLoadOperand()
9680 SDNode *Elt = N->getOperand(i).getNode(); in hasNormalLoadOperand()
9681 if (ISD::isNormalLoad(Elt) && !cast<LoadSDNode>(Elt)->isVolatile()) in hasNormalLoadOperand()
9687 /// PerformBUILD_VECTORCombine - Target-specific dag combine xforms for
9692 // build_vector(N=ARMISD::VMOVRRD(X), N:1) -> bit_convert(X): in PerformBUILD_VECTORCombine()
9697 if (N->getNumOperands() == 2) in PerformBUILD_VECTORCombine()
9703 EVT VT = N->getValueType(0); in PerformBUILD_VECTORCombine()
9710 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(i)); in PerformBUILD_VECTORCombine()
9720 /// \brief Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
9734 // 2. The size of its operands are 32-bits (64-bits are not legal). in PerformARMBUILD_VECTORCombine()
9735 EVT VT = N->getValueType(0); in PerformARMBUILD_VECTORCombine()
9739 if (EltVT.getSizeInBits() != 32 || !N->hasOneUse()) in PerformARMBUILD_VECTORCombine()
9746 SDNode *Use = *N->use_begin(); in PerformARMBUILD_VECTORCombine()
9747 if (Use->getOpcode() != ISD::BITCAST || in PerformARMBUILD_VECTORCombine()
9748 Use->getValueType(0).isFloatingPoint()) in PerformARMBUILD_VECTORCombine()
9760 SDValue Elt = N->getOperand(Idx); in PerformARMBUILD_VECTORCombine()
9761 if (Elt->getOpcode() == ISD::BITCAST) { in PerformARMBUILD_VECTORCombine()
9763 if (Elt->getOperand(0).getValueType() == MVT::i32) in PerformARMBUILD_VECTORCombine()
9768 --NumOfRelevantElts; in PerformARMBUILD_VECTORCombine()
9771 // Check if more than half of the elements require a non-free bitcast. in PerformARMBUILD_VECTORCombine()
9786 // (INSERT_VECTOR_ELT (...), (BITCAST EN-1), N-1), in PerformARMBUILD_VECTORCombine()
9791 SDValue V = N->getOperand(Idx); in PerformARMBUILD_VECTORCombine()
9795 V->getOperand(0).getValueType() == MVT::i32) in PerformARMBUILD_VECTORCombine()
9812 /// PerformInsertEltCombine - Target-specific dag combine xforms for
9818 EVT VT = N->getValueType(0); in PerformInsertEltCombine()
9819 SDNode *Elt = N->getOperand(1).getNode(); in PerformInsertEltCombine()
9821 !ISD::isNormalLoad(Elt) || cast<LoadSDNode>(Elt)->isVolatile()) in PerformInsertEltCombine()
9828 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, N->getOperand(0)); in PerformInsertEltCombine()
9829 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(1)); in PerformInsertEltCombine()
9834 Vec, V, N->getOperand(2)); in PerformInsertEltCombine()
9838 /// PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for
9846 // targets, but for NEON it is better to concatenate two double-register in PerformVECTOR_SHUFFLECombine()
9847 // size vector operands into a single quad-register size vector. Do that in PerformVECTOR_SHUFFLECombine()
9849 // shuffle(concat(v1, undef), concat(v2, undef)) -> in PerformVECTOR_SHUFFLECombine()
9851 SDValue Op0 = N->getOperand(0); in PerformVECTOR_SHUFFLECombine()
9852 SDValue Op1 = N->getOperand(1); in PerformVECTOR_SHUFFLECombine()
9864 EVT VT = N->getValueType(0); in PerformVECTOR_SHUFFLECombine()
9878 int MaskElt = SVN->getMaskElt(n); in PerformVECTOR_SHUFFLECombine()
9879 int NewElt = -1; in PerformVECTOR_SHUFFLECombine()
9883 NewElt = HalfElts + MaskElt - NumElts; in PerformVECTOR_SHUFFLECombine()
9890 /// CombineBaseUpdate - Target-specific DAG combine function for VLDDUP,
9898 const bool isIntrinsic = (N->getOpcode() == ISD::INTRINSIC_VOID || in CombineBaseUpdate()
9899 N->getOpcode() == ISD::INTRINSIC_W_CHAIN); in CombineBaseUpdate()
9900 const bool isStore = N->getOpcode() == ISD::STORE; in CombineBaseUpdate()
9902 SDValue Addr = N->getOperand(AddrOpIdx); in CombineBaseUpdate()
9907 for (SDNode::use_iterator UI = Addr.getNode()->use_begin(), in CombineBaseUpdate()
9908 UE = Addr.getNode()->use_end(); UI != UE; ++UI) { in CombineBaseUpdate()
9910 if (User->getOpcode() != ISD::ADD || in CombineBaseUpdate()
9916 if (User->isPredecessorOf(N) || N->isPredecessorOf(User)) in CombineBaseUpdate()
9925 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); in CombineBaseUpdate()
9959 switch (N->getOpcode()) { in CombineBaseUpdate()
9974 VecTy = N->getValueType(0); in CombineBaseUpdate()
9976 VecTy = N->getOperand(AddrOpIdx+1).getValueType(); in CombineBaseUpdate()
9979 VecTy = N->getOperand(1).getValueType(); in CombineBaseUpdate()
9987 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0); in CombineBaseUpdate()
9989 uint64_t IncVal = CInc->getZExtValue(); in CombineBaseUpdate()
9993 // VLD3/4 and VST3/4 for 128-bit vectors are implemented with two in CombineBaseUpdate()
9994 // separate instructions that make it harder to use a non-constant update. in CombineBaseUpdate()
10002 unsigned Alignment = MemN->getAlignment(); in CombineBaseUpdate()
10004 // If this is a less-than-standard-aligned load/store, change the type to in CombineBaseUpdate()
10008 // There are 3 ways to get to this base-update combine: in CombineBaseUpdate()
10009 // - intrinsics: they are assumed to be properly aligned (to the standard in CombineBaseUpdate()
10011 // - ARMISD::VLDx nodes: they are only generated from the aforementioned in CombineBaseUpdate()
10013 // - generic load/store instructions: the alignment is specified as an in CombineBaseUpdate()
10017 // generate non-standard-aligned ARMISD::VLDx nodes. in CombineBaseUpdate()
10023 assert(NumVecs == 1 && "Unexpected multi-element generic load/store."); in CombineBaseUpdate()
10051 Ops.push_back(N->getOperand(0)); // incoming chain in CombineBaseUpdate()
10052 Ops.push_back(N->getOperand(AddrOpIdx)); in CombineBaseUpdate()
10057 Ops.push_back(StN->getValue()); in CombineBaseUpdate()
10061 for (unsigned i = AddrOpIdx + 1; i < N->getNumOperands() - 1; ++i) in CombineBaseUpdate()
10062 Ops.push_back(N->getOperand(i)); in CombineBaseUpdate()
10068 // If this is a non-standard-aligned STORE, the penultimate operand is the in CombineBaseUpdate()
10070 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) { in CombineBaseUpdate()
10071 SDValue &StVal = Ops[Ops.size()-2]; in CombineBaseUpdate()
10077 MemN->getMemOperand()); in CombineBaseUpdate()
10084 // If this is an non-standard-aligned LOAD, the first result is the loaded in CombineBaseUpdate()
10086 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) { in CombineBaseUpdate()
10108 /// CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a
10109 /// vldN-lane (N > 1) intrinsic, and if all the other uses of that intrinsic
10110 /// are also VDUPLANEs. If so, combine them to a vldN-dup operation and
10114 EVT VT = N->getValueType(0); in CombineVLDDUP()
10115 // vldN-dup instructions only support 64-bit vectors for N > 1. in CombineVLDDUP()
10119 // Check if the VDUPLANE operand is a vldN-dup intrinsic. in CombineVLDDUP()
10120 SDNode *VLD = N->getOperand(0).getNode(); in CombineVLDDUP()
10121 if (VLD->getOpcode() != ISD::INTRINSIC_W_CHAIN) in CombineVLDDUP()
10125 unsigned IntNo = cast<ConstantSDNode>(VLD->getOperand(1))->getZExtValue(); in CombineVLDDUP()
10139 // First check that all the vldN-lane uses are VDUPLANEs and that the lane in CombineVLDDUP()
10142 cast<ConstantSDNode>(VLD->getOperand(NumVecs+3))->getZExtValue(); in CombineVLDDUP()
10143 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end(); in CombineVLDDUP()
10149 if (User->getOpcode() != ARMISD::VDUPLANE || in CombineVLDDUP()
10150 VLDLaneNo != cast<ConstantSDNode>(User->getOperand(1))->getZExtValue()) in CombineVLDDUP()
10154 // Create the vldN-dup node. in CombineVLDDUP()
10161 SDValue Ops[] = { VLD->getOperand(0), VLD->getOperand(2) }; in CombineVLDDUP()
10164 Ops, VLDMemInt->getMemoryVT(), in CombineVLDDUP()
10165 VLDMemInt->getMemOperand()); in CombineVLDDUP()
10168 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end(); in CombineVLDDUP()
10178 // Now the vldN-lane intrinsic is dead except for its chain result. in CombineVLDDUP()
10189 /// PerformVDUPLANECombine - Target-specific dag combine xforms for
10193 SDValue Op = N->getOperand(0); in PerformVDUPLANECombine()
10195 // If the source is a vldN-lane (N > 1) intrinsic, and all the other uses in PerformVDUPLANECombine()
10196 // of that intrinsic are also VDUPLANEs, combine them to a vldN-dup operation. in PerformVDUPLANECombine()
10209 // The canonical VMOV for a zero vector uses a 32-bit element size. in PerformVDUPLANECombine()
10210 unsigned Imm = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); in PerformVDUPLANECombine()
10214 EVT VT = N->getValueType(0); in PerformVDUPLANECombine()
10223 EVT VT = N->getValueType(0); in PerformLOADCombine()
10233 /// PerformSTORECombine - Target-specific dag combine xforms for
10238 if (St->isVolatile()) in PerformSTORECombine()
10244 SDValue StVal = St->getValue(); in PerformSTORECombine()
10246 if (St->isTruncatingStore() && VT.isVector()) { in PerformSTORECombine()
10249 EVT StVT = St->getMemoryVT(); in PerformSTORECombine()
10272 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1); in PerformSTORECombine()
10275 ? (i + 1) * SizeRatio - 1 in PerformSTORECombine()
10297 // Bitcast the original vector into a vector of store-size units in PerformSTORECombine()
10305 SDValue BasePtr = St->getBasePtr(); in PerformSTORECombine()
10313 SDValue Ch = DAG.getStore(St->getChain(), DL, SubVec, BasePtr, in PerformSTORECombine()
10314 St->getPointerInfo(), St->isVolatile(), in PerformSTORECombine()
10315 St->isNonTemporal(), St->getAlignment()); in PerformSTORECombine()
10328 if (StVal.getNode()->getOpcode() == ARMISD::VMOVDRR && in PerformSTORECombine()
10329 StVal.getNode()->hasOneUse()) { in PerformSTORECombine()
10333 SDValue BasePtr = St->getBasePtr(); in PerformSTORECombine()
10334 SDValue NewST1 = DAG.getStore(St->getChain(), DL, in PerformSTORECombine()
10335 StVal.getNode()->getOperand(isBigEndian ? 1 : 0 ), in PerformSTORECombine()
10336 BasePtr, St->getPointerInfo(), St->isVolatile(), in PerformSTORECombine()
10337 St->isNonTemporal(), St->getAlignment()); in PerformSTORECombine()
10342 StVal.getNode()->getOperand(isBigEndian ? 0 : 1), in PerformSTORECombine()
10343 OffsetPtr, St->getPointerInfo(), St->isVolatile(), in PerformSTORECombine()
10344 St->isNonTemporal(), in PerformSTORECombine()
10345 std::min(4U, St->getAlignment() / 2)); in PerformSTORECombine()
10349 StVal.getNode()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in PerformSTORECombine()
10367 return DAG.getStore(St->getChain(), dl, V, St->getBasePtr(), in PerformSTORECombine()
10368 St->getPointerInfo(), St->isVolatile(), in PerformSTORECombine()
10369 St->isNonTemporal(), St->getAlignment(), in PerformSTORECombine()
10370 St->getAAInfo()); in PerformSTORECombine()
10381 /// PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD)
10382 /// can replace combinations of VMUL and VCVT (floating-point to integer)
10392 if (!Subtarget->hasNEON()) in PerformVCVTCombine()
10395 SDValue Op = N->getOperand(0); in PerformVCVTCombine()
10400 SDValue ConstVec = Op->getOperand(1); in PerformVCVTCombine()
10406 MVT IntTy = N->getSimpleValueType(0).getVectorElementType(); in PerformVCVTCombine()
10419 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33); in PerformVCVTCombine()
10420 if (C == -1 || C == 0 || C > 32) in PerformVCVTCombine()
10424 bool isSigned = N->getOpcode() == ISD::FP_TO_SINT; in PerformVCVTCombine()
10429 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32), Op->getOperand(0), in PerformVCVTCombine()
10433 FixConv = DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), FixConv); in PerformVCVTCombine()
10438 /// PerformVDIVCombine - VCVT (fixed-point to floating-point, Advanced SIMD)
10439 /// can replace combinations of VCVT (integer to floating-point) and VDIV
10449 if (!Subtarget->hasNEON()) in PerformVDIVCombine()
10452 SDValue Op = N->getOperand(0); in PerformVDIVCombine()
10453 unsigned OpOpcode = Op.getNode()->getOpcode(); in PerformVDIVCombine()
10454 if (!N->getValueType(0).isVector() || !N->getValueType(0).isSimple() || in PerformVDIVCombine()
10458 SDValue ConstVec = N->getOperand(1); in PerformVDIVCombine()
10462 MVT FloatTy = N->getSimpleValueType(0).getVectorElementType(); in PerformVDIVCombine()
10477 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33); in PerformVDIVCombine()
10478 if (C == -1 || C == 0 || C > 32) in PerformVDIVCombine()
10497 /// Getvshiftimm - Check if this is a valid build_vector for the immediate
10508 if (! BVN || ! BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, in getVShiftImm()
10516 /// isVShiftLImm - Check if this is a valid build_vector for the immediate
10525 return (Cnt >= 0 && (isLong ? Cnt-1 : Cnt) < ElementBits); in isVShiftLImm()
10528 /// isVShiftRImm - Check if this is a valid build_vector for the immediate
10542 if (Cnt >= -(isNarrow ? ElementBits/2 : ElementBits) && Cnt <= -1) { in isVShiftRImm()
10543 Cnt = -Cnt; in isVShiftRImm()
10549 /// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
10551 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); in PerformIntrinsicCombine()
10559 // the build_vectors for 64-bit vector element shift counts are generally in PerformIntrinsicCombine()
10576 EVT VT = N->getOperand(1).getValueType(); in PerformIntrinsicCombine()
10583 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) { in PerformIntrinsicCombine()
10587 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) { in PerformIntrinsicCombine()
10596 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) in PerformIntrinsicCombine()
10602 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) in PerformIntrinsicCombine()
10607 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) in PerformIntrinsicCombine()
10619 if (isVShiftRImm(N->getOperand(2), VT, true, true, Cnt)) in PerformIntrinsicCombine()
10660 return DAG.getNode(VShiftOpc, dl, N->getValueType(0), in PerformIntrinsicCombine()
10661 N->getOperand(1), DAG.getConstant(Cnt, dl, MVT::i32)); in PerformIntrinsicCombine()
10665 EVT VT = N->getOperand(1).getValueType(); in PerformIntrinsicCombine()
10669 if (isVShiftLImm(N->getOperand(3), VT, false, Cnt)) in PerformIntrinsicCombine()
10671 else if (isVShiftRImm(N->getOperand(3), VT, false, true, Cnt)) in PerformIntrinsicCombine()
10678 return DAG.getNode(VShiftOpc, dl, N->getValueType(0), in PerformIntrinsicCombine()
10679 N->getOperand(1), N->getOperand(2), in PerformIntrinsicCombine()
10692 /// PerformShiftCombine - Checks for immediate versions of vector shifts and
10694 /// combining instead of DAG legalizing because the build_vectors for 64-bit
10699 EVT VT = N->getValueType(0); in PerformShiftCombine()
10700 if (N->getOpcode() == ISD::SRL && VT == MVT::i32 && ST->hasV6Ops()) { in PerformShiftCombine()
10702 // 16-bits of x is zero. This optimizes rev + lsr 16 to rev16. in PerformShiftCombine()
10703 SDValue N1 = N->getOperand(1); in PerformShiftCombine()
10705 SDValue N0 = N->getOperand(0); in PerformShiftCombine()
10706 if (C->getZExtValue() == 16 && N0.getOpcode() == ISD::BSWAP && in PerformShiftCombine()
10718 assert(ST->hasNEON() && "unexpected vector shift"); in PerformShiftCombine()
10721 switch (N->getOpcode()) { in PerformShiftCombine()
10725 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) { in PerformShiftCombine()
10727 return DAG.getNode(ARMISD::VSHL, dl, VT, N->getOperand(0), in PerformShiftCombine()
10734 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) { in PerformShiftCombine()
10735 unsigned VShiftOpc = (N->getOpcode() == ISD::SRA ? in PerformShiftCombine()
10738 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), in PerformShiftCombine()
10745 /// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
10749 SDValue N0 = N->getOperand(0); in PerformExtendCombine()
10751 // Check for sign- and zero-extensions of vector extract operations of 8- in PerformExtendCombine()
10752 // and 16-bit vector elements. NEON supports these directly. They are in PerformExtendCombine()
10754 // to 32-bit types and it is messy to recognize the operations after that. in PerformExtendCombine()
10755 if (ST->hasNEON() && N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in PerformExtendCombine()
10758 EVT VT = N->getValueType(0); in PerformExtendCombine()
10768 switch (N->getOpcode()) { in PerformExtendCombine()
10795 const APInt &Mask = CI->getAPIntValue(); in computeKnownBits()
10827 SDValue Op0 = CMOV->getOperand(0); in PerformCMOVToBFICombine()
10828 SDValue Op1 = CMOV->getOperand(1); in PerformCMOVToBFICombine()
10829 auto CCNode = cast<ConstantSDNode>(CMOV->getOperand(2)); in PerformCMOVToBFICombine()
10830 auto CC = CCNode->getAPIntValue().getLimitedValue(); in PerformCMOVToBFICombine()
10831 SDValue CmpZ = CMOV->getOperand(4); in PerformCMOVToBFICombine()
10834 if (!isNullConstant(CmpZ->getOperand(1))) in PerformCMOVToBFICombine()
10837 assert(CmpZ->getOpcode() == ARMISD::CMPZ); in PerformCMOVToBFICombine()
10838 SDValue And = CmpZ->getOperand(0); in PerformCMOVToBFICombine()
10839 if (And->getOpcode() != ISD::AND) in PerformCMOVToBFICombine()
10841 ConstantSDNode *AndC = dyn_cast<ConstantSDNode>(And->getOperand(1)); in PerformCMOVToBFICombine()
10842 if (!AndC || !AndC->getAPIntValue().isPowerOf2()) in PerformCMOVToBFICombine()
10844 SDValue X = And->getOperand(0); in PerformCMOVToBFICombine()
10854 if (Op1->getOpcode() != ISD::OR) in PerformCMOVToBFICombine()
10857 ConstantSDNode *OrC = dyn_cast<ConstantSDNode>(Op1->getOperand(1)); in PerformCMOVToBFICombine()
10860 SDValue Y = Op1->getOperand(0); in PerformCMOVToBFICombine()
10866 APInt OrCI = OrC->getAPIntValue(); in PerformCMOVToBFICombine()
10867 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2; in PerformCMOVToBFICombine()
10882 unsigned BitInX = AndC->getAPIntValue().logBase2(); in PerformCMOVToBFICombine()
10904 /// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
10907 SDValue Cmp = N->getOperand(4); in PerformBRCONDCombine()
10912 EVT VT = N->getValueType(0); in PerformBRCONDCombine()
10916 SDValue Chain = N->getOperand(0); in PerformBRCONDCombine()
10917 SDValue BB = N->getOperand(1); in PerformBRCONDCombine()
10918 SDValue ARMcc = N->getOperand(2); in PerformBRCONDCombine()
10920 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue(); in PerformBRCONDCombine()
10923 // -> (brcond Chain BB CC CPSR Cmp) in PerformBRCONDCombine()
10924 if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() && in PerformBRCONDCombine()
10925 LHS->getOperand(0)->getOpcode() == ARMISD::CMOV && in PerformBRCONDCombine()
10926 LHS->getOperand(0)->hasOneUse()) { in PerformBRCONDCombine()
10927 auto *LHS00C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)->getOperand(0)); in PerformBRCONDCombine()
10928 auto *LHS01C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)->getOperand(1)); in PerformBRCONDCombine()
10929 auto *LHS1C = dyn_cast<ConstantSDNode>(LHS->getOperand(1)); in PerformBRCONDCombine()
10931 if ((LHS00C && LHS00C->getZExtValue() == 0) && in PerformBRCONDCombine()
10932 (LHS01C && LHS01C->getZExtValue() == 1) && in PerformBRCONDCombine()
10933 (LHS1C && LHS1C->getZExtValue() == 1) && in PerformBRCONDCombine()
10934 (RHSC && RHSC->getZExtValue() == 0)) { in PerformBRCONDCombine()
10936 ARMISD::BRCOND, dl, VT, Chain, BB, LHS->getOperand(0)->getOperand(2), in PerformBRCONDCombine()
10937 LHS->getOperand(0)->getOperand(3), LHS->getOperand(0)->getOperand(4)); in PerformBRCONDCombine()
10944 /// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
10947 SDValue Cmp = N->getOperand(4); in PerformCMOVCombine()
10952 EVT VT = N->getValueType(0); in PerformCMOVCombine()
10956 SDValue FalseVal = N->getOperand(0); in PerformCMOVCombine()
10957 SDValue TrueVal = N->getOperand(1); in PerformCMOVCombine()
10958 SDValue ARMcc = N->getOperand(2); in PerformCMOVCombine()
10960 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue(); in PerformCMOVCombine()
10963 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) { in PerformCMOVCombine()
10989 N->getOperand(3), Cmp); in PerformCMOVCombine()
10994 N->getOperand(3), NewCmp); in PerformCMOVCombine()
10998 // -> (cmov F T CC CPSR Cmp) in PerformCMOVCombine()
10999 if (CC == ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse()) { in PerformCMOVCombine()
11000 auto *LHS0C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)); in PerformCMOVCombine()
11001 auto *LHS1C = dyn_cast<ConstantSDNode>(LHS->getOperand(1)); in PerformCMOVCombine()
11003 if ((LHS0C && LHS0C->getZExtValue() == 0) && in PerformCMOVCombine()
11004 (LHS1C && LHS1C->getZExtValue() == 1) && in PerformCMOVCombine()
11005 (RHSC && RHSC->getZExtValue() == 0)) { in PerformCMOVCombine()
11007 LHS->getOperand(2), LHS->getOperand(3), in PerformCMOVCombine()
11008 LHS->getOperand(4)); in PerformCMOVCombine()
11032 switch (N->getOpcode()) { in PerformDAGCombine()
11072 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) { in PerformDAGCombine()
11105 bool AllowsUnaligned = Subtarget->allowsUnalignedMem(); in allowsMisalignedMemoryAccesses()
11116 *Fast = Subtarget->hasV7Ops(); in allowsMisalignedMemoryAccesses()
11123 // For any little-endian targets with neon, we can support unaligned ld/st in allowsMisalignedMemoryAccesses()
11125 // A big-endian target may also explicitly support unaligned accesses in allowsMisalignedMemoryAccesses()
11126 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) { in allowsMisalignedMemoryAccesses()
11150 if ((!IsMemset || ZeroMemset) && Subtarget->hasNEON() && in getOptimalMemOpType()
11151 !F->hasFnAttribute(Attribute::NoImplicitFloat)) { in getOptimalMemOpType()
11171 // Let the target-independent logic figure it out. in getOptimalMemOpType()
11189 // 8-bit and 16-bit loads implicitly zero-extend to 32-bits. in isZExtFree()
11206 if (ExtVal->use_empty() || in isVectorLoadExtDesirable()
11207 !ExtVal->use_begin()->isOnlyUserOf(ExtVal.getNode())) in isVectorLoadExtDesirable()
11210 SDNode *U = *ExtVal->use_begin(); in isVectorLoadExtDesirable()
11211 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB || in isVectorLoadExtDesirable()
11212 U->getOpcode() == ISD::SHL || U->getOpcode() == ARMISD::VSHL)) in isVectorLoadExtDesirable()
11219 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) in allowTruncateForTailCall()
11225 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop"); in allowTruncateForTailCall()
11254 if ((V & (Scale - 1)) != 0) in isLegalT1AddressImmediate()
11257 return V == (V & ((1LL << 5) - 1)); in isLegalT1AddressImmediate()
11265 V = - V; in isLegalT2AddressImmediate()
11274 // + imm12 or - imm8 in isLegalT2AddressImmediate()
11276 return V == (V & ((1LL << 8) - 1)); in isLegalT2AddressImmediate()
11277 return V == (V & ((1LL << 12) - 1)); in isLegalT2AddressImmediate()
11281 if (!Subtarget->hasVFP2()) in isLegalT2AddressImmediate()
11286 return V == (V & ((1LL << 8) - 1)); in isLegalT2AddressImmediate()
11290 /// isLegalAddressImmediate - Return true if the integer value can be used
11301 if (Subtarget->isThumb1Only()) in isLegalAddressImmediate()
11303 else if (Subtarget->isThumb2()) in isLegalAddressImmediate()
11308 V = - V; in isLegalAddressImmediate()
11314 // +- imm12 in isLegalAddressImmediate()
11315 return V == (V & ((1LL << 12) - 1)); in isLegalAddressImmediate()
11317 // +- imm8 in isLegalAddressImmediate()
11318 return V == (V & ((1LL << 8) - 1)); in isLegalAddressImmediate()
11321 if (!Subtarget->hasVFP2()) // FIXME: NEON? in isLegalAddressImmediate()
11326 return V == (V & ((1LL << 8) - 1)); in isLegalAddressImmediate()
11363 /// isLegalAddressingMode - Return true if the addressing mode represented
11380 if (Subtarget->isThumb1Only()) in isLegalAddressingMode()
11391 if (Subtarget->isThumb2()) in isLegalAddressingMode()
11400 if (Scale < 0) Scale = -Scale; in isLegalAddressingMode()
11425 /// isLegalICmpImmediate - Return true if the specified immediate is legal
11431 if (!Subtarget->isThumb()) in isLegalICmpImmediate()
11432 return ARM_AM::getSOImmVal(std::abs(Imm)) != -1; in isLegalICmpImmediate()
11433 if (Subtarget->isThumb2()) in isLegalICmpImmediate()
11434 return ARM_AM::getT2SOImmVal(std::abs(Imm)) != -1; in isLegalICmpImmediate()
11435 // Thumb1 doesn't have cmn, and only 8-bit immediates. in isLegalICmpImmediate()
11439 /// isLegalAddImmediate - Return true if the specified immediate is a legal add
11446 if (!Subtarget->isThumb()) in isLegalAddImmediate()
11447 return ARM_AM::getSOImmVal(AbsImm) != -1; in isLegalAddImmediate()
11448 if (Subtarget->isThumb2()) in isLegalAddImmediate()
11449 return ARM_AM::getT2SOImmVal(AbsImm) != -1; in isLegalAddImmediate()
11450 // Thumb1 only has 8-bit unsigned immediate. in isLegalAddImmediate()
11458 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB) in getARMIndexedAddressParts()
11463 Base = Ptr->getOperand(0); in getARMIndexedAddressParts()
11464 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) { in getARMIndexedAddressParts()
11465 int RHSC = (int)RHS->getZExtValue(); in getARMIndexedAddressParts()
11466 if (RHSC < 0 && RHSC > -256) { in getARMIndexedAddressParts()
11467 assert(Ptr->getOpcode() == ISD::ADD); in getARMIndexedAddressParts()
11469 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0)); in getARMIndexedAddressParts()
11473 isInc = (Ptr->getOpcode() == ISD::ADD); in getARMIndexedAddressParts()
11474 Offset = Ptr->getOperand(1); in getARMIndexedAddressParts()
11478 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) { in getARMIndexedAddressParts()
11479 int RHSC = (int)RHS->getZExtValue(); in getARMIndexedAddressParts()
11480 if (RHSC < 0 && RHSC > -0x1000) { in getARMIndexedAddressParts()
11481 assert(Ptr->getOpcode() == ISD::ADD); in getARMIndexedAddressParts()
11483 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0)); in getARMIndexedAddressParts()
11484 Base = Ptr->getOperand(0); in getARMIndexedAddressParts()
11489 if (Ptr->getOpcode() == ISD::ADD) { in getARMIndexedAddressParts()
11492 ARM_AM::getShiftOpcForNode(Ptr->getOperand(0).getOpcode()); in getARMIndexedAddressParts()
11494 Base = Ptr->getOperand(1); in getARMIndexedAddressParts()
11495 Offset = Ptr->getOperand(0); in getARMIndexedAddressParts()
11497 Base = Ptr->getOperand(0); in getARMIndexedAddressParts()
11498 Offset = Ptr->getOperand(1); in getARMIndexedAddressParts()
11503 isInc = (Ptr->getOpcode() == ISD::ADD); in getARMIndexedAddressParts()
11504 Base = Ptr->getOperand(0); in getARMIndexedAddressParts()
11505 Offset = Ptr->getOperand(1); in getARMIndexedAddressParts()
11517 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB) in getT2IndexedAddressParts()
11520 Base = Ptr->getOperand(0); in getT2IndexedAddressParts()
11521 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) { in getT2IndexedAddressParts()
11522 int RHSC = (int)RHS->getZExtValue(); in getT2IndexedAddressParts()
11523 if (RHSC < 0 && RHSC > -0x100) { // 8 bits. in getT2IndexedAddressParts()
11524 assert(Ptr->getOpcode() == ISD::ADD); in getT2IndexedAddressParts()
11526 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0)); in getT2IndexedAddressParts()
11529 isInc = Ptr->getOpcode() == ISD::ADD; in getT2IndexedAddressParts()
11530 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0)); in getT2IndexedAddressParts()
11538 /// getPreIndexedAddressParts - returns true by value, base pointer and
11540 /// can be legally represented as pre-indexed load / store address.
11546 if (Subtarget->isThumb1Only()) in getPreIndexedAddressParts()
11553 Ptr = LD->getBasePtr(); in getPreIndexedAddressParts()
11554 VT = LD->getMemoryVT(); in getPreIndexedAddressParts()
11555 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD; in getPreIndexedAddressParts()
11557 Ptr = ST->getBasePtr(); in getPreIndexedAddressParts()
11558 VT = ST->getMemoryVT(); in getPreIndexedAddressParts()
11564 if (Subtarget->isThumb2()) in getPreIndexedAddressParts()
11577 /// getPostIndexedAddressParts - returns true by value, base pointer and
11579 /// combined with a load / store to form a post-indexed load / store.
11585 if (Subtarget->isThumb1Only()) in getPostIndexedAddressParts()
11592 VT = LD->getMemoryVT(); in getPostIndexedAddressParts()
11593 Ptr = LD->getBasePtr(); in getPostIndexedAddressParts()
11594 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD; in getPostIndexedAddressParts()
11596 VT = ST->getMemoryVT(); in getPostIndexedAddressParts()
11597 Ptr = ST->getBasePtr(); in getPostIndexedAddressParts()
11603 if (Subtarget->isThumb2()) in getPostIndexedAddressParts()
11613 // Swap base ptr and offset to catch more post-index load / store when in getPostIndexedAddressParts()
11615 if (Ptr == Offset && Op->getOpcode() == ISD::ADD && in getPostIndexedAddressParts()
11616 !Subtarget->isThumb2()) in getPostIndexedAddressParts()
11619 // Post-indexed load / store update the base pointer. in getPostIndexedAddressParts()
11644 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); in computeKnownBitsForTargetNode()
11658 ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1)); in computeKnownBitsForTargetNode()
11659 Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue()); in computeKnownBitsForTargetNode()
11664 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT(); in computeKnownBitsForTargetNode()
11666 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); in computeKnownBitsForTargetNode()
11674 //===----------------------------------------------------------------------===//
11676 //===----------------------------------------------------------------------===//
11680 if (!Subtarget->hasV6Ops()) in ExpandInlineAsm()
11683 InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue()); in ExpandInlineAsm()
11684 std::string AsmStr = IA->getAsmString(); in ExpandInlineAsm()
11698 IA->getConstraintString().compare(0, 4, "=l,l") == 0) { in ExpandInlineAsm()
11699 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType()); in ExpandInlineAsm()
11700 if (Ty && Ty->getBitWidth() == 32) in ExpandInlineAsm()
11717 if (!Subtarget->hasVFP2()) in LowerXConstraint()
11721 if (ConstraintVT.isVector() && Subtarget->hasNEON() && in LowerXConstraint()
11729 /// getConstraintType - Given a constraint letter, return the type of
11768 Type *type = CallOperandVal->getType(); in getSingleConstraintMatchWeight()
11775 if (type->isIntegerTy()) { in getSingleConstraintMatchWeight()
11776 if (Subtarget->isThumb()) in getSingleConstraintMatchWeight()
11783 if (type->isFloatingPointTy()) in getSingleConstraintMatchWeight()
11797 if (Subtarget->isThumb()) in getRegForInlineAsmConstraint()
11801 if (Subtarget->isThumb()) in getRegForInlineAsmConstraint()
11805 if (Subtarget->isThumb1Only()) in getRegForInlineAsmConstraint()
11840 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
11861 int64_t CVal64 = C->getSExtValue(); in LowerAsmOperandForConstraint()
11872 if (Subtarget->hasV6T2Ops()) in LowerAsmOperandForConstraint()
11877 if (Subtarget->isThumb1Only()) { in LowerAsmOperandForConstraint()
11882 } else if (Subtarget->isThumb2()) { in LowerAsmOperandForConstraint()
11884 // data-processing instruction. in LowerAsmOperandForConstraint()
11885 if (ARM_AM::getT2SOImmVal(CVal) != -1) in LowerAsmOperandForConstraint()
11889 // data-processing instruction. in LowerAsmOperandForConstraint()
11890 if (ARM_AM::getSOImmVal(CVal) != -1) in LowerAsmOperandForConstraint()
11896 if (Subtarget->isThumb1Only()) { in LowerAsmOperandForConstraint()
11897 // This must be a constant between -255 and -1, for negated ADD in LowerAsmOperandForConstraint()
11901 if (CVal >= -255 && CVal <= -1) in LowerAsmOperandForConstraint()
11904 // This must be a constant between -4095 and 4095. It is not clear in LowerAsmOperandForConstraint()
11907 if (CVal >= -4095 && CVal <= 4095) in LowerAsmOperandForConstraint()
11913 if (Subtarget->isThumb1Only()) { in LowerAsmOperandForConstraint()
11914 // A 32-bit value where only one byte has a nonzero value. Exclude in LowerAsmOperandForConstraint()
11920 } else if (Subtarget->isThumb2()) { in LowerAsmOperandForConstraint()
11922 // value in a data-processing instruction. This can be used in GCC in LowerAsmOperandForConstraint()
11926 if (ARM_AM::getT2SOImmVal(~CVal) != -1) in LowerAsmOperandForConstraint()
11930 // value in a data-processing instruction. This can be used in GCC in LowerAsmOperandForConstraint()
11934 if (ARM_AM::getSOImmVal(~CVal) != -1) in LowerAsmOperandForConstraint()
11940 if (Subtarget->isThumb1Only()) { in LowerAsmOperandForConstraint()
11941 // This must be a constant between -7 and 7, in LowerAsmOperandForConstraint()
11942 // for 3-operand ADD/SUB immediate instructions. in LowerAsmOperandForConstraint()
11943 if (CVal >= -7 && CVal < 7) in LowerAsmOperandForConstraint()
11945 } else if (Subtarget->isThumb2()) { in LowerAsmOperandForConstraint()
11947 // data-processing instruction. This can be used in GCC with an "n" in LowerAsmOperandForConstraint()
11951 if (ARM_AM::getT2SOImmVal(-CVal) != -1) in LowerAsmOperandForConstraint()
11955 // data-processing instruction. This can be used in GCC with an "n" in LowerAsmOperandForConstraint()
11959 if (ARM_AM::getSOImmVal(-CVal) != -1) in LowerAsmOperandForConstraint()
11965 if (Subtarget->isThumb1Only()) { in LowerAsmOperandForConstraint()
11974 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0)) in LowerAsmOperandForConstraint()
11980 if (Subtarget->isThumb()) { // FIXME thumb2 in LowerAsmOperandForConstraint()
11988 if (Subtarget->isThumb()) { // FIXME thumb2 in LowerAsmOperandForConstraint()
11989 // This must be a multiple of 4 between -508 and 508, for in LowerAsmOperandForConstraint()
11991 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0)) in LowerAsmOperandForConstraint()
12009 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || in getDivRemLibcall()
12010 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && in getDivRemLibcall()
12012 bool isSigned = N->getOpcode() == ISD::SDIVREM || in getDivRemLibcall()
12013 N->getOpcode() == ISD::SREM; in getDivRemLibcall()
12027 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || in getDivRemArgList()
12028 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && in getDivRemArgList()
12030 bool isSigned = N->getOpcode() == ISD::SDIVREM || in getDivRemArgList()
12031 N->getOpcode() == ISD::SREM; in getDivRemArgList()
12034 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { in getDivRemArgList()
12035 EVT ArgVT = N->getOperand(i).getValueType(); in getDivRemArgList()
12037 Entry.Node = N->getOperand(i); in getDivRemArgList()
12047 assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() || in LowerDivRem()
12048 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI()) && in LowerDivRem()
12049 "Register-based DivRem lowering only"); in LowerDivRem()
12050 unsigned Opcode = Op->getOpcode(); in LowerDivRem()
12054 EVT VT = Op->getValueType(0); in LowerDivRem()
12086 switch (N->getValueType(0).getSimpleVT().SimpleTy) { in LowerREM()
12099 RTLIB::Libcall LC = getDivRemLibcall(N, N->getValueType(0).getSimpleVT(). in LowerREM()
12103 bool isSigned = N->getOpcode() == ISD::SREM; in LowerREM()
12116 assert(ResNode->getNumOperands() == 2 && "divmod should return two operands"); in LowerREM()
12117 return ResNode->getOperand(1); in LowerREM()
12122 assert(Subtarget->isTargetWindows() && "unsupported target platform"); in LowerDYNAMIC_STACKALLOC()
12147 assert(Op.getValueType() == MVT::f64 && Subtarget->isFPOnlySP() && in LowerFP_EXTEND()
12148 "Unexpected type for custom-lowering FP_EXTEND"); in LowerFP_EXTEND()
12160 Subtarget->isFPOnlySP() && in LowerFP_ROUND()
12161 "Unexpected type for custom-lowering FP_ROUND"); in LowerFP_ROUND()
12186 /// isFPImmLegal - Returns true if the target can instruction select the
12190 if (!Subtarget->hasVFP3()) in isFPImmLegal()
12193 return ARM_AM::getFP32Imm(Imm) != -1; in isFPImmLegal()
12194 if (VT == MVT::f64 && !Subtarget->isFPOnlySP()) in isFPImmLegal()
12195 return ARM_AM::getFP64Imm(Imm) != -1; in isFPImmLegal()
12199 /// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
12215 auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); in getTgtMemIntrinsic()
12217 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); in getTgtMemIntrinsic()
12220 Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1); in getTgtMemIntrinsic()
12221 Info.align = cast<ConstantInt>(AlignArg)->getZExtValue(); in getTgtMemIntrinsic()
12236 auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); in getTgtMemIntrinsic()
12239 Type *ArgTy = I.getArgOperand(ArgI)->getType(); in getTgtMemIntrinsic()
12240 if (!ArgTy->isVectorTy()) in getTgtMemIntrinsic()
12244 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); in getTgtMemIntrinsic()
12247 Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1); in getTgtMemIntrinsic()
12248 Info.align = cast<ConstantInt>(AlignArg)->getZExtValue(); in getTgtMemIntrinsic()
12256 auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); in getTgtMemIntrinsic()
12257 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType()); in getTgtMemIntrinsic()
12259 Info.memVT = MVT::getVT(PtrTy->getElementType()); in getTgtMemIntrinsic()
12262 Info.align = DL.getABITypeAlignment(PtrTy->getElementType()); in getTgtMemIntrinsic()
12270 auto &DL = I.getCalledFunction()->getParent()->getDataLayout(); in getTgtMemIntrinsic()
12271 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType()); in getTgtMemIntrinsic()
12273 Info.memVT = MVT::getVT(PtrTy->getElementType()); in getTgtMemIntrinsic()
12276 Info.align = DL.getABITypeAlignment(PtrTy->getElementType()); in getTgtMemIntrinsic()
12317 assert(Ty->isIntegerTy()); in shouldConvertConstantLoadToIntImm()
12319 unsigned Bits = Ty->getPrimitiveSizeInBits(); in shouldConvertConstantLoadToIntImm()
12327 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); in makeDMB()
12330 if (!Subtarget->hasDataBarrier()) { in makeDMB()
12332 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get in makeDMB()
12334 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) { in makeDMB()
12347 // Only a full system barrier exists in the M-class architectures. in makeDMB()
12348 Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain; in makeDMB()
12361 llvm_unreachable("Invalid fence: unordered/non-atomic"); in emitLeadingFence()
12371 if (Subtarget->preferISHSTBarriers()) in emitLeadingFence()
12386 llvm_unreachable("Invalid fence: unordered/not-atomic"); in emitTrailingFence()
12398 // Loads and stores less than 64-bits are already atomic; ones above that
12403 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits(); in shouldExpandAtomicStoreInIR()
12404 return (Size == 64) && !Subtarget->isMClass(); in shouldExpandAtomicStoreInIR()
12407 // Loads and stores less than 64-bits are already atomic; ones above that
12413 // sections A8.8.72-74 LDRD)
12416 unsigned Size = LI->getType()->getPrimitiveSizeInBits(); in shouldExpandAtomicLoadInIR()
12417 return ((Size == 64) && !Subtarget->isMClass()) ? AtomicExpansionKind::LLOnly in shouldExpandAtomicLoadInIR()
12422 // and up to 64 bits on the non-M profiles
12425 unsigned Size = AI->getType()->getPrimitiveSizeInBits(); in shouldExpandAtomicRMWInIR()
12426 return (Size <= (Subtarget->isMClass() ? 32U : 64U)) in shouldExpandAtomicRMWInIR()
12433 // At -O0, fast-regalloc cannot cope with the live vregs necessary to in shouldExpandAtomicCmpXchgInIR()
12437 // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead. in shouldExpandAtomicCmpXchgInIR()
12448 return Subtarget->isTargetMachO(); in useLoadStackGuardNode()
12454 if (!Subtarget->hasNEON()) in canCombineStoreAndExtract()
12461 if (VectorTy->isFPOrFPVectorTy()) in canCombineStoreAndExtract()
12469 assert(VectorTy->isVectorTy() && "VectorTy is not a vector type"); in canCombineStoreAndExtract()
12470 unsigned BitWidth = cast<VectorType>(VectorTy)->getBitWidth(); in canCombineStoreAndExtract()
12481 return Subtarget->hasV6T2Ops(); in isCheapToSpeculateCttz()
12485 return Subtarget->hasV6T2Ops(); in isCheapToSpeculateCtlz()
12490 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); in emitLoadLinked()
12491 Type *ValTy = cast<PointerType>(Addr->getType())->getElementType(); in emitLoadLinked()
12494 // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd in emitLoadLinked()
12497 if (ValTy->getPrimitiveSizeInBits() == 64) { in emitLoadLinked()
12502 Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext())); in emitLoadLinked()
12507 if (!Subtarget->isLittle()) in emitLoadLinked()
12515 Type *Tys[] = { Addr->getType() }; in emitLoadLinked()
12521 cast<PointerType>(Addr->getType())->getElementType()); in emitLoadLinked()
12526 if (!Subtarget->hasV7Ops()) in emitAtomicCmpXchgNoStoreLLBalance()
12528 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); in emitAtomicCmpXchgNoStoreLLBalance()
12535 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); in emitStoreConditional()
12541 if (Val->getType()->getPrimitiveSizeInBits() == 64) { in emitStoreConditional()
12545 Type *Int32Ty = Type::getInt32Ty(M->getContext()); in emitStoreConditional()
12549 if (!Subtarget->isLittle()) in emitStoreConditional()
12551 Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext())); in emitStoreConditional()
12556 Type *Tys[] = { Addr->getType() }; in emitStoreConditional()
12561 Val, Strex->getFunctionType()->getParamType(0)), in emitStoreConditional()
12585 VectorType *VecTy = Shuffles[0]->getType(); in lowerInterleavedLoad()
12586 Type *EltTy = VecTy->getVectorElementType(); in lowerInterleavedLoad()
12588 const DataLayout &DL = LI->getModule()->getDataLayout(); in lowerInterleavedLoad()
12594 if (!Subtarget->hasNEON() || (VecSize != 64 && VecSize != 128) || EltIs64Bits) in lowerInterleavedLoad()
12599 if (EltTy->isPointerTy()) in lowerInterleavedLoad()
12601 VectorType::get(DL.getIntPtrType(EltTy), VecTy->getVectorNumElements()); in lowerInterleavedLoad()
12610 Type *Int8Ptr = Builder.getInt8PtrTy(LI->getPointerAddressSpace()); in lowerInterleavedLoad()
12611 Ops.push_back(Builder.CreateBitCast(LI->getPointerOperand(), Int8Ptr)); in lowerInterleavedLoad()
12612 Ops.push_back(Builder.getInt32(LI->getAlignment())); in lowerInterleavedLoad()
12616 Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys); in lowerInterleavedLoad()
12628 if (EltTy->isPointerTy()) in lowerInterleavedLoad()
12629 SubVec = Builder.CreateIntToPtr(SubVec, SV->getType()); in lowerInterleavedLoad()
12631 SV->replaceAllUsesWith(SubVec); in lowerInterleavedLoad()
12639 /// I.e. <Start, Start + 1, ..., Start + NumElts - 1>
12670 VectorType *VecTy = SVI->getType(); in lowerInterleavedStore()
12671 assert(VecTy->getVectorNumElements() % Factor == 0 && in lowerInterleavedStore()
12674 unsigned NumSubElts = VecTy->getVectorNumElements() / Factor; in lowerInterleavedStore()
12675 Type *EltTy = VecTy->getVectorElementType(); in lowerInterleavedStore()
12678 const DataLayout &DL = SI->getModule()->getDataLayout(); in lowerInterleavedStore()
12684 if (!Subtarget->hasNEON() || (SubVecSize != 64 && SubVecSize != 128) || in lowerInterleavedStore()
12688 Value *Op0 = SVI->getOperand(0); in lowerInterleavedStore()
12689 Value *Op1 = SVI->getOperand(1); in lowerInterleavedStore()
12694 if (EltTy->isPointerTy()) { in lowerInterleavedStore()
12699 VectorType::get(IntTy, Op0->getType()->getVectorNumElements()); in lowerInterleavedStore()
12711 Type *Int8Ptr = Builder.getInt8PtrTy(SI->getPointerAddressSpace()); in lowerInterleavedStore()
12712 Ops.push_back(Builder.CreateBitCast(SI->getPointerOperand(), Int8Ptr)); in lowerInterleavedStore()
12716 SI->getModule(), StoreInts[Factor - 2], Tys); in lowerInterleavedStore()
12723 Ops.push_back(Builder.getInt32(SI->getAlignment())); in lowerInterleavedStore()
12739 for (unsigned i = 0; i < ST->getNumElements(); ++i) { in isHomogeneousAggregate()
12741 if (!isHomogeneousAggregate(ST->getElementType(i), Base, SubMembers)) in isHomogeneousAggregate()
12747 if (!isHomogeneousAggregate(AT->getElementType(), Base, SubMembers)) in isHomogeneousAggregate()
12749 Members += SubMembers * AT->getNumElements(); in isHomogeneousAggregate()
12750 } else if (Ty->isFloatTy()) { in isHomogeneousAggregate()
12755 } else if (Ty->isDoubleTy()) { in isHomogeneousAggregate()
12767 return VT->getBitWidth() == 64; in isHomogeneousAggregate()
12769 return VT->getBitWidth() == 128; in isHomogeneousAggregate()
12771 switch (VT->getBitWidth()) { in isHomogeneousAggregate()
12787 /// \brief Return true if a type is an AAPCS-VFP homogeneous aggregate or one of
12788 /// [N x i32] or [N x i64]. This allows front-ends to skip emitting padding when
12799 DEBUG(dbgs() << "isHA: " << IsHA << " "; Ty->dump()); in functionArgumentNeedsConsecutiveRegisters()
12801 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy(); in functionArgumentNeedsConsecutiveRegisters()
12809 return Subtarget->useSjLjEH() ? ARM::NoRegister : ARM::R0; in getExceptionPointerRegister()
12816 return Subtarget->useSjLjEH() ? ARM::NoRegister : ARM::R1; in getExceptionSelectorRegister()
12821 ARMFunctionInfo *AFI = Entry->getParent()->getInfo<ARMFunctionInfo>(); in initializeSplitCSR()
12822 AFI->setIsSplitCSR(true); in initializeSplitCSR()
12828 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo(); in insertCopiesSplitCSR()
12829 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent()); in insertCopiesSplitCSR()
12833 const TargetInstrInfo *TII = Subtarget->getInstrInfo(); in insertCopiesSplitCSR()
12834 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo(); in insertCopiesSplitCSR()
12835 MachineBasicBlock::iterator MBBI = Entry->begin(); in insertCopiesSplitCSR()
12845 unsigned NewVR = MRI->createVirtualRegister(RC); in insertCopiesSplitCSR()
12847 // FIXME: this currently does not emit CFI pseudo-instructions, it works in insertCopiesSplitCSR()
12848 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be in insertCopiesSplitCSR()
12850 // CFI pseudo-instructions. in insertCopiesSplitCSR()
12851 assert(Entry->getParent()->getFunction()->hasFnAttribute( in insertCopiesSplitCSR()
12854 Entry->addLiveIn(*I); in insertCopiesSplitCSR()
12855 BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR) in insertCopiesSplitCSR()
12858 // Insert the copy-back instructions right before the terminator. in insertCopiesSplitCSR()
12860 BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(), in insertCopiesSplitCSR()
12861 TII->get(TargetOpcode::COPY), *I) in insertCopiesSplitCSR()