Lines Matching +full:get +full:- +full:intrinsic
1 //===- InstCombineCalls.cpp -----------------------------------------------===//
8 //===----------------------------------------------------------------------===//
12 //===----------------------------------------------------------------------===//
37 if (ITy->getBitWidth() < 32) in getPromotedType()
38 return Type::getInt32Ty(Ty->getContext()); in getPromotedType()
46 while (!T->isSingleValueType()) { in reduceToSingleValueType()
48 if (STy->getNumElements() == 1) in reduceToSingleValueType()
49 T = STy->getElementType(0); in reduceToSingleValueType()
53 if (ATy->getNumElements() == 1) in reduceToSingleValueType()
54 T = ATy->getElementType(); in reduceToSingleValueType()
68 IntegerType *BoolTy = Type::getInt1Ty(V->getContext()); in getNegativeIsTrueBoolVec()
69 for (unsigned I = 0, E = V->getNumElements(); I != E; ++I) { in getNegativeIsTrueBoolVec()
70 Constant *Elt = V->getElementAsConstant(I); in getNegativeIsTrueBoolVec()
73 bool Sign = V->getElementType()->isIntegerTy() in getNegativeIsTrueBoolVec()
74 ? cast<ConstantInt>(Elt)->isNegative() in getNegativeIsTrueBoolVec()
75 : cast<ConstantFP>(Elt)->isNegative(); in getNegativeIsTrueBoolVec()
76 BoolVec.push_back(ConstantInt::get(BoolTy, Sign)); in getNegativeIsTrueBoolVec()
78 return ConstantVector::get(BoolVec); in getNegativeIsTrueBoolVec()
82 unsigned DstAlign = getKnownAlignment(MI->getArgOperand(0), DL, MI, AC, DT); in SimplifyMemTransfer()
83 unsigned SrcAlign = getKnownAlignment(MI->getArgOperand(1), DL, MI, AC, DT); in SimplifyMemTransfer()
85 unsigned CopyAlign = MI->getAlignment(); in SimplifyMemTransfer()
88 MI->setAlignment(ConstantInt::get(MI->getAlignmentType(), MinAlign, false)); in SimplifyMemTransfer()
94 ConstantInt *MemOpLength = dyn_cast<ConstantInt>(MI->getArgOperand(2)); in SimplifyMemTransfer()
97 // Source and destination pointer types are always "i8*" for intrinsic. See in SimplifyMemTransfer()
101 uint64_t Size = MemOpLength->getLimitedValue(); in SimplifyMemTransfer()
102 assert(Size && "0-sized memory transferring should be removed already."); in SimplifyMemTransfer()
104 if (Size > 8 || (Size&(Size-1))) in SimplifyMemTransfer()
109 cast<PointerType>(MI->getArgOperand(1)->getType())->getAddressSpace(); in SimplifyMemTransfer()
111 cast<PointerType>(MI->getArgOperand(0)->getType())->getAddressSpace(); in SimplifyMemTransfer()
113 IntegerType* IntType = IntegerType::get(MI->getContext(), Size<<3); in SimplifyMemTransfer()
114 Type *NewSrcPtrTy = PointerType::get(IntType, SrcAddrSp); in SimplifyMemTransfer()
115 Type *NewDstPtrTy = PointerType::get(IntType, DstAddrSp); in SimplifyMemTransfer()
118 // that if you're using memcpy to move one double around, you'll get a cast in SimplifyMemTransfer()
123 Value *StrippedDest = MI->getArgOperand(0)->stripPointerCasts(); in SimplifyMemTransfer()
125 if (StrippedDest != MI->getArgOperand(0)) { in SimplifyMemTransfer()
126 Type *SrcETy = cast<PointerType>(StrippedDest->getType()) in SimplifyMemTransfer()
127 ->getElementType(); in SimplifyMemTransfer()
128 if (SrcETy->isSized() && DL.getTypeStoreSize(SrcETy) == Size) { in SimplifyMemTransfer()
133 if (SrcETy->isSingleValueType()) { in SimplifyMemTransfer()
134 NewSrcPtrTy = PointerType::get(SrcETy, SrcAddrSp); in SimplifyMemTransfer()
135 NewDstPtrTy = PointerType::get(SrcETy, DstAddrSp); in SimplifyMemTransfer()
138 // get the TBAA tag describing our copy. in SimplifyMemTransfer()
139 if (MDNode *M = MI->getMetadata(LLVMContext::MD_tbaa_struct)) { in SimplifyMemTransfer()
140 if (M->getNumOperands() == 3 && M->getOperand(0) && in SimplifyMemTransfer()
141 mdconst::hasa<ConstantInt>(M->getOperand(0)) && in SimplifyMemTransfer()
142 mdconst::extract<ConstantInt>(M->getOperand(0))->isNullValue() && in SimplifyMemTransfer()
143 M->getOperand(1) && in SimplifyMemTransfer()
144 mdconst::hasa<ConstantInt>(M->getOperand(1)) && in SimplifyMemTransfer()
145 mdconst::extract<ConstantInt>(M->getOperand(1))->getValue() == in SimplifyMemTransfer()
147 M->getOperand(2) && isa<MDNode>(M->getOperand(2))) in SimplifyMemTransfer()
148 CopyMD = cast<MDNode>(M->getOperand(2)); in SimplifyMemTransfer()
159 Value *Src = Builder->CreateBitCast(MI->getArgOperand(1), NewSrcPtrTy); in SimplifyMemTransfer()
160 Value *Dest = Builder->CreateBitCast(MI->getArgOperand(0), NewDstPtrTy); in SimplifyMemTransfer()
161 LoadInst *L = Builder->CreateLoad(Src, MI->isVolatile()); in SimplifyMemTransfer()
162 L->setAlignment(SrcAlign); in SimplifyMemTransfer()
164 L->setMetadata(LLVMContext::MD_tbaa, CopyMD); in SimplifyMemTransfer()
165 StoreInst *S = Builder->CreateStore(L, Dest, MI->isVolatile()); in SimplifyMemTransfer()
166 S->setAlignment(DstAlign); in SimplifyMemTransfer()
168 S->setMetadata(LLVMContext::MD_tbaa, CopyMD); in SimplifyMemTransfer()
171 MI->setArgOperand(2, Constant::getNullValue(MemOpLength->getType())); in SimplifyMemTransfer()
176 unsigned Alignment = getKnownAlignment(MI->getDest(), DL, MI, AC, DT); in SimplifyMemSet()
177 if (MI->getAlignment() < Alignment) { in SimplifyMemSet()
178 MI->setAlignment(ConstantInt::get(MI->getAlignmentType(), in SimplifyMemSet()
184 ConstantInt *LenC = dyn_cast<ConstantInt>(MI->getLength()); in SimplifyMemSet()
185 ConstantInt *FillC = dyn_cast<ConstantInt>(MI->getValue()); in SimplifyMemSet()
186 if (!LenC || !FillC || !FillC->getType()->isIntegerTy(8)) in SimplifyMemSet()
188 uint64_t Len = LenC->getLimitedValue(); in SimplifyMemSet()
189 Alignment = MI->getAlignment(); in SimplifyMemSet()
190 assert(Len && "0-sized memory setting should be removed already."); in SimplifyMemSet()
192 // memset(s,c,n) -> store s, c (for n=1,2,4,8) in SimplifyMemSet()
194 Type *ITy = IntegerType::get(MI->getContext(), Len*8); // n=1 -> i8. in SimplifyMemSet()
196 Value *Dest = MI->getDest(); in SimplifyMemSet()
197 unsigned DstAddrSp = cast<PointerType>(Dest->getType())->getAddressSpace(); in SimplifyMemSet()
198 Type *NewDstPtrTy = PointerType::get(ITy, DstAddrSp); in SimplifyMemSet()
199 Dest = Builder->CreateBitCast(Dest, NewDstPtrTy); in SimplifyMemSet()
205 uint64_t Fill = FillC->getZExtValue()*0x0101010101010101ULL; in SimplifyMemSet()
206 StoreInst *S = Builder->CreateStore(ConstantInt::get(ITy, Fill), Dest, in SimplifyMemSet()
207 MI->isVolatile()); in SimplifyMemSet()
208 S->setAlignment(Alignment); in SimplifyMemSet()
211 MI->setLength(Constant::getNullValue(LenC->getType())); in SimplifyMemSet()
226 case Intrinsic::x86_sse2_psra_d: in simplifyX86immShift()
227 case Intrinsic::x86_sse2_psra_w: in simplifyX86immShift()
228 case Intrinsic::x86_sse2_psrai_d: in simplifyX86immShift()
229 case Intrinsic::x86_sse2_psrai_w: in simplifyX86immShift()
230 case Intrinsic::x86_avx2_psra_d: in simplifyX86immShift()
231 case Intrinsic::x86_avx2_psra_w: in simplifyX86immShift()
232 case Intrinsic::x86_avx2_psrai_d: in simplifyX86immShift()
233 case Intrinsic::x86_avx2_psrai_w: in simplifyX86immShift()
236 case Intrinsic::x86_sse2_psrl_d: in simplifyX86immShift()
237 case Intrinsic::x86_sse2_psrl_q: in simplifyX86immShift()
238 case Intrinsic::x86_sse2_psrl_w: in simplifyX86immShift()
239 case Intrinsic::x86_sse2_psrli_d: in simplifyX86immShift()
240 case Intrinsic::x86_sse2_psrli_q: in simplifyX86immShift()
241 case Intrinsic::x86_sse2_psrli_w: in simplifyX86immShift()
242 case Intrinsic::x86_avx2_psrl_d: in simplifyX86immShift()
243 case Intrinsic::x86_avx2_psrl_q: in simplifyX86immShift()
244 case Intrinsic::x86_avx2_psrl_w: in simplifyX86immShift()
245 case Intrinsic::x86_avx2_psrli_d: in simplifyX86immShift()
246 case Intrinsic::x86_avx2_psrli_q: in simplifyX86immShift()
247 case Intrinsic::x86_avx2_psrli_w: in simplifyX86immShift()
250 case Intrinsic::x86_sse2_psll_d: in simplifyX86immShift()
251 case Intrinsic::x86_sse2_psll_q: in simplifyX86immShift()
252 case Intrinsic::x86_sse2_psll_w: in simplifyX86immShift()
253 case Intrinsic::x86_sse2_pslli_d: in simplifyX86immShift()
254 case Intrinsic::x86_sse2_pslli_q: in simplifyX86immShift()
255 case Intrinsic::x86_sse2_pslli_w: in simplifyX86immShift()
256 case Intrinsic::x86_avx2_psll_d: in simplifyX86immShift()
257 case Intrinsic::x86_avx2_psll_q: in simplifyX86immShift()
258 case Intrinsic::x86_avx2_psll_w: in simplifyX86immShift()
259 case Intrinsic::x86_avx2_pslli_d: in simplifyX86immShift()
260 case Intrinsic::x86_avx2_pslli_q: in simplifyX86immShift()
261 case Intrinsic::x86_avx2_pslli_w: in simplifyX86immShift()
277 // SSE2/AVX2 uses all the first 64-bits of the 128-bit vector in simplifyX86immShift()
279 auto VT = cast<VectorType>(CDV->getType()); in simplifyX86immShift()
280 unsigned BitWidth = VT->getElementType()->getPrimitiveSizeInBits(); in simplifyX86immShift()
284 // Concatenate the sub-elements to create the 64-bit value. in simplifyX86immShift()
286 unsigned SubEltIdx = (NumSubElts - 1) - i; in simplifyX86immShift()
287 auto SubElt = cast<ConstantInt>(CDV->getElementAsConstant(SubEltIdx)); in simplifyX86immShift()
289 Count |= SubElt->getValue().zextOrTrunc(64); in simplifyX86immShift()
293 Count = CInt->getValue(); in simplifyX86immShift()
296 auto VT = cast<VectorType>(Vec->getType()); in simplifyX86immShift()
297 auto SVT = VT->getElementType(); in simplifyX86immShift()
298 unsigned VWidth = VT->getNumElements(); in simplifyX86immShift()
299 unsigned BitWidth = SVT->getPrimitiveSizeInBits(); in simplifyX86immShift()
301 // If shift-by-zero then just return the original value. in simplifyX86immShift()
307 // If LogicalShift - just return zero. in simplifyX86immShift()
309 return ConstantAggregateZero::get(VT); in simplifyX86immShift()
311 // If ArithmeticShift - clamp Shift to (BitWidth - 1). in simplifyX86immShift()
312 Count = APInt(64, BitWidth - 1); in simplifyX86immShift()
315 // Get a constant vector of the same type as the first operand. in simplifyX86immShift()
316 auto ShiftAmt = ConstantInt::get(SVT, Count.zextOrTrunc(BitWidth)); in simplifyX86immShift()
328 // Attempt to simplify AVX2 per-element shift intrinsics to a generic IR shift.
330 // of range shift amounts (logical - set to zero, arithmetic - splat sign bit).
339 case Intrinsic::x86_avx2_psrav_d: in simplifyX86varShift()
340 case Intrinsic::x86_avx2_psrav_d_256: in simplifyX86varShift()
344 case Intrinsic::x86_avx2_psrlv_d: in simplifyX86varShift()
345 case Intrinsic::x86_avx2_psrlv_d_256: in simplifyX86varShift()
346 case Intrinsic::x86_avx2_psrlv_q: in simplifyX86varShift()
347 case Intrinsic::x86_avx2_psrlv_q_256: in simplifyX86varShift()
351 case Intrinsic::x86_avx2_psllv_d: in simplifyX86varShift()
352 case Intrinsic::x86_avx2_psllv_d_256: in simplifyX86varShift()
353 case Intrinsic::x86_avx2_psllv_q: in simplifyX86varShift()
354 case Intrinsic::x86_avx2_psllv_q_256: in simplifyX86varShift()
368 auto SVT = VT->getVectorElementType(); in simplifyX86varShift()
369 int NumElts = VT->getNumElements(); in simplifyX86varShift()
370 int BitWidth = SVT->getIntegerBitWidth(); in simplifyX86varShift()
373 // We also collect special cases: UNDEF = -1, OUT-OF-RANGE = BitWidth. in simplifyX86varShift()
377 auto *CElt = CShift->getAggregateElement(I); in simplifyX86varShift()
379 ShiftAmts.push_back(-1); in simplifyX86varShift()
388 // If LogicalShift - set to BitWidth (special case). in simplifyX86varShift()
389 // If ArithmeticShift - set to (BitWidth - 1) (sign splat). in simplifyX86varShift()
390 APInt ShiftVal = COp->getValue(); in simplifyX86varShift()
393 ShiftAmts.push_back(LogicalShift ? BitWidth : BitWidth - 1); in simplifyX86varShift()
407 ConstantVec.push_back(UndefValue::get(SVT)); in simplifyX86varShift()
413 return ConstantVector::get(ConstantVec); in simplifyX86varShift()
424 ShiftVecAmts.push_back(UndefValue::get(SVT)); in simplifyX86varShift()
426 ShiftVecAmts.push_back(ConstantInt::get(SVT, Idx)); in simplifyX86varShift()
428 auto ShiftVec = ConstantVector::get(ShiftVecAmts); in simplifyX86varShift()
443 Type *ArgTy = Arg->getType(); in simplifyX86movmsk()
445 // movmsk(undef) -> zero as we must ensure the upper bits are zero. in simplifyX86movmsk()
450 if (!ArgTy->isVectorTy()) in simplifyX86movmsk()
458 APInt Result(ResTy->getPrimitiveSizeInBits(), 0); in simplifyX86movmsk()
459 for (unsigned I = 0, E = ArgTy->getVectorNumElements(); I != E; ++I) { in simplifyX86movmsk()
460 auto *COp = C->getAggregateElement(I); in simplifyX86movmsk()
471 if ((CInt && CInt->isNegative()) || (CFp && CFp->isNegative())) in simplifyX86movmsk()
485 assert(VecTy->getNumElements() == 4 && "insertps with wrong vector type"); in simplifyX86insertps()
488 // [3:0] - zero mask for each 32-bit lane in simplifyX86insertps()
489 // [5:4] - select one 32-bit destination lane in simplifyX86insertps()
490 // [7:6] - select one 32-bit source lane in simplifyX86insertps()
492 uint8_t Imm = CInt->getZExtValue(); in simplifyX86insertps()
497 ConstantAggregateZero *ZeroVector = ConstantAggregateZero::get(VecTy); in simplifyX86insertps()
516 // We may still move 32-bits of the first source vector from one lane in simplifyX86insertps()
542 Constant *Args[] = {ConstantInt::get(IntTy64, Val), in simplifyX86extrq()
543 UndefValue::get(IntTy64)}; in simplifyX86extrq()
544 return ConstantVector::get(Args); in simplifyX86extrq()
550 C0 ? dyn_cast<ConstantInt>(C0->getAggregateElement((unsigned)0)) in simplifyX86extrq()
557 APInt APIndex = CIIndex->getValue().zextOrTrunc(6); in simplifyX86extrq()
558 APInt APLength = CILength->getValue().zextOrTrunc(6); in simplifyX86extrq()
570 // Note that both field index and field length are 8-bit quantities. in simplifyX86extrq()
572 // obtained from zero-extending field index and field length in simplifyX86extrq()
575 return UndefValue::get(II.getType()); in simplifyX86extrq()
586 VectorType *ShufTy = VectorType::get(IntTy8, 16); in simplifyX86extrq()
596 ShuffleMask.push_back(UndefValue::get(IntTy32)); in simplifyX86extrq()
600 ConstantAggregateZero::get(ShufTy), ConstantVector::get(ShuffleMask)); in simplifyX86extrq()
604 // Constant Fold - shift Index'th bit to lowest position and mask off in simplifyX86extrq()
607 APInt Elt = CI0->getValue(); in simplifyX86extrq()
613 if (II.getIntrinsicID() == Intrinsic::x86_sse4a_extrq) { in simplifyX86extrq()
616 Value *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_extrqi); in simplifyX86extrq()
621 // Constant Fold - extraction from zero is always {zero, undef}. in simplifyX86extrq()
622 if (CI0 && CI0->equalsInt(0)) in simplifyX86extrq()
650 // Note that both field index and field length are 8-bit quantities. in simplifyX86insertq()
652 // obtained from zero-extending field index and field length in simplifyX86insertq()
655 return UndefValue::get(II.getType()); in simplifyX86insertq()
666 VectorType *ShufTy = VectorType::get(IntTy8, 16); in simplifyX86insertq()
677 ShuffleMask.push_back(UndefValue::get(IntTy32)); in simplifyX86insertq()
681 ConstantVector::get(ShuffleMask)); in simplifyX86insertq()
689 C0 ? dyn_cast<ConstantInt>(C0->getAggregateElement((unsigned)0)) in simplifyX86insertq()
692 C1 ? dyn_cast<ConstantInt>(C1->getAggregateElement((unsigned)0)) in simplifyX86insertq()
695 // Constant Fold - insert bottom Length bits starting at the Index'th bit. in simplifyX86insertq()
697 APInt V00 = CI00->getValue(); in simplifyX86insertq()
698 APInt V10 = CI10->getValue(); in simplifyX86insertq()
704 Constant *Args[] = {ConstantInt::get(IntTy64, Val.getZExtValue()), in simplifyX86insertq()
705 UndefValue::get(IntTy64)}; in simplifyX86insertq()
706 return ConstantVector::get(Args); in simplifyX86insertq()
711 if (II.getIntrinsicID() == Intrinsic::x86_sse4a_insertq) { in simplifyX86insertq()
713 Constant *CILength = ConstantInt::get(IntTy8, Length, false); in simplifyX86insertq()
714 Constant *CIIndex = ConstantInt::get(IntTy8, Index, false); in simplifyX86insertq()
718 Value *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_insertqi); in simplifyX86insertq()
734 unsigned NumElts = VecTy->getNumElements(); in simplifyX86pshufb()
744 Constant *COp = V->getAggregateElement(I); in simplifyX86pshufb()
749 Indexes[I] = UndefValue::get(MaskEltTy); in simplifyX86pshufb()
753 int8_t Index = cast<ConstantInt>(COp)->getValue().getZExtValue(); in simplifyX86pshufb()
757 // The zero vector is in the right-hand side of the resulting in simplifyX86pshufb()
760 // The value of each index for the high 128-bit lane is the least in simplifyX86pshufb()
763 Indexes[I] = ConstantInt::get(MaskEltTy, Index); in simplifyX86pshufb()
766 auto ShuffleMask = ConstantVector::get(makeArrayRef(Indexes, NumElts)); in simplifyX86pshufb()
780 unsigned NumElts = cast<VectorType>(V->getType())->getNumElements(); in simplifyX86vpermilvar()
788 Constant *COp = V->getAggregateElement(I); in simplifyX86vpermilvar()
793 Indexes[I] = UndefValue::get(MaskEltTy); in simplifyX86vpermilvar()
797 APInt Index = cast<ConstantInt>(COp)->getValue(); in simplifyX86vpermilvar()
800 // The PD variants uses bit 1 to select per-lane element index, so in simplifyX86vpermilvar()
802 if (II.getIntrinsicID() == Intrinsic::x86_avx_vpermilvar_pd || in simplifyX86vpermilvar()
803 II.getIntrinsicID() == Intrinsic::x86_avx_vpermilvar_pd_256) in simplifyX86vpermilvar()
809 if ((II.getIntrinsicID() == Intrinsic::x86_avx_vpermilvar_ps_256 || in simplifyX86vpermilvar()
810 II.getIntrinsicID() == Intrinsic::x86_avx_vpermilvar_pd_256) && in simplifyX86vpermilvar()
815 Indexes[I] = ConstantInt::get(MaskEltTy, Index); in simplifyX86vpermilvar()
818 auto ShuffleMask = ConstantVector::get(makeArrayRef(Indexes, NumElts)); in simplifyX86vpermilvar()
820 auto V2 = UndefValue::get(V1->getType()); in simplifyX86vpermilvar()
833 unsigned Size = VecTy->getNumElements(); in simplifyX86vpermv()
840 Constant *COp = V->getAggregateElement(I); in simplifyX86vpermv()
845 Indexes[I] = UndefValue::get(MaskEltTy); in simplifyX86vpermv()
849 APInt Index = cast<ConstantInt>(COp)->getValue(); in simplifyX86vpermv()
851 Indexes[I] = ConstantInt::get(MaskEltTy, Index); in simplifyX86vpermv()
854 auto ShuffleMask = ConstantVector::get(makeArrayRef(Indexes, Size)); in simplifyX86vpermv()
856 auto V2 = UndefValue::get(VecTy); in simplifyX86vpermv()
860 /// The shuffle mask for a perm2*128 selects any two halves of two 256-bit
870 ConstantAggregateZero *ZeroVector = ConstantAggregateZero::get(VecTy); in simplifyX86vperm2()
873 // [1:0] - select 128 bits from sources for low half of destination in simplifyX86vperm2()
874 // [2] - ignore in simplifyX86vperm2()
875 // [3] - zero low half of destination in simplifyX86vperm2()
876 // [5:4] - select 128 bits from sources for high half of destination in simplifyX86vperm2()
877 // [6] - ignore in simplifyX86vperm2()
878 // [7] - zero high half of destination in simplifyX86vperm2()
880 uint8_t Imm = CInt->getZExtValue(); in simplifyX86vperm2()
891 unsigned NumElts = VecTy->getNumElements(); in simplifyX86vperm2()
931 uint64_t Imm = CInt->getZExtValue() & 0x7; in simplifyX86vpcom()
955 return ConstantInt::getSigned(VecTy, -1); // TRUE in simplifyX86vpcom()
969 // fmin(x, x) -> x in simplifyMinnumMaxnum()
975 // fmin(x, nan) -> x in simplifyMinnumMaxnum()
976 if (C1 && C1->isNaN()) in simplifyMinnumMaxnum()
982 // fmin(undef, x) -> x in simplifyMinnumMaxnum()
986 // fmin(x, undef) -> x in simplifyMinnumMaxnum()
992 if (II.getIntrinsicID() == Intrinsic::minnum) { in simplifyMinnumMaxnum()
993 // fmin(x, fmin(x, y)) -> fmin(x, y) in simplifyMinnumMaxnum()
994 // fmin(y, fmin(x, y)) -> fmin(x, y) in simplifyMinnumMaxnum()
1000 // fmin(fmin(x, y), x) -> fmin(x, y) in simplifyMinnumMaxnum()
1001 // fmin(fmin(x, y), y) -> fmin(x, y) in simplifyMinnumMaxnum()
1007 // TODO: fmin(nnan x, inf) -> x in simplifyMinnumMaxnum()
1008 // TODO: fmin(nnan ninf x, flt_max) -> x in simplifyMinnumMaxnum()
1009 if (C1 && C1->isInfinity()) { in simplifyMinnumMaxnum()
1010 // fmin(x, -inf) -> -inf in simplifyMinnumMaxnum()
1011 if (C1->isNegative()) in simplifyMinnumMaxnum()
1015 assert(II.getIntrinsicID() == Intrinsic::maxnum); in simplifyMinnumMaxnum()
1016 // fmax(x, fmax(x, y)) -> fmax(x, y) in simplifyMinnumMaxnum()
1017 // fmax(y, fmax(x, y)) -> fmax(x, y) in simplifyMinnumMaxnum()
1023 // fmax(fmax(x, y), x) -> fmax(x, y) in simplifyMinnumMaxnum()
1024 // fmax(fmax(x, y), y) -> fmax(x, y) in simplifyMinnumMaxnum()
1030 // TODO: fmax(nnan x, -inf) -> x in simplifyMinnumMaxnum()
1031 // TODO: fmax(nnan ninf x, -flt_max) -> x in simplifyMinnumMaxnum()
1032 if (C1 && C1->isInfinity()) { in simplifyMinnumMaxnum()
1033 // fmax(x, inf) -> inf in simplifyMinnumMaxnum()
1034 if (!C1->isNegative()) in simplifyMinnumMaxnum()
1045 if (ConstMask->isAllOnesValue() || isa<UndefValue>(ConstMask)) in maskIsAllOneOrUndef()
1047 for (unsigned I = 0, E = ConstMask->getType()->getVectorNumElements(); I != E; in maskIsAllOneOrUndef()
1049 if (auto *MaskElt = ConstMask->getAggregateElement(I)) in maskIsAllOneOrUndef()
1050 if (MaskElt->isAllOnesValue() || isa<UndefValue>(MaskElt)) in maskIsAllOneOrUndef()
1063 unsigned Alignment = cast<ConstantInt>(II.getArgOperand(1))->getZExtValue(); in simplifyMaskedLoad()
1076 if (ConstMask->isNullValue()) in simplifyMaskedStore()
1080 if (ConstMask->isAllOnesValue()) { in simplifyMaskedStore()
1082 unsigned Alignment = cast<ConstantInt>(II.getArgOperand(2))->getZExtValue(); in simplifyMaskedStore()
1092 if (ConstMask && ConstMask->isNullValue()) in simplifyMaskedGather()
1101 if (ConstMask && ConstMask->isNullValue()) in simplifyMaskedScatter()
1109 // to LLVM masked intrinsics and remove the x86 masked intrinsic defs.
1124 // The mask is constant. Convert this x86 intrinsic to the LLVM instrinsic in simplifyX86MaskedLoad()
1125 // to allow target-independent optimizations. in simplifyX86MaskedLoad()
1127 // First, cast the x86 intrinsic scalar pointer to a vector pointer to match in simplifyX86MaskedLoad()
1128 // the LLVM intrinsic definition for the pointer argument. in simplifyX86MaskedLoad()
1129 unsigned AddrSpace = cast<PointerType>(Ptr->getType())->getAddressSpace(); in simplifyX86MaskedLoad()
1130 PointerType *VecPtrTy = PointerType::get(II.getType(), AddrSpace); in simplifyX86MaskedLoad()
1131 Value *PtrCast = IC.Builder->CreateBitCast(Ptr, VecPtrTy, "castvec"); in simplifyX86MaskedLoad()
1137 // The pass-through vector for an x86 masked load is a zero vector. in simplifyX86MaskedLoad()
1139 IC.Builder->CreateMaskedLoad(PtrCast, 1, BoolMask, ZeroVec); in simplifyX86MaskedLoad()
1145 // to LLVM masked intrinsics and remove the x86 masked intrinsic defs.
1158 // The SSE2 version is too weird (eg, unaligned but non-temporal) to do in simplifyX86MaskedStore()
1160 if (II.getIntrinsicID() == Intrinsic::x86_sse2_maskmov_dqu) in simplifyX86MaskedStore()
1167 // The mask is constant. Convert this x86 intrinsic to the LLVM instrinsic in simplifyX86MaskedStore()
1168 // to allow target-independent optimizations. in simplifyX86MaskedStore()
1170 // First, cast the x86 intrinsic scalar pointer to a vector pointer to match in simplifyX86MaskedStore()
1171 // the LLVM intrinsic definition for the pointer argument. in simplifyX86MaskedStore()
1172 unsigned AddrSpace = cast<PointerType>(Ptr->getType())->getAddressSpace(); in simplifyX86MaskedStore()
1173 PointerType *VecPtrTy = PointerType::get(Vec->getType(), AddrSpace); in simplifyX86MaskedStore()
1174 Value *PtrCast = IC.Builder->CreateBitCast(Ptr, VecPtrTy, "castvec"); in simplifyX86MaskedStore()
1180 IC.Builder->CreateMaskedStore(Vec, PtrCast, 1, BoolMask); in simplifyX86MaskedStore()
1199 // Remove trivially empty start/end intrinsic ranges, i.e. a start
1211 "Start intrinsic does not have expected ID"); in removeTriviallyEmptyRange()
1212 BasicBlock::iterator BI(I), BE(I.getParent()->end()); in removeTriviallyEmptyRange()
1215 if (isa<DbgInfoIntrinsic>(E) || E->getIntrinsicID() == StartID) in removeTriviallyEmptyRange()
1217 if (E->getIntrinsicID() == EndID && in removeTriviallyEmptyRange()
1218 haveSameOperands(I, *E, E->getNumArgOperands())) { in removeTriviallyEmptyRange()
1231 removeTriviallyEmptyRange(I, Intrinsic::vastart, Intrinsic::vaend, *this); in visitVAStartInst()
1236 removeTriviallyEmptyRange(I, Intrinsic::vacopy, Intrinsic::vaend, *this); in visitVACopyInst()
1240 /// CallInst simplification. This mostly only handles folding of intrinsic
1254 if (CI.getParent()->getParent()->doesNotThrow() && in visitCallInst()
1269 if (Constant *NumBytes = dyn_cast<Constant>(MI->getLength())) { in visitCallInst()
1270 if (NumBytes->isNullValue()) in visitCallInst()
1274 if (CI->getZExtValue() == 1) { in visitCallInst()
1282 if (MI->isVolatile()) in visitCallInst()
1289 if (GlobalVariable *GVSrc = dyn_cast<GlobalVariable>(MMI->getSource())) in visitCallInst()
1290 if (GVSrc->isConstant()) { in visitCallInst()
1292 Intrinsic::ID MemCpyID = Intrinsic::memcpy; in visitCallInst()
1293 Type *Tys[3] = { CI.getArgOperand(0)->getType(), in visitCallInst()
1294 CI.getArgOperand(1)->getType(), in visitCallInst()
1295 CI.getArgOperand(2)->getType() }; in visitCallInst()
1296 CI.setCalledFunction(Intrinsic::getDeclaration(M, MemCpyID, Tys)); in visitCallInst()
1302 // memmove(x,x,size) -> noop. in visitCallInst()
1303 if (MTI->getSource() == MTI->getDest()) in visitCallInst()
1333 switch (II->getIntrinsicID()) { in visitCallInst()
1335 case Intrinsic::objectsize: { in visitCallInst()
1337 if (getObjectSize(II->getArgOperand(0), Size, DL, TLI)) { in visitCallInst()
1338 APInt APSize(II->getType()->getIntegerBitWidth(), Size); in visitCallInst()
1340 // `II->getType()` in visitCallInst()
1342 return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), APSize)); in visitCallInst()
1346 case Intrinsic::bswap: { in visitCallInst()
1347 Value *IIOperand = II->getArgOperand(0); in visitCallInst()
1350 // bswap(bswap(x)) -> x in visitCallInst()
1354 // bswap(trunc(bswap(x))) -> trunc(lshr(x, c)) in visitCallInst()
1356 unsigned C = X->getType()->getPrimitiveSizeInBits() - in visitCallInst()
1357 IIOperand->getType()->getPrimitiveSizeInBits(); in visitCallInst()
1358 Value *CV = ConstantInt::get(X->getType(), C); in visitCallInst()
1359 Value *V = Builder->CreateLShr(X, CV); in visitCallInst()
1360 return new TruncInst(V, IIOperand->getType()); in visitCallInst()
1365 case Intrinsic::bitreverse: { in visitCallInst()
1366 Value *IIOperand = II->getArgOperand(0); in visitCallInst()
1369 // bitreverse(bitreverse(x)) -> x in visitCallInst()
1370 if (match(IIOperand, m_Intrinsic<Intrinsic::bitreverse>(m_Value(X)))) in visitCallInst()
1375 case Intrinsic::masked_load: in visitCallInst()
1379 case Intrinsic::masked_store: in visitCallInst()
1381 case Intrinsic::masked_gather: in visitCallInst()
1383 case Intrinsic::masked_scatter: in visitCallInst()
1386 case Intrinsic::powi: in visitCallInst()
1387 if (ConstantInt *Power = dyn_cast<ConstantInt>(II->getArgOperand(1))) { in visitCallInst()
1388 // powi(x, 0) -> 1.0 in visitCallInst()
1389 if (Power->isZero()) in visitCallInst()
1390 return replaceInstUsesWith(CI, ConstantFP::get(CI.getType(), 1.0)); in visitCallInst()
1391 // powi(x, 1) -> x in visitCallInst()
1392 if (Power->isOne()) in visitCallInst()
1393 return replaceInstUsesWith(CI, II->getArgOperand(0)); in visitCallInst()
1394 // powi(x, -1) -> 1/x in visitCallInst()
1395 if (Power->isAllOnesValue()) in visitCallInst()
1396 return BinaryOperator::CreateFDiv(ConstantFP::get(CI.getType(), 1.0), in visitCallInst()
1397 II->getArgOperand(0)); in visitCallInst()
1400 case Intrinsic::cttz: { in visitCallInst()
1403 IntegerType *IT = dyn_cast<IntegerType>(II->getArgOperand(0)->getType()); in visitCallInst()
1406 uint32_t BitWidth = IT->getBitWidth(); in visitCallInst()
1409 computeKnownBits(II->getArgOperand(0), KnownZero, KnownOne, 0, II); in visitCallInst()
1413 return replaceInstUsesWith(CI, ConstantInt::get(IT, in visitCallInst()
1418 case Intrinsic::ctlz: { in visitCallInst()
1421 IntegerType *IT = dyn_cast<IntegerType>(II->getArgOperand(0)->getType()); in visitCallInst()
1424 uint32_t BitWidth = IT->getBitWidth(); in visitCallInst()
1427 computeKnownBits(II->getArgOperand(0), KnownZero, KnownOne, 0, II); in visitCallInst()
1431 return replaceInstUsesWith(CI, ConstantInt::get(IT, in visitCallInst()
1437 case Intrinsic::uadd_with_overflow: in visitCallInst()
1438 case Intrinsic::sadd_with_overflow: in visitCallInst()
1439 case Intrinsic::umul_with_overflow: in visitCallInst()
1440 case Intrinsic::smul_with_overflow: in visitCallInst()
1441 if (isa<Constant>(II->getArgOperand(0)) && in visitCallInst()
1442 !isa<Constant>(II->getArgOperand(1))) { in visitCallInst()
1444 Value *LHS = II->getArgOperand(0); in visitCallInst()
1445 II->setArgOperand(0, II->getArgOperand(1)); in visitCallInst()
1446 II->setArgOperand(1, LHS); in visitCallInst()
1451 case Intrinsic::usub_with_overflow: in visitCallInst()
1452 case Intrinsic::ssub_with_overflow: { in visitCallInst()
1454 IntrinsicIDToOverflowCheckFlavor(II->getIntrinsicID()); in visitCallInst()
1459 if (OptimizeOverflowCheck(OCF, II->getArgOperand(0), II->getArgOperand(1), in visitCallInst()
1466 case Intrinsic::minnum: in visitCallInst()
1467 case Intrinsic::maxnum: { in visitCallInst()
1468 Value *Arg0 = II->getArgOperand(0); in visitCallInst()
1469 Value *Arg1 = II->getArgOperand(1); in visitCallInst()
1472 II->setArgOperand(0, Arg1); in visitCallInst()
1473 II->setArgOperand(1, Arg0); in visitCallInst()
1480 case Intrinsic::ppc_altivec_lvx: in visitCallInst()
1481 case Intrinsic::ppc_altivec_lvxl: in visitCallInst()
1482 // Turn PPC lvx -> load if the pointer is known aligned. in visitCallInst()
1483 if (getOrEnforceKnownAlignment(II->getArgOperand(0), 16, DL, II, AC, DT) >= in visitCallInst()
1485 Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0), in visitCallInst()
1486 PointerType::getUnqual(II->getType())); in visitCallInst()
1490 case Intrinsic::ppc_vsx_lxvw4x: in visitCallInst()
1491 case Intrinsic::ppc_vsx_lxvd2x: { in visitCallInst()
1493 Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0), in visitCallInst()
1494 PointerType::getUnqual(II->getType())); in visitCallInst()
1497 case Intrinsic::ppc_altivec_stvx: in visitCallInst()
1498 case Intrinsic::ppc_altivec_stvxl: in visitCallInst()
1499 // Turn stvx -> store if the pointer is known aligned. in visitCallInst()
1500 if (getOrEnforceKnownAlignment(II->getArgOperand(1), 16, DL, II, AC, DT) >= in visitCallInst()
1503 PointerType::getUnqual(II->getArgOperand(0)->getType()); in visitCallInst()
1504 Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy); in visitCallInst()
1505 return new StoreInst(II->getArgOperand(0), Ptr); in visitCallInst()
1508 case Intrinsic::ppc_vsx_stxvw4x: in visitCallInst()
1509 case Intrinsic::ppc_vsx_stxvd2x: { in visitCallInst()
1511 Type *OpPtrTy = PointerType::getUnqual(II->getArgOperand(0)->getType()); in visitCallInst()
1512 Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy); in visitCallInst()
1513 return new StoreInst(II->getArgOperand(0), Ptr, false, 1); in visitCallInst()
1515 case Intrinsic::ppc_qpx_qvlfs: in visitCallInst()
1516 // Turn PPC QPX qvlfs -> load if the pointer is known aligned. in visitCallInst()
1517 if (getOrEnforceKnownAlignment(II->getArgOperand(0), 16, DL, II, AC, DT) >= in visitCallInst()
1519 Type *VTy = VectorType::get(Builder->getFloatTy(), in visitCallInst()
1520 II->getType()->getVectorNumElements()); in visitCallInst()
1521 Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0), in visitCallInst()
1523 Value *Load = Builder->CreateLoad(Ptr); in visitCallInst()
1524 return new FPExtInst(Load, II->getType()); in visitCallInst()
1527 case Intrinsic::ppc_qpx_qvlfd: in visitCallInst()
1528 // Turn PPC QPX qvlfd -> load if the pointer is known aligned. in visitCallInst()
1529 if (getOrEnforceKnownAlignment(II->getArgOperand(0), 32, DL, II, AC, DT) >= in visitCallInst()
1531 Value *Ptr = Builder->CreateBitCast(II->getArgOperand(0), in visitCallInst()
1532 PointerType::getUnqual(II->getType())); in visitCallInst()
1536 case Intrinsic::ppc_qpx_qvstfs: in visitCallInst()
1537 // Turn PPC QPX qvstfs -> store if the pointer is known aligned. in visitCallInst()
1538 if (getOrEnforceKnownAlignment(II->getArgOperand(1), 16, DL, II, AC, DT) >= in visitCallInst()
1540 Type *VTy = VectorType::get(Builder->getFloatTy(), in visitCallInst()
1541 II->getArgOperand(0)->getType()->getVectorNumElements()); in visitCallInst()
1542 Value *TOp = Builder->CreateFPTrunc(II->getArgOperand(0), VTy); in visitCallInst()
1544 Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy); in visitCallInst()
1548 case Intrinsic::ppc_qpx_qvstfd: in visitCallInst()
1549 // Turn PPC QPX qvstfd -> store if the pointer is known aligned. in visitCallInst()
1550 if (getOrEnforceKnownAlignment(II->getArgOperand(1), 32, DL, II, AC, DT) >= in visitCallInst()
1553 PointerType::getUnqual(II->getArgOperand(0)->getType()); in visitCallInst()
1554 Value *Ptr = Builder->CreateBitCast(II->getArgOperand(1), OpPtrTy); in visitCallInst()
1555 return new StoreInst(II->getArgOperand(0), Ptr); in visitCallInst()
1559 case Intrinsic::x86_vcvtph2ps_128: in visitCallInst()
1560 case Intrinsic::x86_vcvtph2ps_256: { in visitCallInst()
1561 auto Arg = II->getArgOperand(0); in visitCallInst()
1562 auto ArgType = cast<VectorType>(Arg->getType()); in visitCallInst()
1563 auto RetType = cast<VectorType>(II->getType()); in visitCallInst()
1564 unsigned ArgWidth = ArgType->getNumElements(); in visitCallInst()
1565 unsigned RetWidth = RetType->getNumElements(); in visitCallInst()
1567 assert(ArgType->isIntOrIntVectorTy() && in visitCallInst()
1568 ArgType->getScalarSizeInBits() == 16 && in visitCallInst()
1569 "CVTPH2PS input type should be 16-bit integer vector"); in visitCallInst()
1570 assert(RetType->getScalarType()->isFloatTy() && in visitCallInst()
1571 "CVTPH2PS output type should be 32-bit float vector"); in visitCallInst()
1575 return replaceInstUsesWith(*II, ConstantAggregateZero::get(RetType)); in visitCallInst()
1583 VectorHalfAsShorts = Builder->CreateShuffleVector( in visitCallInst()
1584 Arg, UndefValue::get(ArgType), SubVecMask); in visitCallInst()
1588 VectorType::get(Type::getHalfTy(II->getContext()), RetWidth); in visitCallInst()
1590 Builder->CreateBitCast(VectorHalfAsShorts, VectorHalfType); in visitCallInst()
1591 auto VectorFloats = Builder->CreateFPExt(VectorHalfs, RetType); in visitCallInst()
1597 II->setArgOperand(0, V); in visitCallInst()
1603 case Intrinsic::x86_sse_cvtss2si: in visitCallInst()
1604 case Intrinsic::x86_sse_cvtss2si64: in visitCallInst()
1605 case Intrinsic::x86_sse_cvttss2si: in visitCallInst()
1606 case Intrinsic::x86_sse_cvttss2si64: in visitCallInst()
1607 case Intrinsic::x86_sse2_cvtsd2si: in visitCallInst()
1608 case Intrinsic::x86_sse2_cvtsd2si64: in visitCallInst()
1609 case Intrinsic::x86_sse2_cvttsd2si: in visitCallInst()
1610 case Intrinsic::x86_sse2_cvttsd2si64: { in visitCallInst()
1613 Value *Arg = II->getArgOperand(0); in visitCallInst()
1614 unsigned VWidth = Arg->getType()->getVectorNumElements(); in visitCallInst()
1616 II->setArgOperand(0, V); in visitCallInst()
1622 case Intrinsic::x86_mmx_pmovmskb: in visitCallInst()
1623 case Intrinsic::x86_sse_movmsk_ps: in visitCallInst()
1624 case Intrinsic::x86_sse2_movmsk_pd: in visitCallInst()
1625 case Intrinsic::x86_sse2_pmovmskb_128: in visitCallInst()
1626 case Intrinsic::x86_avx_movmsk_pd_256: in visitCallInst()
1627 case Intrinsic::x86_avx_movmsk_ps_256: in visitCallInst()
1628 case Intrinsic::x86_avx2_pmovmskb: { in visitCallInst()
1634 case Intrinsic::x86_sse_comieq_ss: in visitCallInst()
1635 case Intrinsic::x86_sse_comige_ss: in visitCallInst()
1636 case Intrinsic::x86_sse_comigt_ss: in visitCallInst()
1637 case Intrinsic::x86_sse_comile_ss: in visitCallInst()
1638 case Intrinsic::x86_sse_comilt_ss: in visitCallInst()
1639 case Intrinsic::x86_sse_comineq_ss: in visitCallInst()
1640 case Intrinsic::x86_sse_ucomieq_ss: in visitCallInst()
1641 case Intrinsic::x86_sse_ucomige_ss: in visitCallInst()
1642 case Intrinsic::x86_sse_ucomigt_ss: in visitCallInst()
1643 case Intrinsic::x86_sse_ucomile_ss: in visitCallInst()
1644 case Intrinsic::x86_sse_ucomilt_ss: in visitCallInst()
1645 case Intrinsic::x86_sse_ucomineq_ss: in visitCallInst()
1646 case Intrinsic::x86_sse2_comieq_sd: in visitCallInst()
1647 case Intrinsic::x86_sse2_comige_sd: in visitCallInst()
1648 case Intrinsic::x86_sse2_comigt_sd: in visitCallInst()
1649 case Intrinsic::x86_sse2_comile_sd: in visitCallInst()
1650 case Intrinsic::x86_sse2_comilt_sd: in visitCallInst()
1651 case Intrinsic::x86_sse2_comineq_sd: in visitCallInst()
1652 case Intrinsic::x86_sse2_ucomieq_sd: in visitCallInst()
1653 case Intrinsic::x86_sse2_ucomige_sd: in visitCallInst()
1654 case Intrinsic::x86_sse2_ucomigt_sd: in visitCallInst()
1655 case Intrinsic::x86_sse2_ucomile_sd: in visitCallInst()
1656 case Intrinsic::x86_sse2_ucomilt_sd: in visitCallInst()
1657 case Intrinsic::x86_sse2_ucomineq_sd: { in visitCallInst()
1661 Value *Arg0 = II->getArgOperand(0); in visitCallInst()
1662 Value *Arg1 = II->getArgOperand(1); in visitCallInst()
1663 unsigned VWidth = Arg0->getType()->getVectorNumElements(); in visitCallInst()
1665 II->setArgOperand(0, V); in visitCallInst()
1669 II->setArgOperand(1, V); in visitCallInst()
1677 case Intrinsic::x86_sse_add_ss: in visitCallInst()
1678 case Intrinsic::x86_sse_sub_ss: in visitCallInst()
1679 case Intrinsic::x86_sse_mul_ss: in visitCallInst()
1680 case Intrinsic::x86_sse_div_ss: in visitCallInst()
1681 case Intrinsic::x86_sse_min_ss: in visitCallInst()
1682 case Intrinsic::x86_sse_max_ss: in visitCallInst()
1683 case Intrinsic::x86_sse_cmp_ss: in visitCallInst()
1684 case Intrinsic::x86_sse2_add_sd: in visitCallInst()
1685 case Intrinsic::x86_sse2_sub_sd: in visitCallInst()
1686 case Intrinsic::x86_sse2_mul_sd: in visitCallInst()
1687 case Intrinsic::x86_sse2_div_sd: in visitCallInst()
1688 case Intrinsic::x86_sse2_min_sd: in visitCallInst()
1689 case Intrinsic::x86_sse2_max_sd: in visitCallInst()
1690 case Intrinsic::x86_sse2_cmp_sd: { in visitCallInst()
1693 Value *Arg1 = II->getArgOperand(1); in visitCallInst()
1694 unsigned VWidth = Arg1->getType()->getVectorNumElements(); in visitCallInst()
1696 II->setArgOperand(1, V); in visitCallInst()
1702 case Intrinsic::x86_sse41_round_ss: in visitCallInst()
1703 case Intrinsic::x86_sse41_round_sd: { in visitCallInst()
1707 Value *Arg0 = II->getArgOperand(0); in visitCallInst()
1708 Value *Arg1 = II->getArgOperand(1); in visitCallInst()
1709 unsigned VWidth = Arg0->getType()->getVectorNumElements(); in visitCallInst()
1710 if (Value *V = SimplifyDemandedVectorEltsHigh(Arg0, VWidth, VWidth - 1)) { in visitCallInst()
1711 II->setArgOperand(0, V); in visitCallInst()
1715 II->setArgOperand(1, V); in visitCallInst()
1726 case Intrinsic::x86_sse2_psrai_d: in visitCallInst()
1727 case Intrinsic::x86_sse2_psrai_w: in visitCallInst()
1728 case Intrinsic::x86_avx2_psrai_d: in visitCallInst()
1729 case Intrinsic::x86_avx2_psrai_w: in visitCallInst()
1730 case Intrinsic::x86_sse2_psrli_d: in visitCallInst()
1731 case Intrinsic::x86_sse2_psrli_q: in visitCallInst()
1732 case Intrinsic::x86_sse2_psrli_w: in visitCallInst()
1733 case Intrinsic::x86_avx2_psrli_d: in visitCallInst()
1734 case Intrinsic::x86_avx2_psrli_q: in visitCallInst()
1735 case Intrinsic::x86_avx2_psrli_w: in visitCallInst()
1736 case Intrinsic::x86_sse2_pslli_d: in visitCallInst()
1737 case Intrinsic::x86_sse2_pslli_q: in visitCallInst()
1738 case Intrinsic::x86_sse2_pslli_w: in visitCallInst()
1739 case Intrinsic::x86_avx2_pslli_d: in visitCallInst()
1740 case Intrinsic::x86_avx2_pslli_q: in visitCallInst()
1741 case Intrinsic::x86_avx2_pslli_w: in visitCallInst()
1746 case Intrinsic::x86_sse2_psra_d: in visitCallInst()
1747 case Intrinsic::x86_sse2_psra_w: in visitCallInst()
1748 case Intrinsic::x86_avx2_psra_d: in visitCallInst()
1749 case Intrinsic::x86_avx2_psra_w: in visitCallInst()
1750 case Intrinsic::x86_sse2_psrl_d: in visitCallInst()
1751 case Intrinsic::x86_sse2_psrl_q: in visitCallInst()
1752 case Intrinsic::x86_sse2_psrl_w: in visitCallInst()
1753 case Intrinsic::x86_avx2_psrl_d: in visitCallInst()
1754 case Intrinsic::x86_avx2_psrl_q: in visitCallInst()
1755 case Intrinsic::x86_avx2_psrl_w: in visitCallInst()
1756 case Intrinsic::x86_sse2_psll_d: in visitCallInst()
1757 case Intrinsic::x86_sse2_psll_q: in visitCallInst()
1758 case Intrinsic::x86_sse2_psll_w: in visitCallInst()
1759 case Intrinsic::x86_avx2_psll_d: in visitCallInst()
1760 case Intrinsic::x86_avx2_psll_q: in visitCallInst()
1761 case Intrinsic::x86_avx2_psll_w: { in visitCallInst()
1765 // SSE2/AVX2 uses only the first 64-bits of the 128-bit vector in visitCallInst()
1767 Value *Arg1 = II->getArgOperand(1); in visitCallInst()
1768 assert(Arg1->getType()->getPrimitiveSizeInBits() == 128 && in visitCallInst()
1770 unsigned VWidth = Arg1->getType()->getVectorNumElements(); in visitCallInst()
1773 II->setArgOperand(1, V); in visitCallInst()
1779 case Intrinsic::x86_avx2_psllv_d: in visitCallInst()
1780 case Intrinsic::x86_avx2_psllv_d_256: in visitCallInst()
1781 case Intrinsic::x86_avx2_psllv_q: in visitCallInst()
1782 case Intrinsic::x86_avx2_psllv_q_256: in visitCallInst()
1783 case Intrinsic::x86_avx2_psrav_d: in visitCallInst()
1784 case Intrinsic::x86_avx2_psrav_d_256: in visitCallInst()
1785 case Intrinsic::x86_avx2_psrlv_d: in visitCallInst()
1786 case Intrinsic::x86_avx2_psrlv_d_256: in visitCallInst()
1787 case Intrinsic::x86_avx2_psrlv_q: in visitCallInst()
1788 case Intrinsic::x86_avx2_psrlv_q_256: in visitCallInst()
1793 case Intrinsic::x86_sse41_insertps: in visitCallInst()
1798 case Intrinsic::x86_sse4a_extrq: { in visitCallInst()
1799 Value *Op0 = II->getArgOperand(0); in visitCallInst()
1800 Value *Op1 = II->getArgOperand(1); in visitCallInst()
1801 unsigned VWidth0 = Op0->getType()->getVectorNumElements(); in visitCallInst()
1802 unsigned VWidth1 = Op1->getType()->getVectorNumElements(); in visitCallInst()
1803 assert(Op0->getType()->getPrimitiveSizeInBits() == 128 && in visitCallInst()
1804 Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth0 == 2 && in visitCallInst()
1810 C1 ? dyn_cast<ConstantInt>(C1->getAggregateElement((unsigned)0)) in visitCallInst()
1813 C1 ? dyn_cast<ConstantInt>(C1->getAggregateElement((unsigned)1)) in visitCallInst()
1820 // EXTRQ only uses the lowest 64-bits of the first 128-bit vector in visitCallInst()
1821 // operands and the lowest 16-bits of the second. in visitCallInst()
1824 II->setArgOperand(0, V); in visitCallInst()
1828 II->setArgOperand(1, V); in visitCallInst()
1836 case Intrinsic::x86_sse4a_extrqi: { in visitCallInst()
1838 // bits of the lower 64-bits. The upper 64-bits are undefined. in visitCallInst()
1839 Value *Op0 = II->getArgOperand(0); in visitCallInst()
1840 unsigned VWidth = Op0->getType()->getVectorNumElements(); in visitCallInst()
1841 assert(Op0->getType()->getPrimitiveSizeInBits() == 128 && VWidth == 2 && in visitCallInst()
1845 ConstantInt *CILength = dyn_cast<ConstantInt>(II->getArgOperand(1)); in visitCallInst()
1846 ConstantInt *CIIndex = dyn_cast<ConstantInt>(II->getArgOperand(2)); in visitCallInst()
1852 // EXTRQI only uses the lowest 64-bits of the first 128-bit vector in visitCallInst()
1855 II->setArgOperand(0, V); in visitCallInst()
1861 case Intrinsic::x86_sse4a_insertq: { in visitCallInst()
1862 Value *Op0 = II->getArgOperand(0); in visitCallInst()
1863 Value *Op1 = II->getArgOperand(1); in visitCallInst()
1864 unsigned VWidth = Op0->getType()->getVectorNumElements(); in visitCallInst()
1865 assert(Op0->getType()->getPrimitiveSizeInBits() == 128 && in visitCallInst()
1866 Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth == 2 && in visitCallInst()
1867 Op1->getType()->getVectorNumElements() == 2 && in visitCallInst()
1873 C1 ? dyn_cast<ConstantInt>(C1->getAggregateElement((unsigned)1)) in visitCallInst()
1878 const APInt &V11 = CI11->getValue(); in visitCallInst()
1885 // INSERTQ only uses the lowest 64-bits of the first 128-bit vector in visitCallInst()
1888 II->setArgOperand(0, V); in visitCallInst()
1894 case Intrinsic::x86_sse4a_insertqi: { in visitCallInst()
1896 // insert over first source starting at Index bit. The upper 64-bits are in visitCallInst()
1898 Value *Op0 = II->getArgOperand(0); in visitCallInst()
1899 Value *Op1 = II->getArgOperand(1); in visitCallInst()
1900 unsigned VWidth0 = Op0->getType()->getVectorNumElements(); in visitCallInst()
1901 unsigned VWidth1 = Op1->getType()->getVectorNumElements(); in visitCallInst()
1902 assert(Op0->getType()->getPrimitiveSizeInBits() == 128 && in visitCallInst()
1903 Op1->getType()->getPrimitiveSizeInBits() == 128 && VWidth0 == 2 && in visitCallInst()
1907 ConstantInt *CILength = dyn_cast<ConstantInt>(II->getArgOperand(2)); in visitCallInst()
1908 ConstantInt *CIIndex = dyn_cast<ConstantInt>(II->getArgOperand(3)); in visitCallInst()
1912 APInt Len = CILength->getValue().zextOrTrunc(6); in visitCallInst()
1913 APInt Idx = CIIndex->getValue().zextOrTrunc(6); in visitCallInst()
1918 // INSERTQI only uses the lowest 64-bits of the first two 128-bit vector in visitCallInst()
1922 II->setArgOperand(0, V); in visitCallInst()
1926 II->setArgOperand(1, V); in visitCallInst()
1934 case Intrinsic::x86_sse41_pblendvb: in visitCallInst()
1935 case Intrinsic::x86_sse41_blendvps: in visitCallInst()
1936 case Intrinsic::x86_sse41_blendvpd: in visitCallInst()
1937 case Intrinsic::x86_avx_blendv_ps_256: in visitCallInst()
1938 case Intrinsic::x86_avx_blendv_pd_256: in visitCallInst()
1939 case Intrinsic::x86_avx2_pblendvb: { in visitCallInst()
1941 // This optimization is convoluted because the intrinsic is defined as in visitCallInst()
1945 Value *Op0 = II->getArgOperand(0); in visitCallInst()
1946 Value *Op1 = II->getArgOperand(1); in visitCallInst()
1947 Value *Mask = II->getArgOperand(2); in visitCallInst()
1949 // fold (blend A, A, Mask) -> A in visitCallInst()
1953 // Zero Mask - select 1st argument. in visitCallInst()
1957 // Constant Mask - select 1st/2nd argument lane based on top bit of mask. in visitCallInst()
1965 case Intrinsic::x86_ssse3_pshuf_b_128: in visitCallInst()
1966 case Intrinsic::x86_avx2_pshuf_b: in visitCallInst()
1971 case Intrinsic::x86_avx_vpermilvar_ps: in visitCallInst()
1972 case Intrinsic::x86_avx_vpermilvar_ps_256: in visitCallInst()
1973 case Intrinsic::x86_avx_vpermilvar_pd: in visitCallInst()
1974 case Intrinsic::x86_avx_vpermilvar_pd_256: in visitCallInst()
1979 case Intrinsic::x86_avx2_permd: in visitCallInst()
1980 case Intrinsic::x86_avx2_permps: in visitCallInst()
1985 case Intrinsic::x86_avx_vperm2f128_pd_256: in visitCallInst()
1986 case Intrinsic::x86_avx_vperm2f128_ps_256: in visitCallInst()
1987 case Intrinsic::x86_avx_vperm2f128_si_256: in visitCallInst()
1988 case Intrinsic::x86_avx2_vperm2i128: in visitCallInst()
1993 case Intrinsic::x86_avx_maskload_ps: in visitCallInst()
1994 case Intrinsic::x86_avx_maskload_pd: in visitCallInst()
1995 case Intrinsic::x86_avx_maskload_ps_256: in visitCallInst()
1996 case Intrinsic::x86_avx_maskload_pd_256: in visitCallInst()
1997 case Intrinsic::x86_avx2_maskload_d: in visitCallInst()
1998 case Intrinsic::x86_avx2_maskload_q: in visitCallInst()
1999 case Intrinsic::x86_avx2_maskload_d_256: in visitCallInst()
2000 case Intrinsic::x86_avx2_maskload_q_256: in visitCallInst()
2005 case Intrinsic::x86_sse2_maskmov_dqu: in visitCallInst()
2006 case Intrinsic::x86_avx_maskstore_ps: in visitCallInst()
2007 case Intrinsic::x86_avx_maskstore_pd: in visitCallInst()
2008 case Intrinsic::x86_avx_maskstore_ps_256: in visitCallInst()
2009 case Intrinsic::x86_avx_maskstore_pd_256: in visitCallInst()
2010 case Intrinsic::x86_avx2_maskstore_d: in visitCallInst()
2011 case Intrinsic::x86_avx2_maskstore_q: in visitCallInst()
2012 case Intrinsic::x86_avx2_maskstore_d_256: in visitCallInst()
2013 case Intrinsic::x86_avx2_maskstore_q_256: in visitCallInst()
2018 case Intrinsic::x86_xop_vpcomb: in visitCallInst()
2019 case Intrinsic::x86_xop_vpcomd: in visitCallInst()
2020 case Intrinsic::x86_xop_vpcomq: in visitCallInst()
2021 case Intrinsic::x86_xop_vpcomw: in visitCallInst()
2026 case Intrinsic::x86_xop_vpcomub: in visitCallInst()
2027 case Intrinsic::x86_xop_vpcomud: in visitCallInst()
2028 case Intrinsic::x86_xop_vpcomuq: in visitCallInst()
2029 case Intrinsic::x86_xop_vpcomuw: in visitCallInst()
2034 case Intrinsic::ppc_altivec_vperm: in visitCallInst()
2035 // Turn vperm(V1,V2,mask) -> shuffle(V1,V2,mask) if mask is a constant. in visitCallInst()
2036 // Note that ppc_altivec_vperm has a big-endian bias, so when creating in visitCallInst()
2041 if (Constant *Mask = dyn_cast<Constant>(II->getArgOperand(2))) { in visitCallInst()
2042 assert(Mask->getType()->getVectorNumElements() == 16 && in visitCallInst()
2043 "Bad type for intrinsic!"); in visitCallInst()
2048 Constant *Elt = Mask->getAggregateElement(i); in visitCallInst()
2057 Value *Op0 = Builder->CreateBitCast(II->getArgOperand(0), in visitCallInst()
2058 Mask->getType()); in visitCallInst()
2059 Value *Op1 = Builder->CreateBitCast(II->getArgOperand(1), in visitCallInst()
2060 Mask->getType()); in visitCallInst()
2061 Value *Result = UndefValue::get(Op0->getType()); in visitCallInst()
2068 if (isa<UndefValue>(Mask->getAggregateElement(i))) in visitCallInst()
2071 cast<ConstantInt>(Mask->getAggregateElement(i))->getZExtValue(); in visitCallInst()
2074 Idx = 31 - Idx; in visitCallInst()
2080 Builder->CreateExtractElement(Idx < 16 ? Op0ToUse : Op1ToUse, in visitCallInst()
2081 Builder->getInt32(Idx&15)); in visitCallInst()
2085 Result = Builder->CreateInsertElement(Result, ExtractedElts[Idx], in visitCallInst()
2086 Builder->getInt32(i)); in visitCallInst()
2093 case Intrinsic::arm_neon_vld1: in visitCallInst()
2094 case Intrinsic::arm_neon_vld2: in visitCallInst()
2095 case Intrinsic::arm_neon_vld3: in visitCallInst()
2096 case Intrinsic::arm_neon_vld4: in visitCallInst()
2097 case Intrinsic::arm_neon_vld2lane: in visitCallInst()
2098 case Intrinsic::arm_neon_vld3lane: in visitCallInst()
2099 case Intrinsic::arm_neon_vld4lane: in visitCallInst()
2100 case Intrinsic::arm_neon_vst1: in visitCallInst()
2101 case Intrinsic::arm_neon_vst2: in visitCallInst()
2102 case Intrinsic::arm_neon_vst3: in visitCallInst()
2103 case Intrinsic::arm_neon_vst4: in visitCallInst()
2104 case Intrinsic::arm_neon_vst2lane: in visitCallInst()
2105 case Intrinsic::arm_neon_vst3lane: in visitCallInst()
2106 case Intrinsic::arm_neon_vst4lane: { in visitCallInst()
2107 unsigned MemAlign = getKnownAlignment(II->getArgOperand(0), DL, II, AC, DT); in visitCallInst()
2108 unsigned AlignArg = II->getNumArgOperands() - 1; in visitCallInst()
2109 ConstantInt *IntrAlign = dyn_cast<ConstantInt>(II->getArgOperand(AlignArg)); in visitCallInst()
2110 if (IntrAlign && IntrAlign->getZExtValue() < MemAlign) { in visitCallInst()
2111 II->setArgOperand(AlignArg, in visitCallInst()
2112 ConstantInt::get(Type::getInt32Ty(II->getContext()), in visitCallInst()
2119 case Intrinsic::arm_neon_vmulls: in visitCallInst()
2120 case Intrinsic::arm_neon_vmullu: in visitCallInst()
2121 case Intrinsic::aarch64_neon_smull: in visitCallInst()
2122 case Intrinsic::aarch64_neon_umull: { in visitCallInst()
2123 Value *Arg0 = II->getArgOperand(0); in visitCallInst()
2124 Value *Arg1 = II->getArgOperand(1); in visitCallInst()
2128 return replaceInstUsesWith(CI, ConstantAggregateZero::get(II->getType())); in visitCallInst()
2131 // Check for constant LHS & RHS - in this case we just simplify. in visitCallInst()
2132 bool Zext = (II->getIntrinsicID() == Intrinsic::arm_neon_vmullu || in visitCallInst()
2133 II->getIntrinsicID() == Intrinsic::aarch64_neon_umull); in visitCallInst()
2134 VectorType *NewVT = cast<VectorType>(II->getType()); in visitCallInst()
2143 // Couldn't simplify - canonicalize constant to the RHS. in visitCallInst()
2150 dyn_cast_or_null<ConstantInt>(CV1->getSplatValue())) in visitCallInst()
2151 if (Splat->isOne()) in visitCallInst()
2152 return CastInst::CreateIntegerCast(Arg0, II->getType(), in visitCallInst()
2158 case Intrinsic::amdgcn_rcp: { in visitCallInst()
2159 if (const ConstantFP *C = dyn_cast<ConstantFP>(II->getArgOperand(0))) { in visitCallInst()
2160 const APFloat &ArgVal = C->getValueAPF(); in visitCallInst()
2167 return replaceInstUsesWith(CI, ConstantFP::get(II->getContext(), Val)); in visitCallInst()
2172 case Intrinsic::amdgcn_frexp_mant: in visitCallInst()
2173 case Intrinsic::amdgcn_frexp_exp: { in visitCallInst()
2174 Value *Src = II->getArgOperand(0); in visitCallInst()
2177 APFloat Significand = frexp(C->getValueAPF(), Exp, in visitCallInst()
2180 if (II->getIntrinsicID() == Intrinsic::amdgcn_frexp_mant) { in visitCallInst()
2181 return replaceInstUsesWith(CI, ConstantFP::get(II->getContext(), in visitCallInst()
2189 return replaceInstUsesWith(CI, ConstantInt::get(II->getType(), Exp)); in visitCallInst()
2193 return replaceInstUsesWith(CI, UndefValue::get(II->getType())); in visitCallInst()
2197 case Intrinsic::stackrestore: { in visitCallInst()
2200 if (IntrinsicInst *SS = dyn_cast<IntrinsicInst>(II->getArgOperand(0))) { in visitCallInst()
2201 if (SS->getIntrinsicID() == Intrinsic::stacksave) { in visitCallInst()
2202 if (&*++SS->getIterator() == II) in visitCallInst()
2210 TerminatorInst *TI = II->getParent()->getTerminator(); in visitCallInst()
2220 if (II->getIntrinsicID() == Intrinsic::stackrestore) in visitCallInst()
2223 // Bail if we cross over an intrinsic with side effects, such as in visitCallInst()
2225 if (II->mayHaveSideEffects()) { in visitCallInst()
2230 // If we found a non-intrinsic call, we can't remove the stack in visitCallInst()
2245 case Intrinsic::lifetime_start: in visitCallInst()
2246 if (removeTriviallyEmptyRange(*II, Intrinsic::lifetime_start, in visitCallInst()
2247 Intrinsic::lifetime_end, *this)) in visitCallInst()
2250 case Intrinsic::assume: { in visitCallInst()
2251 Value *IIOperand = II->getArgOperand(0); in visitCallInst()
2253 if (match(II->getNextNode(), in visitCallInst()
2254 m_Intrinsic<Intrinsic::assume>(m_Specific(IIOperand)))) in visitCallInst()
2257 // Canonicalize assume(a && b) -> assume(a); assume(b); in visitCallInst()
2260 Value *AssumeIntrinsic = II->getCalledValue(), *A, *B; in visitCallInst()
2262 Builder->CreateCall(AssumeIntrinsic, A, II->getName()); in visitCallInst()
2263 Builder->CreateCall(AssumeIntrinsic, B, II->getName()); in visitCallInst()
2266 // assume(!(a || b)) -> assume(!a); assume(!b); in visitCallInst()
2268 Builder->CreateCall(AssumeIntrinsic, Builder->CreateNot(A), in visitCallInst()
2269 II->getName()); in visitCallInst()
2270 Builder->CreateCall(AssumeIntrinsic, Builder->CreateNot(B), in visitCallInst()
2271 II->getName()); in visitCallInst()
2275 // assume( (load addr) != null ) -> add 'nonnull' metadata to load in visitCallInst()
2278 Value *LHS = ICmp->getOperand(0); in visitCallInst()
2279 Value *RHS = ICmp->getOperand(1); in visitCallInst()
2280 if (ICmpInst::ICMP_NE == ICmp->getPredicate() && in visitCallInst()
2283 RHS->getType()->isPointerTy() && in visitCallInst()
2284 cast<Constant>(RHS)->isNullValue()) { in visitCallInst()
2287 MDNode *MD = MDNode::get(II->getContext(), None); in visitCallInst()
2288 LI->setMetadata(LLVMContext::MD_nonnull, MD); in visitCallInst()
2304 case Intrinsic::experimental_gc_relocate: { in visitCallInst()
2308 Value *DerivedPtr = cast<GCRelocateInst>(II)->getDerivedPtr(); in visitCallInst()
2312 if (II->use_empty()) in visitCallInst()
2321 return replaceInstUsesWith(*II, UndefValue::get(II->getType())); in visitCallInst()
2323 if (auto *PT = dyn_cast<PointerType>(II->getType())) { in visitCallInst()
2328 // Use null-pointer of gc_relocate's type to replace it. in visitCallInst()
2329 return replaceInstUsesWith(*II, ConstantPointerNull::get(PT)); in visitCallInst()
2331 // isKnownNonNull -> nonnull attribute in visitCallInst()
2333 II->addAttribute(AttributeSet::ReturnIndex, Attribute::NonNull); in visitCallInst()
2336 // TODO: bitcast(relocate(p)) -> relocate(bitcast(p)) in visitCallInst()
2339 // TODO: relocate((gep p, C, C2, ...)) -> gep(relocate(p), C, C2, ...) in visitCallInst()
2359 if (!CI->isLosslessCast()) in isSafeToEliminateVarargsCast()
2362 // If this is a GC intrinsic, avoid munging types. We need types for in isSafeToEliminateVarargsCast()
2377 cast<PointerType>(CI->getOperand(0)->getType())->getElementType(); in isSafeToEliminateVarargsCast()
2378 Type* DstTy = cast<PointerType>(CI->getType())->getElementType(); in isSafeToEliminateVarargsCast()
2379 if (!SrcTy->isSized() || !DstTy->isSized()) in isSafeToEliminateVarargsCast()
2387 if (!CI->getCalledFunction()) return nullptr; in tryOptimizeCall()
2395 return CI->use_empty() ? CI : replaceInstUsesWith(*CI, With); in tryOptimizeCall()
2404 Value *Underlying = TrampMem->stripPointerCasts(); in findInitTrampolineFromAlloca()
2406 (!Underlying->hasOneUse() || Underlying->user_back() != TrampMem)) in findInitTrampolineFromAlloca()
2412 for (User *U : TrampMem->users()) { in findInitTrampolineFromAlloca()
2416 if (II->getIntrinsicID() == Intrinsic::init_trampoline) { in findInitTrampolineFromAlloca()
2423 if (II->getIntrinsicID() == Intrinsic::adjust_trampoline) in findInitTrampolineFromAlloca()
2434 if (InitTrampoline->getOperand(0) != TrampMem) in findInitTrampolineFromAlloca()
2444 for (BasicBlock::iterator I = AdjustTramp->getIterator(), in findInitTrampolineFromBB()
2445 E = AdjustTramp->getParent()->begin(); in findInitTrampolineFromBB()
2447 Instruction *Inst = &*--I; in findInitTrampolineFromBB()
2449 if (II->getIntrinsicID() == Intrinsic::init_trampoline && in findInitTrampolineFromBB()
2450 II->getOperand(0) == TrampMem) in findInitTrampolineFromBB()
2452 if (Inst->mayWriteToMemory()) in findInitTrampolineFromBB()
2463 Callee = Callee->stripPointerCasts(); in findInitTrampoline()
2466 AdjustTramp->getIntrinsicID() != Intrinsic::adjust_trampoline) in findInitTrampoline()
2469 Value *TrampMem = AdjustTramp->getOperand(0); in findInitTrampoline()
2486 // Mark any parameters that are known to be non-null with the nonnull in visitCallSite()
2493 if (V->getType()->isPointerTy() && in visitCallSite()
2504 LLVMContext &Ctx = CS.getInstruction()->getContext(); in visitCallSite()
2506 Attribute::get(Ctx, Attribute::NonNull)); in visitCallSite()
2519 if (CS.isConvergent() && !CalleeF->isConvergent() && in visitCallSite()
2520 !CalleeF->isIntrinsic()) { in visitCallSite()
2529 if (CalleeF->getCallingConv() != CS.getCallingConv() && in visitCallSite()
2533 !CalleeF->isDeclaration()) { in visitCallSite()
2535 new StoreInst(ConstantInt::getTrue(Callee->getContext()), in visitCallSite()
2536 UndefValue::get(Type::getInt1PtrTy(Callee->getContext())), in visitCallSite()
2540 if (!OldCall->getType()->isVoidTy()) in visitCallSite()
2541 replaceInstUsesWith(*OldCall, UndefValue::get(OldCall->getType())); in visitCallSite()
2547 cast<InvokeInst>(OldCall)->setCalledFunction( in visitCallSite()
2548 Constant::getNullValue(CalleeF->getType())); in visitCallSite()
2556 if (!CS.getInstruction()->getType()->isVoidTy()) in visitCallSite()
2558 UndefValue::get(CS.getInstruction()->getType())); in visitCallSite()
2568 new StoreInst(ConstantInt::getTrue(Callee->getContext()), in visitCallSite()
2569 UndefValue::get(Type::getInt1PtrTy(Callee->getContext())), in visitCallSite()
2578 PointerType *PTy = cast<PointerType>(Callee->getType()); in visitCallSite()
2579 FunctionType *FTy = cast<FunctionType>(PTy->getElementType()); in visitCallSite()
2580 if (FTy->isVarArg()) { in visitCallSite()
2581 int ix = FTy->getNumParams(); in visitCallSite()
2584 for (CallSite::arg_iterator I = CS.arg_begin() + FTy->getNumParams(), in visitCallSite()
2588 *I = CI->getOperand(0); in visitCallSite()
2595 // Inline asm calls cannot throw - mark them 'nounwind'. in visitCallSite()
2617 dyn_cast<Function>(CS.getCalledValue()->stripPointerCasts()); in transformConstExprCastCall()
2622 if (Callee->hasFnAttribute("thunk")) in transformConstExprCastCall()
2631 FunctionType *FT = Callee->getFunctionType(); in transformConstExprCastCall()
2632 Type *OldRetTy = Caller->getType(); in transformConstExprCastCall()
2633 Type *NewRetTy = FT->getReturnType(); in transformConstExprCastCall()
2638 if (NewRetTy->isStructTy()) in transformConstExprCastCall()
2642 if (Callee->isDeclaration()) in transformConstExprCastCall()
2645 if (!Caller->use_empty() && in transformConstExprCastCall()
2646 // void -> non-void is handled specially in transformConstExprCastCall()
2647 !NewRetTy->isVoidTy()) in transformConstExprCastCall()
2651 if (!CallerPAL.isEmpty() && !Caller->use_empty()) { in transformConstExprCastCall()
2661 if (!Caller->use_empty()) in transformConstExprCastCall()
2663 for (User *U : II->users()) in transformConstExprCastCall()
2665 if (PN->getParent() == II->getNormalDest() || in transformConstExprCastCall()
2666 PN->getParent() == II->getUnwindDest()) in transformConstExprCastCall()
2671 unsigned NumCommonArgs = std::min(FT->getNumParams(), NumActualArgs); in transformConstExprCastCall()
2681 if (Callee->getAttributes().hasAttrSomewhere(Attribute::InAlloca) || in transformConstExprCastCall()
2682 Callee->getAttributes().hasAttrSomewhere(Attribute::ByVal)) in transformConstExprCastCall()
2687 Type *ParamTy = FT->getParamType(i); in transformConstExprCastCall()
2688 Type *ActTy = (*AI)->getType(); in transformConstExprCastCall()
2706 if (!ParamPTy || !ParamPTy->getElementType()->isSized()) in transformConstExprCastCall()
2709 Type *CurElTy = ActTy->getPointerElementType(); in transformConstExprCastCall()
2711 DL.getTypeAllocSize(ParamPTy->getElementType())) in transformConstExprCastCall()
2716 if (Callee->isDeclaration()) { in transformConstExprCastCall()
2718 if (FT->getNumParams() < NumActualArgs && !FT->isVarArg()) in transformConstExprCastCall()
2724 PointerType *APTy = cast<PointerType>(CS.getCalledValue()->getType()); in transformConstExprCastCall()
2725 if (FT->isVarArg()!=cast<FunctionType>(APTy->getElementType())->isVarArg()) in transformConstExprCastCall()
2731 if (FT->isVarArg() && in transformConstExprCastCall()
2732 cast<FunctionType>(APTy->getElementType())->isVarArg() && in transformConstExprCastCall()
2733 FT->getNumParams() != in transformConstExprCastCall()
2734 cast<FunctionType>(APTy->getElementType())->getNumParams()) in transformConstExprCastCall()
2738 if (FT->getNumParams() < NumActualArgs && FT->isVarArg() && in transformConstExprCastCall()
2743 for (unsigned i = CallerPAL.getNumSlots(); i; --i) { in transformConstExprCastCall()
2744 unsigned Index = CallerPAL.getSlotIndex(i - 1); in transformConstExprCastCall()
2745 if (Index <= FT->getNumParams()) in transformConstExprCastCall()
2749 AttributeSet PAttrs = CallerPAL.getSlotAttributes(i - 1); in transformConstExprCastCall()
2762 // Get any return attributes. in transformConstExprCastCall()
2771 attrVec.push_back(AttributeSet::get(Caller->getContext(), in transformConstExprCastCall()
2776 Type *ParamTy = FT->getParamType(i); in transformConstExprCastCall()
2778 if ((*AI)->getType() == ParamTy) { in transformConstExprCastCall()
2781 Args.push_back(Builder->CreateBitOrPointerCast(*AI, ParamTy)); in transformConstExprCastCall()
2787 attrVec.push_back(AttributeSet::get(Caller->getContext(), i + 1, in transformConstExprCastCall()
2793 for (unsigned i = NumCommonArgs; i != FT->getNumParams(); ++i) in transformConstExprCastCall()
2794 Args.push_back(Constant::getNullValue(FT->getParamType(i))); in transformConstExprCastCall()
2797 if (FT->getNumParams() < NumActualArgs) { in transformConstExprCastCall()
2798 // TODO: if (!FT->isVarArg()) this call may be unreachable. PR14722 in transformConstExprCastCall()
2799 if (FT->isVarArg()) { in transformConstExprCastCall()
2801 for (unsigned i = FT->getNumParams(); i != NumActualArgs; ++i, ++AI) { in transformConstExprCastCall()
2802 Type *PTy = getPromotedType((*AI)->getType()); in transformConstExprCastCall()
2803 if (PTy != (*AI)->getType()) { in transformConstExprCastCall()
2807 Args.push_back(Builder->CreateCast(opcode, *AI, PTy)); in transformConstExprCastCall()
2815 attrVec.push_back(AttributeSet::get(FT->getContext(), i + 1, in transformConstExprCastCall()
2823 attrVec.push_back(AttributeSet::get(Callee->getContext(), FnAttrs)); in transformConstExprCastCall()
2825 if (NewRetTy->isVoidTy()) in transformConstExprCastCall()
2826 Caller->setName(""); // Void type should not have a name. in transformConstExprCastCall()
2828 const AttributeSet &NewCallerPAL = AttributeSet::get(Callee->getContext(), in transformConstExprCastCall()
2836 NC = Builder->CreateInvoke(Callee, II->getNormalDest(), II->getUnwindDest(), in transformConstExprCastCall()
2838 NC->takeName(II); in transformConstExprCastCall()
2839 cast<InvokeInst>(NC)->setCallingConv(II->getCallingConv()); in transformConstExprCastCall()
2840 cast<InvokeInst>(NC)->setAttributes(NewCallerPAL); in transformConstExprCastCall()
2843 NC = Builder->CreateCall(Callee, Args, OpBundles); in transformConstExprCastCall()
2844 NC->takeName(CI); in transformConstExprCastCall()
2845 if (CI->isTailCall()) in transformConstExprCastCall()
2846 cast<CallInst>(NC)->setTailCall(); in transformConstExprCastCall()
2847 cast<CallInst>(NC)->setCallingConv(CI->getCallingConv()); in transformConstExprCastCall()
2848 cast<CallInst>(NC)->setAttributes(NewCallerPAL); in transformConstExprCastCall()
2853 if (OldRetTy != NV->getType() && !Caller->use_empty()) { in transformConstExprCastCall()
2854 if (!NV->getType()->isVoidTy()) { in transformConstExprCastCall()
2856 NC->setDebugLoc(Caller->getDebugLoc()); in transformConstExprCastCall()
2859 // non-phi, instruction in the normal successor block. in transformConstExprCastCall()
2861 BasicBlock::iterator I = II->getNormalDest()->getFirstInsertionPt(); in transformConstExprCastCall()
2869 NV = UndefValue::get(Caller->getType()); in transformConstExprCastCall()
2873 if (!Caller->use_empty()) in transformConstExprCastCall()
2875 else if (Caller->hasValueHandle()) { in transformConstExprCastCall()
2876 if (OldRetTy == NV->getType()) in transformConstExprCastCall()
2889 /// intrinsic pair into a direct call to the underlying function.
2894 PointerType *PTy = cast<PointerType>(Callee->getType()); in transformCallThroughTrampoline()
2895 FunctionType *FTy = cast<FunctionType>(PTy->getElementType()); in transformCallThroughTrampoline()
2898 // If the call already has the 'nest' attribute somewhere then give up - in transformCallThroughTrampoline()
2906 Function *NestF =cast<Function>(Tramp->getArgOperand(1)->stripPointerCasts()); in transformCallThroughTrampoline()
2907 FunctionType *NestFTy = cast<FunctionType>(NestF->getValueType()); in transformCallThroughTrampoline()
2909 const AttributeSet &NestAttrs = NestF->getAttributes(); in transformCallThroughTrampoline()
2916 for (FunctionType::param_iterator I = NestFTy->param_begin(), in transformCallThroughTrampoline()
2917 E = NestFTy->param_end(); I != E; ++NestIdx, ++I) in transformCallThroughTrampoline()
2938 NewAttrs.push_back(AttributeSet::get(Caller->getContext(), in transformCallThroughTrampoline()
2947 Value *NestVal = Tramp->getArgOperand(2); in transformCallThroughTrampoline()
2948 if (NestVal->getType() != NestTy) in transformCallThroughTrampoline()
2949 NestVal = Builder->CreateBitCast(NestVal, NestTy, "nest"); in transformCallThroughTrampoline()
2951 NewAttrs.push_back(AttributeSet::get(Caller->getContext(), in transformCallThroughTrampoline()
2963 NewAttrs.push_back(AttributeSet::get(Caller->getContext(), in transformCallThroughTrampoline()
2974 NewAttrs.push_back(AttributeSet::get(FTy->getContext(), in transformCallThroughTrampoline()
2982 NewTypes.reserve(FTy->getNumParams()+1); in transformCallThroughTrampoline()
2988 FunctionType::param_iterator I = FTy->param_begin(), in transformCallThroughTrampoline()
2989 E = FTy->param_end(); in transformCallThroughTrampoline()
3009 FunctionType *NewFTy = FunctionType::get(FTy->getReturnType(), NewTypes, in transformCallThroughTrampoline()
3010 FTy->isVarArg()); in transformCallThroughTrampoline()
3012 NestF->getType() == PointerType::getUnqual(NewFTy) ? in transformCallThroughTrampoline()
3016 AttributeSet::get(FTy->getContext(), NewAttrs); in transformCallThroughTrampoline()
3024 II->getNormalDest(), II->getUnwindDest(), in transformCallThroughTrampoline()
3026 cast<InvokeInst>(NewCaller)->setCallingConv(II->getCallingConv()); in transformCallThroughTrampoline()
3027 cast<InvokeInst>(NewCaller)->setAttributes(NewPAL); in transformCallThroughTrampoline()
3030 if (cast<CallInst>(Caller)->isTailCall()) in transformCallThroughTrampoline()
3031 cast<CallInst>(NewCaller)->setTailCall(); in transformCallThroughTrampoline()
3032 cast<CallInst>(NewCaller)-> in transformCallThroughTrampoline()
3033 setCallingConv(cast<CallInst>(Caller)->getCallingConv()); in transformCallThroughTrampoline()
3034 cast<CallInst>(NewCaller)->setAttributes(NewPAL); in transformCallThroughTrampoline()
3045 NestF->getType() == PTy ? NestF : in transformCallThroughTrampoline()