Lines Matching +full:asm +full:- +full:operand +full:- +full:widths
1 //===- CodeGenPrepare.cpp - Prepare a function for code generation --------===//
8 //===----------------------------------------------------------------------===//
11 // SelectionDAG-based code generation. This works around limitations in it's
12 // basic-block-at-a-time approach. It should eventually be removed.
14 //===----------------------------------------------------------------------===//
80 "disable-cgp-branch-opts", cl::Hidden, cl::init(false),
84 DisableGCOpts("disable-cgp-gc-opts", cl::Hidden, cl::init(false),
88 "disable-cgp-select2branch", cl::Hidden, cl::init(false),
92 "addr-sink-using-gep", cl::Hidden, cl::init(false),
96 "enable-andcmp-sinking", cl::Hidden, cl::init(true),
100 "disable-cgp-store-extract", cl::Hidden, cl::init(false),
104 "stress-cgp-store-extract", cl::Hidden, cl::init(false),
108 "disable-cgp-ext-ld-promotion", cl::Hidden, cl::init(false),
109 cl::desc("Disable ext(promotable(ld)) -> promoted(ext(ld)) optimization in "
113 "stress-cgp-ext-ld-promotion", cl::Hidden, cl::init(false),
114 cl::desc("Stress test ext(promotable(ld)) -> promoted(ext(ld)) "
118 "disable-preheader-prot", cl::Hidden, cl::init(false),
138 /// Keeps track of non-local addresses that have been sunk into a block.
218 DL = &F.getParent()->getDataLayout(); in runOnFunction()
227 TLI = TM->getSubtargetImpl(F)->getTargetLowering(); in runOnFunction()
235 if (!OptSize && TLI && TLI->isSlowDivBypassed()) { in runOnFunction()
237 TLI->getBypassSlowDivWidths(); in runOnFunction()
242 BasicBlock* Next = BB->getNextNode(); in runOnFunction()
259 // users. Do this before OptimizeBlock -> OptimizeInst -> in runOnFunction()
343 BasicBlock *SinglePred = BB->getSinglePredecessor(); in eliminateFallThrough()
346 if (!SinglePred || SinglePred == BB || BB->hasAddressTaken()) continue; in eliminateFallThrough()
348 BranchInst *Term = dyn_cast<BranchInst>(SinglePred->getTerminator()); in eliminateFallThrough()
349 if (Term && !Term->isConditional()) { in eliminateFallThrough()
354 bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock(); in eliminateFallThrough()
357 if (isEntry && BB != &BB->getParent()->getEntryBlock()) in eliminateFallThrough()
358 BB->moveBefore(&BB->getParent()->getEntryBlock()); in eliminateFallThrough()
361 I = BB->getIterator(); in eliminateFallThrough()
369 /// edges in ways that are non-optimal for isel. Start by eliminating these
373 SmallVector<Loop *, 16> LoopList(LI->begin(), LI->end()); in eliminateMostlyEmptyBlocks()
376 LoopList.insert(LoopList.end(), L->begin(), L->end()); in eliminateMostlyEmptyBlocks()
377 if (BasicBlock *Preheader = L->getLoopPreheader()) in eliminateMostlyEmptyBlocks()
387 BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator()); in eliminateMostlyEmptyBlocks()
388 if (!BI || !BI->isUnconditional()) in eliminateMostlyEmptyBlocks()
393 BasicBlock::iterator BBI = BI->getIterator(); in eliminateMostlyEmptyBlocks()
394 if (BBI != BB->begin()) { in eliminateMostlyEmptyBlocks()
395 --BBI; in eliminateMostlyEmptyBlocks()
397 if (BBI == BB->begin()) in eliminateMostlyEmptyBlocks()
399 --BBI; in eliminateMostlyEmptyBlocks()
406 BasicBlock *DestBB = BI->getSuccessor(0); in eliminateMostlyEmptyBlocks()
418 !(BB->getSinglePredecessor() && BB->getSinglePredecessor()->getSingleSuccessor())) in eliminateMostlyEmptyBlocks()
428 /// unconditional branch between them, and BB contains no other non-phi
435 BasicBlock::const_iterator BBI = BB->begin(); in canMergeBlocks()
437 for (const User *U : PN->users()) { in canMergeBlocks()
439 if (UI->getParent() != DestBB || !isa<PHINode>(UI)) in canMergeBlocks()
444 if (UI->getParent() == DestBB) { in canMergeBlocks()
446 for (unsigned I = 0, E = UPN->getNumIncomingValues(); I != E; ++I) { in canMergeBlocks()
447 Instruction *Insn = dyn_cast<Instruction>(UPN->getIncomingValue(I)); in canMergeBlocks()
448 if (Insn && Insn->getParent() == BB && in canMergeBlocks()
449 Insn->getParent() != UPN->getIncomingBlock(I)) in canMergeBlocks()
459 const PHINode *DestBBPN = dyn_cast<PHINode>(DestBB->begin()); in canMergeBlocks()
464 if (const PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) { in canMergeBlocks()
466 for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i) in canMergeBlocks()
467 BBPreds.insert(BBPN->getIncomingBlock(i)); in canMergeBlocks()
473 for (unsigned i = 0, e = DestBBPN->getNumIncomingValues(); i != e; ++i) { in canMergeBlocks()
474 BasicBlock *Pred = DestBBPN->getIncomingBlock(i); in canMergeBlocks()
476 BBI = DestBB->begin(); in canMergeBlocks()
478 const Value *V1 = PN->getIncomingValueForBlock(Pred); in canMergeBlocks()
479 const Value *V2 = PN->getIncomingValueForBlock(BB); in canMergeBlocks()
483 if (V2PN->getParent() == BB) in canMergeBlocks()
484 V2 = V2PN->getIncomingValueForBlock(Pred); in canMergeBlocks()
499 BranchInst *BI = cast<BranchInst>(BB->getTerminator()); in eliminateMostlyEmptyBlock()
500 BasicBlock *DestBB = BI->getSuccessor(0); in eliminateMostlyEmptyBlock()
502 DEBUG(dbgs() << "MERGING MOSTLY EMPTY BLOCKS - BEFORE:\n" << *BB << *DestBB); in eliminateMostlyEmptyBlock()
506 if (BasicBlock *SinglePred = DestBB->getSinglePredecessor()) { in eliminateMostlyEmptyBlock()
510 bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock(); in eliminateMostlyEmptyBlock()
513 if (isEntry && BB != &BB->getParent()->getEntryBlock()) in eliminateMostlyEmptyBlock()
514 BB->moveBefore(&BB->getParent()->getEntryBlock()); in eliminateMostlyEmptyBlock()
524 for (BasicBlock::iterator BBI = DestBB->begin(); in eliminateMostlyEmptyBlock()
527 Value *InVal = PN->removeIncomingValue(BB, false); in eliminateMostlyEmptyBlock()
532 if (InValPhi && InValPhi->getParent() == BB) { in eliminateMostlyEmptyBlock()
534 for (unsigned i = 0, e = InValPhi->getNumIncomingValues(); i != e; ++i) in eliminateMostlyEmptyBlock()
535 PN->addIncoming(InValPhi->getIncomingValue(i), in eliminateMostlyEmptyBlock()
536 InValPhi->getIncomingBlock(i)); in eliminateMostlyEmptyBlock()
540 if (PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) { in eliminateMostlyEmptyBlock()
541 for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i) in eliminateMostlyEmptyBlock()
542 PN->addIncoming(InVal, BBPN->getIncomingBlock(i)); in eliminateMostlyEmptyBlock()
545 PN->addIncoming(InVal, *PI); in eliminateMostlyEmptyBlock()
552 BB->replaceAllUsesWith(DestBB); in eliminateMostlyEmptyBlock()
553 BB->eraseFromParent(); in eliminateMostlyEmptyBlock()
570 auto K = std::make_pair(ThisRelocate->getBasePtrIndex(), in computeBaseDerivedRelocateMap()
571 ThisRelocate->getDerivedPtrIndex()); in computeBaseDerivedRelocateMap()
590 RelocateInstMap[MaybeBase->second].push_back(I); in computeBaseDerivedRelocateMap()
598 for (unsigned i = 1; i < GEP->getNumOperands(); i++) { in getGEPSmallConstantIntOffsetV()
600 auto Op = dyn_cast<ConstantInt>(GEP->getOperand(i)); in getGEPSmallConstantIntOffsetV()
601 if (!Op || Op->getZExtValue() > 20) in getGEPSmallConstantIntOffsetV()
605 for (unsigned i = 1; i < GEP->getNumOperands(); i++) in getGEPSmallConstantIntOffsetV()
606 OffsetV.push_back(GEP->getOperand(i)); in getGEPSmallConstantIntOffsetV()
617 assert(ToReplace->getBasePtrIndex() == RelocatedBase->getBasePtrIndex() && in simplifyRelocatesOffABase()
619 if (ToReplace->getBasePtrIndex() == ToReplace->getDerivedPtrIndex()) { in simplifyRelocatesOffABase()
624 if (RelocatedBase->getParent() != ToReplace->getParent()) { in simplifyRelocatesOffABase()
632 Value *Base = ToReplace->getBasePtr(); in simplifyRelocatesOffABase()
633 auto Derived = dyn_cast<GetElementPtrInst>(ToReplace->getDerivedPtr()); in simplifyRelocatesOffABase()
634 if (!Derived || Derived->getPointerOperand() != Base) in simplifyRelocatesOffABase()
642 assert(RelocatedBase->getNextNode() && in simplifyRelocatesOffABase()
646 IRBuilder<> Builder(RelocatedBase->getNextNode()); in simplifyRelocatesOffABase()
647 Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc()); in simplifyRelocatesOffABase()
671 if (RelocatedBase->getType() != Base->getType()) { in simplifyRelocatesOffABase()
673 Builder.CreateBitCast(RelocatedBase, Base->getType()); in simplifyRelocatesOffABase()
676 Derived->getSourceElementType(), ActualRelocatedBase, makeArrayRef(OffsetV)); in simplifyRelocatesOffABase()
677 Replacement->takeName(ToReplace); in simplifyRelocatesOffABase()
681 if (Replacement->getType() != ToReplace->getType()) { in simplifyRelocatesOffABase()
683 Builder.CreateBitCast(Replacement, ToReplace->getType()); in simplifyRelocatesOffABase()
685 ToReplace->replaceAllUsesWith(ActualReplacement); in simplifyRelocatesOffABase()
686 ToReplace->eraseFromParent(); in simplifyRelocatesOffABase()
738 /// SinkCast - Sink the specified cast instruction into its user blocks
740 BasicBlock *DefBB = CI->getParent(); in SinkCast()
742 /// InsertedCasts - Only insert a cast in each block once. in SinkCast()
746 for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end(); in SinkCast()
753 BasicBlock *UserBB = User->getParent(); in SinkCast()
755 UserBB = PN->getIncomingBlock(TheUse); in SinkCast()
763 if (User->isEHPad()) in SinkCast()
767 // allow non-PHI instructions before the terminator, we can't sink the in SinkCast()
769 if (UserBB->getTerminator()->isEHPad()) in SinkCast()
779 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); in SinkCast()
780 assert(InsertPt != UserBB->end()); in SinkCast()
781 InsertedCast = CastInst::Create(CI->getOpcode(), CI->getOperand(0), in SinkCast()
782 CI->getType(), "", &*InsertPt); in SinkCast()
792 if (CI->use_empty()) { in SinkCast()
793 CI->eraseFromParent(); in SinkCast()
801 /// one pointer type to another, i32->i8 on PPC), sink it into user blocks to
809 EVT SrcVT = TLI.getValueType(DL, CI->getOperand(0)->getType()); in OptimizeNoopCopyExpression()
810 EVT DstVT = TLI.getValueType(DL, CI->getType()); in OptimizeNoopCopyExpression()
812 // This is an fp<->int conversion? in OptimizeNoopCopyExpression()
823 if (TLI.getTypeAction(CI->getContext(), SrcVT) == in OptimizeNoopCopyExpression()
825 SrcVT = TLI.getTypeToTransformTo(CI->getContext(), SrcVT); in OptimizeNoopCopyExpression()
826 if (TLI.getTypeAction(CI->getContext(), DstVT) == in OptimizeNoopCopyExpression()
828 DstVT = TLI.getTypeToTransformTo(CI->getContext(), DstVT); in OptimizeNoopCopyExpression()
848 Type *Ty = AddI->getType(); in CombineUAddWithOverflow()
856 if (AddI->getParent() != CI->getParent() && !AddI->hasOneUse()) in CombineUAddWithOverflow()
862 if (AddI->hasOneUse()) in CombineUAddWithOverflow()
863 assert(*AddI->user_begin() == CI && "expected!"); in CombineUAddWithOverflow()
866 Module *M = CI->getModule(); in CombineUAddWithOverflow()
869 auto *InsertPt = AddI->hasOneUse() ? CI : AddI; in CombineUAddWithOverflow()
877 CI->replaceAllUsesWith(Overflow); in CombineUAddWithOverflow()
878 AddI->replaceAllUsesWith(UAdd); in CombineUAddWithOverflow()
879 CI->eraseFromParent(); in CombineUAddWithOverflow()
880 AddI->eraseFromParent(); in CombineUAddWithOverflow()
891 BasicBlock *DefBB = CI->getParent(); in SinkCmpExpression()
893 // Avoid sinking soft-FP comparisons, since this can move them into a loop. in SinkCmpExpression()
894 if (TLI && TLI->useSoftFloat() && isa<FCmpInst>(CI)) in SinkCmpExpression()
901 for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end(); in SinkCmpExpression()
914 BasicBlock *UserBB = User->getParent(); in SinkCmpExpression()
923 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); in SinkCmpExpression()
924 assert(InsertPt != UserBB->end()); in SinkCmpExpression()
926 CmpInst::Create(CI->getOpcode(), CI->getPredicate(), in SinkCmpExpression()
927 CI->getOperand(0), CI->getOperand(1), "", &*InsertPt); in SinkCmpExpression()
937 if (CI->use_empty()) { in SinkCmpExpression()
938 CI->eraseFromParent(); in SinkCmpExpression()
962 if (User->getOpcode() != Instruction::And || in isExtractBitsCandidateUse()
963 !isa<ConstantInt>(User->getOperand(1))) in isExtractBitsCandidateUse()
966 const APInt &Cimm = cast<ConstantInt>(User->getOperand(1))->getValue(); in isExtractBitsCandidateUse()
979 BasicBlock *UserBB = User->getParent(); in SinkShiftAndTruncate()
984 for (Value::user_iterator TruncUI = TruncI->user_begin(), in SinkShiftAndTruncate()
985 TruncE = TruncI->user_end(); in SinkShiftAndTruncate()
994 int ISDOpcode = TLI.InstructionOpcodeToISD(TruncUser->getOpcode()); in SinkShiftAndTruncate()
1002 // operand or other means. There's no good way to find out though. in SinkShiftAndTruncate()
1004 ISDOpcode, TLI.getValueType(DL, TruncUser->getType(), true))) in SinkShiftAndTruncate()
1011 BasicBlock *TruncUserBB = TruncUser->getParent(); in SinkShiftAndTruncate()
1020 BasicBlock::iterator InsertPt = TruncUserBB->getFirstInsertionPt(); in SinkShiftAndTruncate()
1021 assert(InsertPt != TruncUserBB->end()); in SinkShiftAndTruncate()
1023 if (ShiftI->getOpcode() == Instruction::AShr) in SinkShiftAndTruncate()
1024 InsertedShift = BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, in SinkShiftAndTruncate()
1027 InsertedShift = BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, in SinkShiftAndTruncate()
1031 BasicBlock::iterator TruncInsertPt = TruncUserBB->getFirstInsertionPt(); in SinkShiftAndTruncate()
1033 assert(TruncInsertPt != TruncUserBB->end()); in SinkShiftAndTruncate()
1035 InsertedTrunc = CastInst::Create(TruncI->getOpcode(), InsertedShift, in SinkShiftAndTruncate()
1036 TruncI->getType(), "", &*TruncInsertPt); in SinkShiftAndTruncate()
1066 BasicBlock *DefBB = ShiftI->getParent(); in OptimizeExtractBits()
1071 bool shiftIsLegal = TLI.isTypeLegal(TLI.getValueType(DL, ShiftI->getType())); in OptimizeExtractBits()
1074 for (Value::user_iterator UI = ShiftI->user_begin(), E = ShiftI->user_end(); in OptimizeExtractBits()
1088 BasicBlock *UserBB = User->getParent(); in OptimizeExtractBits()
1101 // ----> We will have an implicit truncate here if the architecture does in OptimizeExtractBits()
1109 (!TLI.isTypeLegal(TLI.getValueType(DL, User->getType())))) in OptimizeExtractBits()
1119 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); in OptimizeExtractBits()
1120 assert(InsertPt != UserBB->end()); in OptimizeExtractBits()
1122 if (ShiftI->getOpcode() == Instruction::AShr) in OptimizeExtractBits()
1123 InsertedShift = BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, in OptimizeExtractBits()
1126 InsertedShift = BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, in OptimizeExtractBits()
1137 if (ShiftI->use_empty()) in OptimizeExtractBits()
1138 ShiftI->eraseFromParent(); in OptimizeExtractBits()
1146 // to a chain of basic blocks, with loading element one-by-one if
1179 Value *Ptr = CI->getArgOperand(0); in scalarizeMaskedLoad()
1180 Value *Alignment = CI->getArgOperand(1); in scalarizeMaskedLoad()
1181 Value *Mask = CI->getArgOperand(2); in scalarizeMaskedLoad()
1182 Value *Src0 = CI->getArgOperand(3); in scalarizeMaskedLoad()
1184 unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue(); in scalarizeMaskedLoad()
1185 VectorType *VecType = dyn_cast<VectorType>(CI->getType()); in scalarizeMaskedLoad()
1188 Type *EltTy = CI->getType()->getVectorElementType(); in scalarizeMaskedLoad()
1190 IRBuilder<> Builder(CI->getContext()); in scalarizeMaskedLoad()
1192 BasicBlock *IfBlock = CI->getParent(); in scalarizeMaskedLoad()
1194 BasicBlock *PrevIfBlock = CI->getParent(); in scalarizeMaskedLoad()
1197 Builder.SetCurrentDebugLocation(CI->getDebugLoc()); in scalarizeMaskedLoad()
1199 // Short-cut if the mask is all-true. in scalarizeMaskedLoad()
1201 cast<Constant>(Mask)->isAllOnesValue(); in scalarizeMaskedLoad()
1205 CI->replaceAllUsesWith(NewI); in scalarizeMaskedLoad()
1206 CI->eraseFromParent(); in scalarizeMaskedLoad()
1211 AlignVal = std::min(AlignVal, VecType->getScalarSizeInBits()/8); in scalarizeMaskedLoad()
1214 EltTy->getPointerTo(cast<PointerType>(Ptr->getType())->getAddressSpace()); in scalarizeMaskedLoad()
1216 unsigned VectorWidth = VecType->getNumElements(); in scalarizeMaskedLoad()
1225 if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue()) in scalarizeMaskedLoad()
1234 CI->replaceAllUsesWith(NewI); in scalarizeMaskedLoad()
1235 CI->eraseFromParent(); in scalarizeMaskedLoad()
1253 Phi->addIncoming(VResult, CondBlock); in scalarizeMaskedLoad()
1254 Phi->addIncoming(PrevPhi, PrevIfBlock); in scalarizeMaskedLoad()
1261 ConstantInt::get(Predicate->getType(), 1)); in scalarizeMaskedLoad()
1269 CondBlock = IfBlock->splitBasicBlock(InsertPt->getIterator(), "cond.load"); in scalarizeMaskedLoad()
1279 CondBlock->splitBasicBlock(InsertPt->getIterator(), "else"); in scalarizeMaskedLoad()
1281 Instruction *OldBr = IfBlock->getTerminator(); in scalarizeMaskedLoad()
1283 OldBr->eraseFromParent(); in scalarizeMaskedLoad()
1289 Phi->addIncoming(VResult, CondBlock); in scalarizeMaskedLoad()
1290 Phi->addIncoming(PrevPhi, PrevIfBlock); in scalarizeMaskedLoad()
1292 CI->replaceAllUsesWith(NewI); in scalarizeMaskedLoad()
1293 CI->eraseFromParent(); in scalarizeMaskedLoad()
1299 // to a chain of basic blocks, that stores element one-by-one if
1325 Value *Src = CI->getArgOperand(0); in scalarizeMaskedStore()
1326 Value *Ptr = CI->getArgOperand(1); in scalarizeMaskedStore()
1327 Value *Alignment = CI->getArgOperand(2); in scalarizeMaskedStore()
1328 Value *Mask = CI->getArgOperand(3); in scalarizeMaskedStore()
1330 unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue(); in scalarizeMaskedStore()
1331 VectorType *VecType = dyn_cast<VectorType>(Src->getType()); in scalarizeMaskedStore()
1334 Type *EltTy = VecType->getElementType(); in scalarizeMaskedStore()
1336 IRBuilder<> Builder(CI->getContext()); in scalarizeMaskedStore()
1338 BasicBlock *IfBlock = CI->getParent(); in scalarizeMaskedStore()
1340 Builder.SetCurrentDebugLocation(CI->getDebugLoc()); in scalarizeMaskedStore()
1342 // Short-cut if the mask is all-true. in scalarizeMaskedStore()
1344 cast<Constant>(Mask)->isAllOnesValue(); in scalarizeMaskedStore()
1348 CI->eraseFromParent(); in scalarizeMaskedStore()
1353 AlignVal = std::max(AlignVal, VecType->getScalarSizeInBits()/8); in scalarizeMaskedStore()
1356 EltTy->getPointerTo(cast<PointerType>(Ptr->getType())->getAddressSpace()); in scalarizeMaskedStore()
1358 unsigned VectorWidth = VecType->getNumElements(); in scalarizeMaskedStore()
1362 if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue()) in scalarizeMaskedStore()
1369 CI->eraseFromParent(); in scalarizeMaskedStore()
1383 ConstantInt::get(Predicate->getType(), 1)); in scalarizeMaskedStore()
1392 IfBlock->splitBasicBlock(InsertPt->getIterator(), "cond.store"); in scalarizeMaskedStore()
1402 CondBlock->splitBasicBlock(InsertPt->getIterator(), "else"); in scalarizeMaskedStore()
1404 Instruction *OldBr = IfBlock->getTerminator(); in scalarizeMaskedStore()
1406 OldBr->eraseFromParent(); in scalarizeMaskedStore()
1409 CI->eraseFromParent(); in scalarizeMaskedStore()
1415 // to a chain of basic blocks, with loading element one-by-one if
1444 Value *Ptrs = CI->getArgOperand(0); in scalarizeMaskedGather()
1445 Value *Alignment = CI->getArgOperand(1); in scalarizeMaskedGather()
1446 Value *Mask = CI->getArgOperand(2); in scalarizeMaskedGather()
1447 Value *Src0 = CI->getArgOperand(3); in scalarizeMaskedGather()
1449 VectorType *VecType = dyn_cast<VectorType>(CI->getType()); in scalarizeMaskedGather()
1453 IRBuilder<> Builder(CI->getContext()); in scalarizeMaskedGather()
1455 BasicBlock *IfBlock = CI->getParent(); in scalarizeMaskedGather()
1457 BasicBlock *PrevIfBlock = CI->getParent(); in scalarizeMaskedGather()
1459 unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue(); in scalarizeMaskedGather()
1461 Builder.SetCurrentDebugLocation(CI->getDebugLoc()); in scalarizeMaskedGather()
1467 unsigned VectorWidth = VecType->getNumElements(); in scalarizeMaskedGather()
1474 if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue()) in scalarizeMaskedGather()
1485 CI->replaceAllUsesWith(NewI); in scalarizeMaskedGather()
1486 CI->eraseFromParent(); in scalarizeMaskedGather()
1503 Phi->addIncoming(VResult, CondBlock); in scalarizeMaskedGather()
1504 Phi->addIncoming(PrevPhi, PrevIfBlock); in scalarizeMaskedGather()
1513 ConstantInt::get(Predicate->getType(), 1), in scalarizeMaskedGather()
1522 CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.load"); in scalarizeMaskedGather()
1533 BasicBlock *NewIfBlock = CondBlock->splitBasicBlock(InsertPt, "else"); in scalarizeMaskedGather()
1535 Instruction *OldBr = IfBlock->getTerminator(); in scalarizeMaskedGather()
1537 OldBr->eraseFromParent(); in scalarizeMaskedGather()
1543 Phi->addIncoming(VResult, CondBlock); in scalarizeMaskedGather()
1544 Phi->addIncoming(PrevPhi, PrevIfBlock); in scalarizeMaskedGather()
1546 CI->replaceAllUsesWith(NewI); in scalarizeMaskedGather()
1547 CI->eraseFromParent(); in scalarizeMaskedGather()
1553 // to a chain of basic blocks, that stores element one-by-one if
1579 Value *Src = CI->getArgOperand(0); in scalarizeMaskedScatter()
1580 Value *Ptrs = CI->getArgOperand(1); in scalarizeMaskedScatter()
1581 Value *Alignment = CI->getArgOperand(2); in scalarizeMaskedScatter()
1582 Value *Mask = CI->getArgOperand(3); in scalarizeMaskedScatter()
1584 assert(isa<VectorType>(Src->getType()) && in scalarizeMaskedScatter()
1586 assert(isa<VectorType>(Ptrs->getType()) && in scalarizeMaskedScatter()
1587 isa<PointerType>(Ptrs->getType()->getVectorElementType()) && in scalarizeMaskedScatter()
1590 IRBuilder<> Builder(CI->getContext()); in scalarizeMaskedScatter()
1592 BasicBlock *IfBlock = CI->getParent(); in scalarizeMaskedScatter()
1594 Builder.SetCurrentDebugLocation(CI->getDebugLoc()); in scalarizeMaskedScatter()
1596 unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue(); in scalarizeMaskedScatter()
1597 unsigned VectorWidth = Src->getType()->getVectorNumElements(); in scalarizeMaskedScatter()
1604 if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue()) in scalarizeMaskedScatter()
1612 CI->eraseFromParent(); in scalarizeMaskedScatter()
1627 ConstantInt::get(Predicate->getType(), 1), in scalarizeMaskedScatter()
1636 BasicBlock *CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.store"); in scalarizeMaskedScatter()
1646 BasicBlock *NewIfBlock = CondBlock->splitBasicBlock(InsertPt, "else"); in scalarizeMaskedScatter()
1648 Instruction *OldBr = IfBlock->getTerminator(); in scalarizeMaskedScatter()
1650 OldBr->eraseFromParent(); in scalarizeMaskedScatter()
1653 CI->eraseFromParent(); in scalarizeMaskedScatter()
1681 if (match(CountZeros->getOperand(1), m_One())) in despeculateCountZeros()
1685 auto IntrinsicID = CountZeros->getIntrinsicID(); in despeculateCountZeros()
1686 if ((IntrinsicID == Intrinsic::cttz && TLI->isCheapToSpeculateCttz()) || in despeculateCountZeros()
1687 (IntrinsicID == Intrinsic::ctlz && TLI->isCheapToSpeculateCtlz())) in despeculateCountZeros()
1691 Type *Ty = CountZeros->getType(); in despeculateCountZeros()
1692 unsigned SizeInBits = Ty->getPrimitiveSizeInBits(); in despeculateCountZeros()
1693 if (Ty->isVectorTy() || SizeInBits > DL->getLargestLegalIntTypeSizeInBits()) in despeculateCountZeros()
1697 BasicBlock *StartBlock = CountZeros->getParent(); in despeculateCountZeros()
1698 BasicBlock *CallBlock = StartBlock->splitBasicBlock(CountZeros, "cond.false"); in despeculateCountZeros()
1701 // in this block to select the result of the intrinsic or the bit-width in despeculateCountZeros()
1704 BasicBlock *EndBlock = CallBlock->splitBasicBlock(SplitPt, "cond.end"); in despeculateCountZeros()
1707 IRBuilder<> Builder(CountZeros->getContext()); in despeculateCountZeros()
1708 Builder.SetInsertPoint(StartBlock->getTerminator()); in despeculateCountZeros()
1709 Builder.SetCurrentDebugLocation(CountZeros->getDebugLoc()); in despeculateCountZeros()
1714 Value *Cmp = Builder.CreateICmpEQ(CountZeros->getOperand(0), Zero, "cmpz"); in despeculateCountZeros()
1716 StartBlock->getTerminator()->eraseFromParent(); in despeculateCountZeros()
1719 // or the bit width of the operand. in despeculateCountZeros()
1720 Builder.SetInsertPoint(&EndBlock->front()); in despeculateCountZeros()
1722 CountZeros->replaceAllUsesWith(PN); in despeculateCountZeros()
1724 PN->addIncoming(BitWidth, StartBlock); in despeculateCountZeros()
1725 PN->addIncoming(CountZeros, CallBlock); in despeculateCountZeros()
1730 CountZeros->setArgOperand(1, Builder.getTrue()); in despeculateCountZeros()
1736 BasicBlock *BB = CI->getParent(); in optimizeCallInst()
1739 // If we found an inline asm expession, and if the target knows how to in optimizeCallInst()
1741 if (TLI && isa<InlineAsm>(CI->getCalledValue())) { in optimizeCallInst()
1742 if (TLI->ExpandInlineAsm(CI)) { in optimizeCallInst()
1744 CurInstIterator = BB->begin(); in optimizeCallInst()
1758 if (TLI && TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) { in optimizeCallInst()
1759 for (auto &Arg : CI->arg_operands()) { in optimizeCallInst()
1763 // if size - offset meets the size threshold. in optimizeCallInst()
1764 if (!Arg->getType()->isPointerTy()) in optimizeCallInst()
1766 APInt Offset(DL->getPointerSizeInBits( in optimizeCallInst()
1767 cast<PointerType>(Arg->getType())->getAddressSpace()), in optimizeCallInst()
1769 Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*DL, Offset); in optimizeCallInst()
1771 if ((Offset2 & (PrefAlign-1)) != 0) in optimizeCallInst()
1774 if ((AI = dyn_cast<AllocaInst>(Val)) && AI->getAlignment() < PrefAlign && in optimizeCallInst()
1775 DL->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2) in optimizeCallInst()
1776 AI->setAlignment(PrefAlign); in optimizeCallInst()
1779 // over-aligning global variables that have an explicit section is in optimizeCallInst()
1782 if ((GV = dyn_cast<GlobalVariable>(Val)) && GV->canIncreaseAlignment() && in optimizeCallInst()
1783 GV->getAlignment() < PrefAlign && in optimizeCallInst()
1784 DL->getTypeAllocSize(GV->getValueType()) >= in optimizeCallInst()
1786 GV->setAlignment(PrefAlign); in optimizeCallInst()
1791 unsigned Align = getKnownAlignment(MI->getDest(), *DL); in optimizeCallInst()
1793 Align = std::min(Align, getKnownAlignment(MTI->getSource(), *DL)); in optimizeCallInst()
1794 if (Align > MI->getAlignment()) in optimizeCallInst()
1795 MI->setAlignment(ConstantInt::get(MI->getAlignmentType(), Align)); in optimizeCallInst()
1803 if (!OptSize && CI->hasFnAttr(Attribute::Cold)) in optimizeCallInst()
1804 for (auto &Arg : CI->arg_operands()) { in optimizeCallInst()
1805 if (!Arg->getType()->isPointerTy()) in optimizeCallInst()
1807 unsigned AS = Arg->getType()->getPointerAddressSpace(); in optimizeCallInst()
1808 return optimizeMemoryInst(CI, Arg, Arg->getType(), AS); in optimizeCallInst()
1813 switch (II->getIntrinsicID()) { in optimizeCallInst()
1818 Type *ReturnTy = CI->getType(); in optimizeCallInst()
1820 ConstantInt *Op1 = cast<ConstantInt>(II->getArgOperand(1)); in optimizeCallInst()
1821 ObjSizeMode Mode = Op1->isZero() ? ObjSizeMode::Max : ObjSizeMode::Min; in optimizeCallInst()
1822 if (getObjectSize(II->getArgOperand(0), in optimizeCallInst()
1827 Mode == ObjSizeMode::Min ? 0 : -1ULL); in optimizeCallInst()
1840 CurInstIterator = BB->begin(); in optimizeCallInst()
1847 if (!TTI->isLegalMaskedLoad(CI->getType())) { in optimizeCallInst()
1855 if (!TTI->isLegalMaskedStore(CI->getArgOperand(0)->getType())) { in optimizeCallInst()
1863 if (!TTI->isLegalMaskedGather(CI->getType())) { in optimizeCallInst()
1871 if (!TTI->isLegalMaskedScatter(CI->getArgOperand(0)->getType())) { in optimizeCallInst()
1880 ZExtInst *ExtVal = dyn_cast<ZExtInst>(CI->getArgOperand(0)); in optimizeCallInst()
1881 if (!ExtVal || !ExtVal->hasOneUse() || in optimizeCallInst()
1882 ExtVal->getParent() == CI->getParent()) in optimizeCallInst()
1885 ExtVal->moveBefore(CI); in optimizeCallInst()
1892 II->replaceAllUsesWith(II->getArgOperand(0)); in optimizeCallInst()
1893 II->eraseFromParent(); in optimizeCallInst()
1909 if (TLI->GetAddrModeArguments(II, PtrOps, AccessTy, AddrSpace)) in optimizeCallInst()
1917 if (!CI->getCalledFunction()) return false; in optimizeCallInst()
1925 CI->replaceAllUsesWith(V); in optimizeCallInst()
1926 CI->eraseFromParent(); in optimizeCallInst()
1966 ReturnInst *RI = dyn_cast<ReturnInst>(BB->getTerminator()); in dupRetToEnableTailCallOpts()
1972 Value *V = RI->getReturnValue(); in dupRetToEnableTailCallOpts()
1976 V = BCI->getOperand(0); in dupRetToEnableTailCallOpts()
1983 if (PN && PN->getParent() != BB) in dupRetToEnableTailCallOpts()
1988 const Function *F = BB->getParent(); in dupRetToEnableTailCallOpts()
1989 AttributeSet CallerAttrs = F->getAttributes(); in dupRetToEnableTailCallOpts()
1997 BasicBlock::iterator BI = BB->begin(); in dupRetToEnableTailCallOpts()
2005 BasicBlock::iterator BI = BB->begin(); in dupRetToEnableTailCallOpts()
2015 for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I) { in dupRetToEnableTailCallOpts()
2016 CallInst *CI = dyn_cast<CallInst>(PN->getIncomingValue(I)); in dupRetToEnableTailCallOpts()
2018 if (CI && CI->hasOneUse() && CI->getParent() == PN->getIncomingBlock(I) && in dupRetToEnableTailCallOpts()
2019 TLI->mayBeEmittedAsTailCall(CI)) in dupRetToEnableTailCallOpts()
2028 BasicBlock::InstListType &InstList = (*PI)->getInstList(); in dupRetToEnableTailCallOpts()
2036 if (CI && CI->use_empty() && TLI->mayBeEmittedAsTailCall(CI)) in dupRetToEnableTailCallOpts()
2057 BasicBlock *CallBB = CI->getParent(); in dupRetToEnableTailCallOpts()
2058 BranchInst *BI = dyn_cast<BranchInst>(CallBB->getTerminator()); in dupRetToEnableTailCallOpts()
2059 if (!BI || !BI->isUnconditional() || BI->getSuccessor(0) != BB) in dupRetToEnableTailCallOpts()
2069 if (Changed && !BB->hasAddressTaken() && pred_begin(BB) == pred_end(BB)) in dupRetToEnableTailCallOpts()
2070 BB->eraseFromParent(); in dupRetToEnableTailCallOpts()
2075 //===----------------------------------------------------------------------===//
2077 //===----------------------------------------------------------------------===//
2110 BaseGV->printAsOperand(OS, /*PrintType=*/false); in print()
2123 BaseReg->printAsOperand(OS, /*PrintType=*/false); in print()
2129 ScaledReg->printAsOperand(OS, /*PrintType=*/false); in print()
2181 /// - Is the first in a basic block: BB is used.
2182 /// - Has a previous instructon: PrevInst is used.
2193 BasicBlock::iterator It = Inst->getIterator(); in InsertionHandler()
2194 HasPrevInstruction = (It != (Inst->getParent()->begin())); in InsertionHandler()
2196 Point.PrevInst = &*--It; in InsertionHandler()
2198 Point.BB = Inst->getParent(); in InsertionHandler()
2204 if (Inst->getParent()) in insert()
2205 Inst->removeFromParent(); in insert()
2206 Inst->insertAfter(Point.PrevInst); in insert()
2208 Instruction *Position = &*Point.BB->getFirstInsertionPt(); in insert()
2209 if (Inst->getParent()) in insert()
2210 Inst->moveBefore(Position); in insert()
2212 Inst->insertBefore(Position); in insert()
2227 Inst->moveBefore(Before); in InstructionMoveBefore()
2237 /// \brief Set the operand of an instruction with a new value.
2239 /// Original operand of the instruction.
2245 /// \brief Set \p Idx operand of \p Inst with \p NewVal.
2251 Origin = Inst->getOperand(Idx); in OperandSetter()
2252 Inst->setOperand(Idx, NewVal); in OperandSetter()
2260 Inst->setOperand(Idx, Origin); in undo()
2274 unsigned NumOpnds = Inst->getNumOperands(); in OperandsHider()
2277 // Save the current operand. in OperandsHider()
2278 Value *Val = Inst->getOperand(It); in OperandsHider()
2283 Inst->setOperand(It, UndefValue::get(Val->getType())); in OperandsHider()
2291 Inst->setOperand(It, OriginalValues[It]); in undo()
2315 IVal->eraseFromParent(); in undo()
2340 IVal->eraseFromParent(); in undo()
2365 IVal->eraseFromParent(); in undo()
2377 : TypePromotionAction(Inst), OrigTy(Inst->getType()) { in TypeMutator()
2380 Inst->mutateType(NewTy); in TypeMutator()
2387 Inst->mutateType(OrigTy); in undo()
2413 for (Use &U : Inst->uses()) { in UsesReplacer()
2418 Inst->replaceAllUsesWith(New); in UsesReplacer()
2427 UseIt->Inst->setOperand(UseIt->Idx, Inst); in undo()
2445 /// \pre If !Inst->use_empty(), then New != nullptr
2452 Inst->removeFromParent(); in InstructionRemover()
2466 Replacer->undo(); in undo()
2533 Value *Val = Ptr->getBuiltValue(); in createTrunc()
2541 Value *Val = Ptr->getBuiltValue(); in createSExt()
2549 Value *Val = Ptr->getBuiltValue(); in createZExt()
2568 (*It)->commit(); in commit()
2576 Curr->undo(); in rollback()
2582 /// This encapsulates the logic for matching the target-legal addressing modes.
2589 /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and
2617 TLI(*TM.getSubtargetImpl(*MI->getParent()->getParent()) in AddressingModeMatcher()
2618 ->getTargetLowering()), in AddressingModeMatcher()
2619 DL(MI->getModule()->getDataLayout()), AccessTy(AT), AddrSpace(AS), in AddressingModeMatcher()
2682 // Add scale to turn X*4+X*3 -> X*7. This could also do things like in matchScaledValue()
2683 // [A+B + A*7] -> [B+A*8]. in matchScaledValue()
2701 TestAddrMode.BaseOffs += CI->getSExtValue()*TestAddrMode.Scale; in matchScaledValue()
2721 switch (I->getOpcode()) { in MightBeFoldableInst()
2725 if (I->getType() == I->getOperand(0)->getType()) in MightBeFoldableInst()
2727 return I->getType()->isPointerTy() || I->getType()->isIntegerTy(); in MightBeFoldableInst()
2739 return isa<ConstantInt>(I->getOperand(1)); in MightBeFoldableInst()
2750 /// to be legal, as the non-promoted value would have had the same state.
2756 int ISDOpcode = TLI.InstructionOpcodeToISD(PromotedInst->getOpcode()); in isPromotedInstructionLegal()
2762 ISDOpcode, TLI.getValueType(DL, PromotedInst->getType())); in isPromotedInstructionLegal()
2775 /// #2 Operand reuses:
2787 /// \brief Utility function to promote the operand of \p Ext when this
2788 /// operand is a promotable trunc or sext or zext.
2791 /// created to promote the operand of Ext.
2802 /// \brief Utility function to promote the operand of \p Ext when this
2803 /// operand is promotable and is not a supported trunc or sext.
2806 /// created to promote the operand of Ext.
2840 /// Type for the utility function that promotes the operand of Ext.
2848 /// action to promote the operand of \p Ext instead of using Ext.
2868 if (Inst->getType()->isVectorTy()) in canGetThrough()
2883 ((!IsSExt && BinOp->hasNoUnsignedWrap()) || in canGetThrough()
2884 (IsSExt && BinOp->hasNoSignedWrap()))) in canGetThrough()
2888 // ext(trunc(opnd)) --> ext(opnd) in canGetThrough()
2892 Value *OpndVal = Inst->getOperand(0); in canGetThrough()
2893 // Check if we can use this operand in the extension. in canGetThrough()
2895 if (!OpndVal->getType()->isIntegerTy() || in canGetThrough()
2896 OpndVal->getType()->getIntegerBitWidth() > in canGetThrough()
2897 ConsideredExtType->getIntegerBitWidth()) in canGetThrough()
2900 // If the operand of the truncate is not an instruction, we will not have in canGetThrough()
2910 // #1 get the type of the operand and check the kind of the extended bits. in canGetThrough()
2913 if (It != PromotedInsts.end() && It->second.getInt() == IsSExt) in canGetThrough()
2914 OpndType = It->second.getPointer(); in canGetThrough()
2916 OpndType = Opnd->getOperand(0)->getType(); in canGetThrough()
2921 return Inst->getType()->getIntegerBitWidth() >= in canGetThrough()
2922 OpndType->getIntegerBitWidth(); in canGetThrough()
2930 Instruction *ExtOpnd = dyn_cast<Instruction>(Ext->getOperand(0)); in getAction()
2931 Type *ExtTy = Ext->getType(); in getAction()
2933 // If the operand of the extension is not an instruction, we cannot in getAction()
2939 // Do not promote if the operand has been added by codegenprepare. in getAction()
2952 // Abort early if we will have to insert non-free instructions. in getAction()
2953 if (!ExtOpnd->hasOneUse() && !TLI.isTruncateFree(ExtTy, ExtOpnd->getType())) in getAction()
2963 // By construction, the operand of SExt is an instruction. Otherwise we cannot in promoteOperandForTruncAndAnyExt()
2965 Instruction *SExtOpnd = cast<Instruction>(SExt->getOperand(0)); in promoteOperandForTruncAndAnyExt()
2973 TPT.createZExt(SExt, SExtOpnd->getOperand(0), SExt->getType()); in promoteOperandForTruncAndAnyExt()
2980 TPT.setOperand(SExt, 0, SExtOpnd->getOperand(0)); in promoteOperandForTruncAndAnyExt()
2985 if (SExtOpnd->use_empty()) in promoteOperandForTruncAndAnyExt()
2990 if (!ExtInst || ExtInst->getType() != ExtInst->getOperand(0)->getType()) { in promoteOperandForTruncAndAnyExt()
2993 Exts->push_back(ExtInst); in promoteOperandForTruncAndAnyExt()
3001 Value *NextVal = ExtInst->getOperand(0); in promoteOperandForTruncAndAnyExt()
3012 // By construction, the operand of Ext is an instruction. Otherwise we cannot in promoteOperandForOther()
3014 Instruction *ExtOpnd = cast<Instruction>(Ext->getOperand(0)); in promoteOperandForOther()
3016 if (!ExtOpnd->hasOneUse()) { in promoteOperandForOther()
3021 Value *Trunc = TPT.createTrunc(Ext, ExtOpnd->getType()); in promoteOperandForOther()
3023 ITrunc->removeFromParent(); in promoteOperandForOther()
3025 ITrunc->insertAfter(ExtOpnd); in promoteOperandForOther()
3027 Truncs->push_back(ITrunc); in promoteOperandForOther()
3031 // Restore the operand of Ext (which has been replaced by the previous call in promoteOperandForOther()
3032 // to replaceAllUsesWith) to avoid creating a cycle trunc <-> sext. in promoteOperandForOther()
3039 // 3. Extend each operand that needs to be extended. in promoteOperandForOther()
3044 ExtOpnd, TypeIsSExt(ExtOpnd->getType(), IsSExt))); in promoteOperandForOther()
3046 TPT.mutateType(ExtOpnd, Ext->getType()); in promoteOperandForOther()
3053 for (int OpIdx = 0, EndOpIdx = ExtOpnd->getNumOperands(); OpIdx != EndOpIdx; in promoteOperandForOther()
3055 DEBUG(dbgs() << "Operand:\n" << *(ExtOpnd->getOperand(OpIdx)) << '\n'); in promoteOperandForOther()
3056 if (ExtOpnd->getOperand(OpIdx)->getType() == Ext->getType() || in promoteOperandForOther()
3061 // Check if we can statically extend the operand. in promoteOperandForOther()
3062 Value *Opnd = ExtOpnd->getOperand(OpIdx); in promoteOperandForOther()
3065 unsigned BitWidth = Ext->getType()->getIntegerBitWidth(); in promoteOperandForOther()
3066 APInt CstVal = IsSExt ? Cst->getValue().sext(BitWidth) in promoteOperandForOther()
3067 : Cst->getValue().zext(BitWidth); in promoteOperandForOther()
3068 TPT.setOperand(ExtOpnd, OpIdx, ConstantInt::get(Ext->getType(), CstVal)); in promoteOperandForOther()
3074 TPT.setOperand(ExtOpnd, OpIdx, UndefValue::get(Ext->getType())); in promoteOperandForOther()
3078 // Otherwise we have to explicity sign extend the operand. in promoteOperandForOther()
3079 // Check if Ext was reused to extend an operand. in promoteOperandForOther()
3083 Value *ValForExtOpnd = IsSExt ? TPT.createSExt(Ext, Opnd, Ext->getType()) in promoteOperandForOther()
3084 : TPT.createZExt(Ext, Opnd, Ext->getType()); in promoteOperandForOther()
3092 Exts->push_back(ExtForOpnd); in promoteOperandForOther()
3157 return matchAddr(AddrInst->getOperand(0), Depth); in matchOperationAddr()
3159 auto AS = AddrInst->getType()->getPointerAddressSpace(); in matchOperationAddr()
3161 // This inttoptr is a no-op if the integer type is pointer sized. in matchOperationAddr()
3162 if (TLI.getValueType(DL, AddrInst->getOperand(0)->getType()) == PtrTy) in matchOperationAddr()
3163 return matchAddr(AddrInst->getOperand(0), Depth); in matchOperationAddr()
3168 // int->int or pointer->pointer (we don't want int<->fp or something). in matchOperationAddr()
3169 if ((AddrInst->getOperand(0)->getType()->isPointerTy() || in matchOperationAddr()
3170 AddrInst->getOperand(0)->getType()->isIntegerTy()) && in matchOperationAddr()
3174 AddrInst->getOperand(0)->getType() != AddrInst->getType()) in matchOperationAddr()
3175 return matchAddr(AddrInst->getOperand(0), Depth); in matchOperationAddr()
3179 = AddrInst->getOperand(0)->getType()->getPointerAddressSpace(); in matchOperationAddr()
3180 unsigned DestAS = AddrInst->getType()->getPointerAddressSpace(); in matchOperationAddr()
3182 return matchAddr(AddrInst->getOperand(0), Depth); in matchOperationAddr()
3196 if (matchAddr(AddrInst->getOperand(1), Depth+1) && in matchOperationAddr()
3197 matchAddr(AddrInst->getOperand(0), Depth+1)) in matchOperationAddr()
3205 // Otherwise this was over-aggressive. Try merging in the LHS then the RHS. in matchOperationAddr()
3206 if (matchAddr(AddrInst->getOperand(0), Depth+1) && in matchOperationAddr()
3207 matchAddr(AddrInst->getOperand(1), Depth+1)) in matchOperationAddr()
3222 ConstantInt *RHS = dyn_cast<ConstantInt>(AddrInst->getOperand(1)); in matchOperationAddr()
3225 int64_t Scale = RHS->getSExtValue(); in matchOperationAddr()
3229 return matchScaledValue(AddrInst->getOperand(0), Scale, Depth); in matchOperationAddr()
3234 int VariableOperand = -1; in matchOperationAddr()
3239 for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) { in matchOperationAddr()
3243 cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue(); in matchOperationAddr()
3244 ConstantOffset += SL->getElementOffset(Idx); in matchOperationAddr()
3247 if (ConstantInt *CI = dyn_cast<ConstantInt>(AddrInst->getOperand(i))) { in matchOperationAddr()
3248 ConstantOffset += CI->getSExtValue()*TypeSize; in matchOperationAddr()
3251 if (VariableOperand != -1) in matchOperationAddr()
3263 if (VariableOperand == -1) { in matchOperationAddr()
3268 if (matchAddr(AddrInst->getOperand(0), Depth+1)) in matchOperationAddr()
3271 AddrMode.BaseOffs -= ConstantOffset; in matchOperationAddr()
3282 // Match the base operand of the GEP. in matchOperationAddr()
3283 if (!matchAddr(AddrInst->getOperand(0), Depth+1)) { in matchOperationAddr()
3291 AddrMode.BaseReg = AddrInst->getOperand(0); in matchOperationAddr()
3295 if (!matchScaledValue(AddrInst->getOperand(VariableOperand), VariableScale, in matchOperationAddr()
3304 AddrMode.BaseReg = AddrInst->getOperand(0); in matchOperationAddr()
3306 if (!matchScaledValue(AddrInst->getOperand(VariableOperand), in matchOperationAddr()
3344 // promotedOpnd = ext opnd <- no match here in matchOperationAddr()
3345 // op = promoted_add promotedOpnd, 1 <- match (later in recursive calls) in matchOperationAddr()
3346 // addr = gep base, op <- match in matchOperationAddr()
3362 ExtCost + (AddrModeInsts.size() - OldSize), in matchOperationAddr()
3388 AddrMode.BaseOffs += CI->getSExtValue(); in matchAddr()
3391 AddrMode.BaseOffs -= CI->getSExtValue(); in matchAddr()
3406 if (matchOperationAddr(I, I->getOpcode(), Depth, &MovedAway)) { in matchAddr()
3414 if (I->hasOneUse() || in matchAddr()
3427 if (matchOperationAddr(CE, CE->getOpcode(), Depth)) in matchAddr()
3460 /// Check to see if all uses of OpVal by the specified inline asm call are due
3464 const Function *F = CI->getParent()->getParent(); in IsOperandAMemoryOperand()
3465 const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering(); in IsOperandAMemoryOperand()
3466 const TargetRegisterInfo *TRI = TM.getSubtargetImpl(*F)->getRegisterInfo(); in IsOperandAMemoryOperand()
3468 TLI->ParseConstraints(F->getParent()->getDataLayout(), TRI, in IsOperandAMemoryOperand()
3474 TLI->ComputeConstraintToUse(OpInfo, SDValue()); in IsOperandAMemoryOperand()
3476 // If this asm operand is our Value*, and if it isn't an indirect memory in IsOperandAMemoryOperand()
3477 // operand, we can't fold it! in IsOperandAMemoryOperand()
3488 /// If we find an obviously non-foldable instruction, return true.
3502 const bool OptSize = I->getFunction()->optForSize(); in FindAllMemoryUses()
3505 for (Use &U : I->uses()) { in FindAllMemoryUses()
3523 if (!OptSize && CI->hasFnAttr(Attribute::Cold)) in FindAllMemoryUses()
3526 InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledValue()); in FindAllMemoryUses()
3529 // If this is a memory operand, we're cool, otherwise bail out. in FindAllMemoryUses()
3548 // If Val is either of the known-live values, we know it is live! in valueAlreadyLiveAtInst()
3559 if (AI->isStaticAlloca()) in valueAlreadyLiveAtInst()
3565 return Val->isUsedInBasicBlock(MemoryInst->getParent()); in valueAlreadyLiveAtInst()
3576 /// use(Y) -> nonload/store
3624 return false; // Has a non-memory, non-foldable use! in isProfitableToFoldIntoAddressingMode()
3642 Value *Address = User->getOperand(OpNo); in isProfitableToFoldIntoAddressingMode()
3643 PointerType *AddrTy = dyn_cast<PointerType>(Address->getType()); in isProfitableToFoldIntoAddressingMode()
3646 Type *AddressAccessTy = AddrTy->getElementType(); in isProfitableToFoldIntoAddressingMode()
3647 unsigned AS = AddrTy->getAddressSpace(); in isProfitableToFoldIntoAddressingMode()
3684 return I->getParent() != BB; in IsNonLocalValue()
3711 // Try to collapse single-value PHI nodes. This is necessary to undo in optimizeMemoryInst()
3718 // the addressing mode obtained from the non-PHI roots of the graph in optimizeMemoryInst()
3732 // Break use-def graph loops. in optimizeMemoryInst()
3740 for (Value *IncValue : P->incoming_values()) in optimizeMemoryInst()
3745 // For non-PHIs, determine the addressing mode being computed. Note that in optimizeMemoryInst()
3755 // calling getNumUses() unconditionally caused a significant compile-time in optimizeMemoryInst()
3764 NumUsesConsensus = Consensus->getNumUses(); in optimizeMemoryInst()
3773 unsigned NumUses = V->getNumUses(); in optimizeMemoryInst()
3795 // non-local to I's BB. in optimizeMemoryInst()
3798 if (IsNonLocalValue(AddrModeInsts[i], MemoryInst->getParent())) { in optimizeMemoryInst()
3823 if (SunkAddr->getType() != Addr->getType()) in optimizeMemoryInst()
3824 SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->getType()); in optimizeMemoryInst()
3827 TM->getSubtargetImpl(*MemoryInst->getParent()->getParent()) in optimizeMemoryInst()
3828 ->useAA())) { in optimizeMemoryInst()
3829 // By default, we use the GEP-based method when AA is used later. This in optimizeMemoryInst()
3833 Type *IntPtrTy = DL->getIntPtrType(Addr->getType()); in optimizeMemoryInst()
3837 if (AddrMode.BaseReg && AddrMode.BaseReg->getType()->isPointerTy()) { in optimizeMemoryInst()
3842 if (AddrMode.Scale && AddrMode.ScaledReg->getType()->isPointerTy()) { in optimizeMemoryInst()
3864 Builder.CreateIntToPtr(AddrMode.BaseReg, Addr->getType(), "sunkaddr"); in optimizeMemoryInst()
3868 Builder.CreateIntToPtr(AddrMode.ScaledReg, Addr->getType(), "sunkaddr"); in optimizeMemoryInst()
3874 SunkAddr = Constant::getNullValue(Addr->getType()); in optimizeMemoryInst()
3879 Builder.getInt8PtrTy(Addr->getType()->getPointerAddressSpace()); in optimizeMemoryInst()
3889 if (V->getType() != IntPtrTy) in optimizeMemoryInst()
3898 if (V->getType() == IntPtrTy) { in optimizeMemoryInst()
3900 } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() < in optimizeMemoryInst()
3901 cast<IntegerType>(V->getType())->getBitWidth()) { in optimizeMemoryInst()
3907 // the AddrMode was created we need to bail out gracefully if widths in optimizeMemoryInst()
3911 I->eraseFromParent(); in optimizeMemoryInst()
3930 if (ResultPtr->getType() != I8PtrTy) in optimizeMemoryInst()
3941 if (ResultPtr->getType() != I8PtrTy) in optimizeMemoryInst()
3946 if (SunkAddr->getType() != Addr->getType()) in optimizeMemoryInst()
3947 SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->getType()); in optimizeMemoryInst()
3952 Type *IntPtrTy = DL->getIntPtrType(Addr->getType()); in optimizeMemoryInst()
3962 if (V->getType()->isPointerTy()) in optimizeMemoryInst()
3964 if (V->getType() != IntPtrTy) in optimizeMemoryInst()
3972 if (V->getType() == IntPtrTy) { in optimizeMemoryInst()
3974 } else if (V->getType()->isPointerTy()) { in optimizeMemoryInst()
3976 } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() < in optimizeMemoryInst()
3977 cast<IntegerType>(V->getType())->getBitWidth()) { in optimizeMemoryInst()
3983 // the AddrMode was created we need to bail out gracefully if widths in optimizeMemoryInst()
3987 I->eraseFromParent(); in optimizeMemoryInst()
4018 SunkAddr = Constant::getNullValue(Addr->getType()); in optimizeMemoryInst()
4020 SunkAddr = Builder.CreateIntToPtr(Result, Addr->getType(), "sunkaddr"); in optimizeMemoryInst()
4023 MemoryInst->replaceUsesOfWith(Repl, SunkAddr); in optimizeMemoryInst()
4027 if (Repl->use_empty()) { in optimizeMemoryInst()
4032 BasicBlock *BB = CurInstIterator->getParent(); in optimizeMemoryInst()
4039 CurInstIterator = BB->begin(); in optimizeMemoryInst()
4053 TM->getSubtargetImpl(*CS->getParent()->getParent())->getRegisterInfo(); in optimizeInlineAsmInst()
4055 TLI->ParseConstraints(*DL, TRI, CS); in optimizeInlineAsmInst()
4061 TLI->ComputeConstraintToUse(OpInfo, SDValue()); in optimizeInlineAsmInst()
4065 Value *OpVal = CS->getArgOperand(ArgNo++); in optimizeInlineAsmInst()
4066 MadeChange |= optimizeMemoryInst(CS, OpVal, OpVal->getType(), ~0u); in optimizeInlineAsmInst()
4077 assert(!Inst->use_empty() && "Input must have at least one use"); in hasSameExtUse()
4078 const Instruction *FirstUser = cast<Instruction>(*Inst->user_begin()); in hasSameExtUse()
4080 Type *ExtTy = FirstUser->getType(); in hasSameExtUse()
4081 for (const User *U : Inst->users()) { in hasSameExtUse()
4085 Type *CurTy = UI->getType(); in hasSameExtUse()
4106 if (ExtTy->getScalarType()->getIntegerBitWidth() > in hasSameExtUse()
4107 CurTy->getScalarType()->getIntegerBitWidth()) { in hasSameExtUse()
4153 if ((LI = dyn_cast<LoadInst>(I->getOperand(0)))) { in extLdPromotion()
4159 if (!TLI || !TLI->enableExtLdPromotion() || DisableExtLdPromotion) in extLdPromotion()
4172 unsigned ExtCost = !TLI->isExtFree(I); in extLdPromotion()
4186 TotalCreatedInstsCost -= ExtCost; in extLdPromotion()
4201 (LI->hasOneUse() || hasSameExtUse(LI, *TLI)))) in extLdPromotion()
4241 if (!HasPromoted && LI->getParent() == I->getParent()) in moveExtToFormExtLoad()
4244 EVT VT = TLI->getValueType(*DL, I->getType()); in moveExtToFormExtLoad()
4245 EVT LoadVT = TLI->getValueType(*DL, LI->getType()); in moveExtToFormExtLoad()
4249 if (!LI->hasOneUse() && TLI && in moveExtToFormExtLoad()
4250 (TLI->isTypeLegal(LoadVT) || !TLI->isTypeLegal(VT)) && in moveExtToFormExtLoad()
4251 !TLI->isTruncateFree(I->getType(), LI->getType())) { in moveExtToFormExtLoad()
4265 if (TLI && !TLI->isLoadExtLegal(LType, VT, LoadVT)) { in moveExtToFormExtLoad()
4274 I->removeFromParent(); in moveExtToFormExtLoad()
4275 I->insertAfter(LI); in moveExtToFormExtLoad()
4281 BasicBlock *DefBB = I->getParent(); in optimizeExtUses()
4285 Value *Src = I->getOperand(0); in optimizeExtUses()
4286 if (Src->hasOneUse()) in optimizeExtUses()
4290 if (TLI && !TLI->isTruncateFree(I->getType(), Src->getType())) in optimizeExtUses()
4295 if (!isa<Instruction>(Src) || DefBB != cast<Instruction>(Src)->getParent()) in optimizeExtUses()
4299 for (User *U : I->users()) { in optimizeExtUses()
4303 BasicBlock *UserBB = UI->getParent(); in optimizeExtUses()
4312 for (User *U : Src->users()) { in optimizeExtUses()
4314 BasicBlock *UserBB = UI->getParent(); in optimizeExtUses()
4322 // InsertedTruncs - Only insert one trunc in each block once. in optimizeExtUses()
4326 for (Use &U : Src->uses()) { in optimizeExtUses()
4330 BasicBlock *UserBB = User->getParent(); in optimizeExtUses()
4337 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); in optimizeExtUses()
4338 assert(InsertPt != UserBB->end()); in optimizeExtUses()
4339 InsertedTrunc = new TruncInst(I, Src->getType(), "", &*InsertPt); in optimizeExtUses()
4407 if (!Load->isSimple() || in optimizeLoadExt()
4408 !(Load->getType()->isIntegerTy() || Load->getType()->isPointerTy())) in optimizeLoadExt()
4412 if (Load->hasOneUse()) { in optimizeLoadExt()
4413 User *LoadUser = *Load->user_begin(); in optimizeLoadExt()
4414 if (cast<Instruction>(LoadUser)->getParent() == Load->getParent() && in optimizeLoadExt()
4424 for (auto *U : Load->users()) in optimizeLoadExt()
4427 EVT LoadResultVT = TLI->getValueType(*DL, Load->getType()); in optimizeLoadExt()
4436 // Break use-def graph loops. in optimizeLoadExt()
4442 for (auto *U : Phi->users()) in optimizeLoadExt()
4447 switch (I->getOpcode()) { in optimizeLoadExt()
4449 auto *AndC = dyn_cast<ConstantInt>(I->getOperand(1)); in optimizeLoadExt()
4452 APInt AndBits = AndC->getValue(); in optimizeLoadExt()
4457 if (AndBits == WidestAndBits && I->getOperand(0) == Load) in optimizeLoadExt()
4463 auto *ShlC = dyn_cast<ConstantInt>(I->getOperand(1)); in optimizeLoadExt()
4466 uint64_t ShiftAmt = ShlC->getLimitedValue(BitWidth - 1); in optimizeLoadExt()
4473 EVT TruncVT = TLI->getValueType(*DL, I->getType()); in optimizeLoadExt()
4501 LLVMContext &Ctx = Load->getType()->getContext(); in optimizeLoadExt()
4503 EVT TruncVT = TLI->getValueType(*DL, TruncTy); in optimizeLoadExt()
4507 !TLI->isLoadExtLegal(ISD::ZEXTLOAD, LoadResultVT, TruncVT)) in optimizeLoadExt()
4510 IRBuilder<> Builder(Load->getNextNode()); in optimizeLoadExt()
4516 Load->replaceAllUsesWith(NewAnd); in optimizeLoadExt()
4517 NewAnd->setOperand(0, Load); in optimizeLoadExt()
4523 if (cast<ConstantInt>(And->getOperand(1))->getValue() == DemandBits) { in optimizeLoadExt()
4524 And->replaceAllUsesWith(NewAnd); in optimizeLoadExt()
4526 CurInstIterator = std::next(And->getIterator()); in optimizeLoadExt()
4527 And->eraseFromParent(); in optimizeLoadExt()
4535 /// Check if V (an operand of a select instruction) is an expensive instruction
4541 return I && I->hasOneUse() && isSafeToSpeculativelyExecute(I) && in sinkSelectOperand()
4542 TTI->getUserCost(I) >= TargetTransformInfo::TCC_Expensive; in sinkSelectOperand()
4550 if (!TLI->isPredictableSelectExpensive()) in isFormingBranchFromSelectProfitable()
4559 if (SI->extractProfMetadata(TrueWeight, FalseWeight)) { in isFormingBranchFromSelectProfitable()
4564 if (Probability > TLI->getPredictableBranchThreshold()) in isFormingBranchFromSelectProfitable()
4569 CmpInst *Cmp = dyn_cast<CmpInst>(SI->getCondition()); in isFormingBranchFromSelectProfitable()
4571 // If a branch is predictable, an out-of-order CPU can avoid blocking on its in isFormingBranchFromSelectProfitable()
4574 if (!Cmp || !Cmp->hasOneUse()) in isFormingBranchFromSelectProfitable()
4577 // If either operand of the select is expensive and only needed on one side in isFormingBranchFromSelectProfitable()
4579 if (sinkSelectOperand(TTI, SI->getTrueValue()) || in isFormingBranchFromSelectProfitable()
4580 sinkSelectOperand(TTI, SI->getFalseValue())) in isFormingBranchFromSelectProfitable()
4590 bool VectorCond = !SI->getCondition()->getType()->isIntegerTy(1); in optimizeSelectInst()
4594 SI->getMetadata(LLVMContext::MD_unpredictable)) in optimizeSelectInst()
4600 else if (SI->getType()->isVectorTy()) in optimizeSelectInst()
4605 if (TLI->isSelectSupported(SelectKind) && in optimizeSelectInst()
4635 BasicBlock *StartBlock = SI->getParent(); in optimizeSelectInst()
4637 BasicBlock *EndBlock = StartBlock->splitBasicBlock(SplitPt, "select.end"); in optimizeSelectInst()
4640 StartBlock->getTerminator()->eraseFromParent(); in optimizeSelectInst()
4649 if (sinkSelectOperand(TTI, SI->getTrueValue())) { in optimizeSelectInst()
4650 TrueBlock = BasicBlock::Create(SI->getContext(), "select.true.sink", in optimizeSelectInst()
4651 EndBlock->getParent(), EndBlock); in optimizeSelectInst()
4653 auto *TrueInst = cast<Instruction>(SI->getTrueValue()); in optimizeSelectInst()
4654 TrueInst->moveBefore(TrueBranch); in optimizeSelectInst()
4656 if (sinkSelectOperand(TTI, SI->getFalseValue())) { in optimizeSelectInst()
4657 FalseBlock = BasicBlock::Create(SI->getContext(), "select.false.sink", in optimizeSelectInst()
4658 EndBlock->getParent(), EndBlock); in optimizeSelectInst()
4660 auto *FalseInst = cast<Instruction>(SI->getFalseValue()); in optimizeSelectInst()
4661 FalseInst->moveBefore(FalseBranch); in optimizeSelectInst()
4670 FalseBlock = BasicBlock::Create(SI->getContext(), "select.false", in optimizeSelectInst()
4671 EndBlock->getParent(), EndBlock); in optimizeSelectInst()
4681 BranchInst::Create(EndBlock, FalseBlock, SI->getCondition(), SI); in optimizeSelectInst()
4684 BranchInst::Create(TrueBlock, EndBlock, SI->getCondition(), SI); in optimizeSelectInst()
4687 BranchInst::Create(TrueBlock, FalseBlock, SI->getCondition(), SI); in optimizeSelectInst()
4691 PHINode *PN = PHINode::Create(SI->getType(), 2, "", &EndBlock->front()); in optimizeSelectInst()
4692 PN->takeName(SI); in optimizeSelectInst()
4693 PN->addIncoming(SI->getTrueValue(), TrueBlock); in optimizeSelectInst()
4694 PN->addIncoming(SI->getFalseValue(), FalseBlock); in optimizeSelectInst()
4696 SI->replaceAllUsesWith(PN); in optimizeSelectInst()
4697 SI->eraseFromParent(); in optimizeSelectInst()
4700 CurInstIterator = StartBlock->end(); in optimizeSelectInst()
4706 SmallVector<int, 16> Mask(SVI->getShuffleMask()); in isBroadcastShuffle()
4707 int SplatElem = -1; in isBroadcastShuffle()
4709 if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem) in isBroadcastShuffle()
4722 BasicBlock *DefBB = SVI->getParent(); in optimizeShuffleVectorInst()
4725 if (!TLI || !TLI->isVectorShiftByScalarCheap(SVI->getType())) in optimizeShuffleVectorInst()
4733 // InsertedShuffles - Only insert a shuffle in each block once. in optimizeShuffleVectorInst()
4737 for (User *U : SVI->users()) { in optimizeShuffleVectorInst()
4741 BasicBlock *UserBB = UI->getParent(); in optimizeShuffleVectorInst()
4745 if (!UI->isShift()) continue; in optimizeShuffleVectorInst()
4752 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); in optimizeShuffleVectorInst()
4753 assert(InsertPt != UserBB->end()); in optimizeShuffleVectorInst()
4755 new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1), in optimizeShuffleVectorInst()
4756 SVI->getOperand(2), "", &*InsertPt); in optimizeShuffleVectorInst()
4759 UI->replaceUsesOfWith(SVI, InsertedShuffle); in optimizeShuffleVectorInst()
4764 if (SVI->use_empty()) { in optimizeShuffleVectorInst()
4765 SVI->eraseFromParent(); in optimizeShuffleVectorInst()
4776 Value *Cond = SI->getCondition(); in optimizeSwitchInst()
4777 Type *OldType = Cond->getType(); in optimizeSwitchInst()
4778 LLVMContext &Context = Cond->getContext(); in optimizeSwitchInst()
4779 MVT RegType = TLI->getRegisterType(Context, TLI->getValueType(*DL, OldType)); in optimizeSwitchInst()
4782 if (RegWidth <= cast<IntegerType>(OldType)->getBitWidth()) in optimizeSwitchInst()
4789 // preferred register width, so we will potentially eliminate N-1 extends, in optimizeSwitchInst()
4793 // Zero-extend the switch condition and case constants unless the switch in optimizeSwitchInst()
4794 // condition is a function argument that is already being sign-extended. in optimizeSwitchInst()
4795 // In that case, we can avoid an unnecessary mask/extension by sign-extending in optimizeSwitchInst()
4799 if (Arg->hasSExtAttr()) in optimizeSwitchInst()
4803 ExtInst->insertBefore(SI); in optimizeSwitchInst()
4804 SI->setCondition(ExtInst); in optimizeSwitchInst()
4805 for (SwitchInst::CaseIt Case : SI->cases()) { in optimizeSwitchInst()
4806 APInt NarrowConst = Case.getCaseValue()->getValue(); in optimizeSwitchInst()
4882 return Transition->getOperand(getTransitionOriginalValueIdx())->getType(); in getTransitionType()
4897 Value *ValIdx = Transition->getOperand(getTransitionOriginalValueIdx()); in isProfitableToPromote()
4899 ? cast<ConstantInt>(ValIdx)->getZExtValue() in isProfitableToPromote()
4900 : -1; in isProfitableToPromote()
4904 unsigned AS = ST->getPointerAddressSpace(); in isProfitableToPromote()
4905 unsigned Align = ST->getAlignment(); in isProfitableToPromote()
4908 TLI.getValueType(DL, ST->getValueOperand()->getType()), AS, in isProfitableToPromote()
4919 TTI.getVectorInstrCost(Transition->getOpcode(), PromotedType, Index); in isProfitableToPromote()
4926 Value *Arg0 = Inst->getOperand(0); in isProfitableToPromote()
4936 Inst->getOpcode(), Inst->getType(), Arg0OVK, Arg1OVK); in isProfitableToPromote()
4937 VectorCost += TTI.getArithmeticInstrCost(Inst->getOpcode(), PromotedType, in isProfitableToPromote()
4958 Value *ValExtractIdx = Transition->getOperand(getTransitionIdx()); in getConstantVector()
4960 ExtractIdx = CstVal->getSExtValue(); in getConstantVector()
4965 unsigned End = getTransitionType()->getVectorNumElements(); in getConstantVector()
4970 UndefValue *UndefVal = UndefValue::get(Val->getType()); in getConstantVector()
4980 /// \brief Check if promoting to a vector type an operand at \p OperandIdx
4984 // This is not safe to introduce undef when the operand is on in canCauseUndefinedBehavior()
4985 // the right hand side of a division-like instruction. in canCauseUndefinedBehavior()
4988 switch (Use->getOpcode()) { in canCauseUndefinedBehavior()
4998 return !Use->hasNoNaNs(); in canCauseUndefinedBehavior()
5023 for (const Use &U : ToBePromoted->operands()) { in shouldPromote()
5038 int ISDOpcode = TLI.InstructionOpcodeToISD(ToBePromoted->getOpcode()); in shouldPromote()
5094 assert(ToBePromoted->getType() == Transition->getType() && in promoteImpl()
5097 ToBePromoted->replaceAllUsesWith(Transition); in promoteImpl()
5101 ToBePromoted->mutateType(TransitionTy); in promoteImpl()
5105 for (Use &U : ToBePromoted->operands()) { in promoteImpl()
5109 NewVal = Transition->getOperand(getTransitionOriginalValueIdx()); in promoteImpl()
5120 ToBePromoted->setOperand(U.getOperandNo(), NewVal); in promoteImpl()
5122 Transition->removeFromParent(); in promoteImpl()
5123 Transition->insertAfter(ToBePromoted); in promoteImpl()
5124 Transition->setOperand(getTransitionOriginalValueIdx(), ToBePromoted); in promoteImpl()
5134 !TLI->canCombineStoreAndExtract(Inst->getOperand(0)->getType(), in optimizeExtractElementInst()
5135 Inst->getOperand(1), CombineCost))) in optimizeExtractElementInst()
5139 // Try to move it down the def-use chain, until: in optimizeExtractElementInst()
5140 // - We can combine the transition with its single use in optimizeExtractElementInst()
5142 // - We escape the current basic block in optimizeExtractElementInst()
5145 BasicBlock *Parent = Inst->getParent(); in optimizeExtractElementInst()
5150 while (Inst->hasOneUse()) { in optimizeExtractElementInst()
5151 Instruction *ToBePromoted = cast<Instruction>(*Inst->user_begin()); in optimizeExtractElementInst()
5154 if (ToBePromoted->getParent() != Parent) { in optimizeExtractElementInst()
5156 << ToBePromoted->getParent()->getName() in optimizeExtractElementInst()
5157 << ") than the transition (" << Parent->getName() << ").\n"); in optimizeExtractElementInst()
5193 P->replaceAllUsesWith(V); in optimizeInst()
5194 P->eraseFromParent(); in optimizeInst()
5207 // want to forward-subst the cast. in optimizeInst()
5208 if (isa<Constant>(CI->getOperand(0))) in optimizeInst()
5218 TLI->getTypeAction(CI->getContext(), in optimizeInst()
5219 TLI->getValueType(*DL, CI->getType())) == in optimizeInst()
5231 if (!TLI || !TLI->hasMultipleConditionRegisters()) in optimizeInst()
5238 unsigned AS = LI->getPointerAddressSpace(); in optimizeInst()
5239 Modified |= optimizeMemoryInst(I, I->getOperand(0), LI->getType(), AS); in optimizeInst()
5248 unsigned AS = SI->getPointerAddressSpace(); in optimizeInst()
5249 return optimizeMemoryInst(I, SI->getOperand(1), in optimizeInst()
5250 SI->getOperand(0)->getType(), AS); in optimizeInst()
5257 if (BinOp && (BinOp->getOpcode() == Instruction::AShr || in optimizeInst()
5258 BinOp->getOpcode() == Instruction::LShr)) { in optimizeInst()
5259 ConstantInt *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1)); in optimizeInst()
5260 if (TLI && CI && TLI->hasExtractBitsInsn()) in optimizeInst()
5267 if (GEPI->hasAllZeroIndices()) { in optimizeInst()
5268 /// The GEP operand must be a pointer, so must its result -> BitCast in optimizeInst()
5269 Instruction *NC = new BitCastInst(GEPI->getOperand(0), GEPI->getType(), in optimizeInst()
5270 GEPI->getName(), GEPI); in optimizeInst()
5271 GEPI->replaceAllUsesWith(NC); in optimizeInst()
5272 GEPI->eraseFromParent(); in optimizeInst()
5302 if (!I.getType()->isIntegerTy() || in makeBitReverse()
5317 // across basic blocks and rewrite them to improve basic-block-at-a-time
5361 if (!DVI || (DVI->getValue() && isa<AllocaInst>(DVI->getValue()))) { in placeDbgValues()
5366 Instruction *VI = dyn_cast_or_null<Instruction>(DVI->getValue()); in placeDbgValues()
5367 if (VI && VI != PrevNonDbgInst && !VI->isTerminator()) { in placeDbgValues()
5370 if (isa<PHINode>(VI) && VI->getParent()->getTerminator()->isEHPad()) in placeDbgValues()
5373 DVI->removeFromParent(); in placeDbgValues()
5375 DVI->insertBefore(&*VI->getParent()->getFirstInsertionPt()); in placeDbgValues()
5377 DVI->insertAfter(VI); in placeDbgValues()
5389 // mask and compare into the branch uses. Do this before OptimizeBlock ->
5390 // OptimizeInst -> OptimizeCmpExpression, which perturbs the pattern being
5395 if (!TLI || !TLI->isMaskAndBranchFoldingLegal()) in sinkAndCmp()
5400 // %andVal = and %val, #single-bit-set in sinkAndCmp()
5404 if (!Brcc || !Brcc->isConditional()) in sinkAndCmp()
5406 ICmpInst *Cmp = dyn_cast<ICmpInst>(Brcc->getOperand(0)); in sinkAndCmp()
5407 if (!Cmp || Cmp->getParent() != &BB) in sinkAndCmp()
5409 ConstantInt *Zero = dyn_cast<ConstantInt>(Cmp->getOperand(1)); in sinkAndCmp()
5410 if (!Zero || !Zero->isZero()) in sinkAndCmp()
5412 Instruction *And = dyn_cast<Instruction>(Cmp->getOperand(0)); in sinkAndCmp()
5413 if (!And || And->getOpcode() != Instruction::And || And->getParent() != &BB) in sinkAndCmp()
5415 ConstantInt* Mask = dyn_cast<ConstantInt>(And->getOperand(1)); in sinkAndCmp()
5416 if (!Mask || !Mask->getUniqueInteger().isPowerOf2()) in sinkAndCmp()
5423 for (Use &TheUse : Cmp->uses()) { in sinkAndCmp()
5426 if (!BrccUser || !BrccUser->isConditional()) in sinkAndCmp()
5428 BasicBlock *UserBB = BrccUser->getParent(); in sinkAndCmp()
5435 BinaryOperator::CreateAnd(And->getOperand(0), And->getOperand(1), "", in sinkAndCmp()
5438 CmpInst::Create(Cmp->getOpcode(), Cmp->getPredicate(), NewAnd, Zero, in sinkAndCmp()
5442 DEBUG(BrccUser->getParent()->dump()); in sinkAndCmp()
5479 if (!TM || !TM->Options.EnableFastISel || !TLI || TLI->isJumpExpensive()) in splitBranchCondition()
5495 if (Br1->getMetadata(LLVMContext::MD_unpredictable)) in splitBranchCondition()
5522 Br1->setCondition(Cond1); in splitBranchCondition()
5523 LogicOp->eraseFromParent(); in splitBranchCondition()
5528 Br1->setSuccessor(0, TmpBB); in splitBranchCondition()
5530 Br1->setSuccessor(1, TmpBB); in splitBranchCondition()
5535 I->removeFromParent(); in splitBranchCondition()
5536 I->insertBefore(Br2); in splitBranchCondition()
5557 while ((i = PN->getBasicBlockIndex(&BB)) >= 0) in splitBranchCondition()
5558 PN->setIncomingBlock(i, TmpBB); in splitBranchCondition()
5566 auto *Val = PN->getIncomingValueForBlock(&BB); in splitBranchCondition()
5567 PN->addIncoming(Val, TmpBB); in splitBranchCondition()
5593 if (Br1->extractProfMetadata(TrueWeight, FalseWeight)) { in splitBranchCondition()
5597 Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext()) in splitBranchCondition()
5603 Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext()) in splitBranchCondition()
5626 if (Br1->extractProfMetadata(TrueWeight, FalseWeight)) { in splitBranchCondition()
5630 Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext()) in splitBranchCondition()
5636 Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext()) in splitBranchCondition()
5648 TmpBB->dump()); in splitBranchCondition()
5655 I.dropUnknownNonDebugMetadata(InvariantMD->getMetadataID()); in stripInvariantGroupMetadata()