Lines Matching refs:FI
193 static bool checkPHIs(struct FlattenInfo &FI, in checkPHIs() argument
210 SafeOuterPHIs.insert(FI.OuterInductionPHI); in checkPHIs()
214 for (PHINode &InnerPHI : FI.InnerLoop->getHeader()->phis()) { in checkPHIs()
217 if (&InnerPHI == FI.InnerInductionPHI) in checkPHIs()
224 InnerPHI.getIncomingValueForBlock(FI.InnerLoop->getLoopPreheader()); in checkPHIs()
226 InnerPHI.getIncomingValueForBlock(FI.InnerLoop->getLoopLatch()); in checkPHIs()
232 if (!OuterPHI || OuterPHI->getParent() != FI.OuterLoop->getHeader()) { in checkPHIs()
242 OuterPHI->getIncomingValueForBlock(FI.OuterLoop->getLoopLatch())); in checkPHIs()
260 FI.InnerPHIsToTransform.insert(&InnerPHI); in checkPHIs()
263 for (PHINode &OuterPHI : FI.OuterLoop->getHeader()->phis()) { in checkPHIs()
275 checkOuterLoopInsts(struct FlattenInfo &FI, in checkOuterLoopInsts() argument
284 for (auto *B : FI.OuterLoop->getBlocks()) { in checkOuterLoopInsts()
285 if (FI.InnerLoop->contains(B)) in checkOuterLoopInsts()
306 Br->getSuccessor(0) == FI.InnerLoop->getHeader()) in checkOuterLoopInsts()
310 if (match(&I, m_c_Mul(m_Specific(FI.OuterInductionPHI), in checkOuterLoopInsts()
311 m_Specific(FI.InnerLimit)))) in checkOuterLoopInsts()
332 static bool checkIVUsers(struct FlattenInfo &FI) { in checkIVUsers() argument
341 Value *InnerLimit = FI.InnerLimit; in checkIVUsers()
342 if (FI.Widened && in checkIVUsers()
349 for (User *U : FI.InnerInductionPHI->users()) { in checkIVUsers()
350 if (U == FI.InnerIncrement) in checkIVUsers()
365 bool IsAdd = match(U, m_c_Add(m_Specific(FI.InnerInductionPHI), in checkIVUsers()
367 match(MatchedMul, m_c_Mul(m_Specific(FI.OuterInductionPHI), in checkIVUsers()
372 bool IsAddTrunc = match(U, m_c_Add(m_Trunc(m_Specific(FI.InnerInductionPHI)), in checkIVUsers()
375 m_c_Mul(m_Trunc(m_Specific(FI.OuterInductionPHI)), in checkIVUsers()
381 FI.LinearIVUses.insert(U); in checkIVUsers()
390 for (User *U : FI.OuterInductionPHI->users()) { in checkIVUsers()
391 if (U == FI.OuterIncrement) in checkIVUsers()
417 dbgs() << "Found " << FI.LinearIVUses.size() in checkIVUsers()
419 for (Value *V : FI.LinearIVUses) { in checkIVUsers()
428 static OverflowResult checkOverflow(struct FlattenInfo &FI, in checkOverflow() argument
430 Function *F = FI.OuterLoop->getHeader()->getParent(); in checkOverflow()
440 FI.InnerLimit, FI.OuterLimit, DL, AC, in checkOverflow()
441 FI.OuterLoop->getLoopPreheader()->getTerminator(), DT); in checkOverflow()
445 for (Value *V : FI.LinearIVUses) { in checkOverflow()
467 static bool CanFlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT, in CanFlattenLoopPair() argument
471 if (!findLoopComponents(FI.InnerLoop, IterationInstructions, FI.InnerInductionPHI, in CanFlattenLoopPair()
472 FI.InnerLimit, FI.InnerIncrement, FI.InnerBranch, SE)) in CanFlattenLoopPair()
474 if (!findLoopComponents(FI.OuterLoop, IterationInstructions, FI.OuterInductionPHI, in CanFlattenLoopPair()
475 FI.OuterLimit, FI.OuterIncrement, FI.OuterBranch, SE)) in CanFlattenLoopPair()
480 if (!FI.OuterLoop->isLoopInvariant(FI.InnerLimit)) { in CanFlattenLoopPair()
484 if (!FI.OuterLoop->isLoopInvariant(FI.OuterLimit)) { in CanFlattenLoopPair()
489 if (!checkPHIs(FI, TTI)) in CanFlattenLoopPair()
493 if (FI.InnerInductionPHI->getType() != FI.OuterInductionPHI->getType()) in CanFlattenLoopPair()
496 if (!checkOuterLoopInsts(FI, IterationInstructions, TTI)) in CanFlattenLoopPair()
504 if (!checkIVUsers(FI)) in CanFlattenLoopPair()
511 static bool DoFlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT, in DoFlattenLoopPair() argument
515 Function *F = FI.OuterLoop->getHeader()->getParent(); in DoFlattenLoopPair()
519 OptimizationRemark Remark(DEBUG_TYPE, "Flattened", FI.InnerLoop->getStartLoc(), in DoFlattenLoopPair()
520 FI.InnerLoop->getHeader()); in DoFlattenLoopPair()
527 BinaryOperator::CreateMul(FI.InnerLimit, FI.OuterLimit, "flatten.tripcount", in DoFlattenLoopPair()
528 FI.OuterLoop->getLoopPreheader()->getTerminator()); in DoFlattenLoopPair()
534 FI.InnerInductionPHI->removeIncomingValue(FI.InnerLoop->getLoopLatch()); in DoFlattenLoopPair()
538 for (PHINode *PHI : FI.InnerPHIsToTransform) in DoFlattenLoopPair()
539 PHI->removeIncomingValue(FI.InnerLoop->getLoopLatch()); in DoFlattenLoopPair()
543 cast<User>(FI.OuterBranch->getCondition())->setOperand(1, NewTripCount); in DoFlattenLoopPair()
546 BasicBlock *InnerExitBlock = FI.InnerLoop->getExitBlock(); in DoFlattenLoopPair()
547 BasicBlock *InnerExitingBlock = FI.InnerLoop->getExitingBlock(); in DoFlattenLoopPair()
550 DT->deleteEdge(InnerExitingBlock, FI.InnerLoop->getHeader()); in DoFlattenLoopPair()
554 IRBuilder<> Builder(FI.OuterInductionPHI->getParent()->getTerminator()); in DoFlattenLoopPair()
555 for (Value *V : FI.LinearIVUses) { in DoFlattenLoopPair()
556 Value *OuterValue = FI.OuterInductionPHI; in DoFlattenLoopPair()
557 if (FI.Widened) in DoFlattenLoopPair()
558 OuterValue = Builder.CreateTrunc(FI.OuterInductionPHI, V->getType(), in DoFlattenLoopPair()
568 SE->forgetLoop(FI.OuterLoop); in DoFlattenLoopPair()
569 SE->forgetLoop(FI.InnerLoop); in DoFlattenLoopPair()
570 LI->erase(FI.InnerLoop); in DoFlattenLoopPair()
574 static bool CanWidenIV(struct FlattenInfo &FI, DominatorTree *DT, in CanWidenIV() argument
583 Module *M = FI.InnerLoop->getHeader()->getParent()->getParent(); in CanWidenIV()
585 auto *InnerType = FI.InnerInductionPHI->getType(); in CanWidenIV()
586 auto *OuterType = FI.OuterInductionPHI->getType(); in CanWidenIV()
603 WideIVs.push_back( {FI.InnerInductionPHI, MaxLegalType, false }); in CanWidenIV()
604 WideIVs.push_back( {FI.OuterInductionPHI, MaxLegalType, false }); in CanWidenIV()
620 FI.Widened = true; in CanWidenIV()
621 return CanFlattenLoopPair(FI, DT, LI, SE, AC, TTI); in CanWidenIV()
624 static bool FlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT, in FlattenLoopPair() argument
630 << FI.OuterLoop->getHeader()->getName() << " and inner loop " in FlattenLoopPair()
631 << FI.InnerLoop->getHeader()->getName() << " in " in FlattenLoopPair()
632 << FI.OuterLoop->getHeader()->getParent()->getName() << "\n"); in FlattenLoopPair()
634 if (!CanFlattenLoopPair(FI, DT, LI, SE, AC, TTI)) in FlattenLoopPair()
638 if (CanWidenIV(FI, DT, LI, SE, AC, TTI)) in FlattenLoopPair()
639 return DoFlattenLoopPair(FI, DT, LI, SE, AC, TTI); in FlattenLoopPair()
645 OverflowResult OR = checkOverflow(FI, DT, AC); in FlattenLoopPair()
656 return DoFlattenLoopPair(FI, DT, LI, SE, AC, TTI); in FlattenLoopPair()
666 struct FlattenInfo FI(OuterLoop, InnerLoop); in Flatten() local
667 Changed |= FlattenLoopPair(FI, DT, LI, SE, AC, TTI); in Flatten()