896   Instruction *cloneIVUser(NarrowIVDefUse DU, const SCEVAddRecExpr *WideAR);
897   Instruction *cloneArithmeticIVUser(NarrowIVDefUse DU,
899   Instruction *cloneBitwiseIVUser(NarrowIVDefUse DU);
903   const SCEVAddRecExpr* getExtendedOperandRecurrence(NarrowIVDefUse DU);
908   Instruction *widenIVUse(NarrowIVDefUse DU, SCEVExpander &Rewriter);
910   bool widenLoopCompare(NarrowIVDefUse DU);
944 Instruction *WidenIV::cloneIVUser(NarrowIVDefUse DU,  in cloneIVUser()  argument
946   unsigned Opcode = DU.NarrowUse->getOpcode();  in cloneIVUser()
954     return cloneArithmeticIVUser(DU, WideAR);  in cloneIVUser()
962     return cloneBitwiseIVUser(DU);  in cloneIVUser()
966 Instruction *WidenIV::cloneBitwiseIVUser(NarrowIVDefUse DU) {  in cloneBitwiseIVUser()  argument
967   Instruction *NarrowUse = DU.NarrowUse;  in cloneBitwiseIVUser()
968   Instruction *NarrowDef = DU.NarrowDef;  in cloneBitwiseIVUser()
969   Instruction *WideDef = DU.WideDef;  in cloneBitwiseIVUser()
995 Instruction *WidenIV::cloneArithmeticIVUser(NarrowIVDefUse DU,  in cloneArithmeticIVUser()  argument
997   Instruction *NarrowUse = DU.NarrowUse;  in cloneArithmeticIVUser()
998   Instruction *NarrowDef = DU.NarrowDef;  in cloneArithmeticIVUser()
999   Instruction *WideDef = DU.WideDef;  in cloneArithmeticIVUser()
1103 const SCEVAddRecExpr* WidenIV::getExtendedOperandRecurrence(NarrowIVDefUse DU) {  in getExtendedOperandRecurrence()  argument
1106   const unsigned OpCode = DU.NarrowUse->getOpcode();  in getExtendedOperandRecurrence()
1115       DU.NarrowUse->getOperand(0) == DU.NarrowDef ? 1 : 0;  in getExtendedOperandRecurrence()
1116   assert(DU.NarrowUse->getOperand(1-ExtendOperIdx) == DU.NarrowDef && "bad DU");  in getExtendedOperandRecurrence()
1120     cast<OverflowingBinaryOperator>(DU.NarrowUse);  in getExtendedOperandRecurrence()
1123       SE->getSCEV(DU.NarrowUse->getOperand(ExtendOperIdx)), WideType);  in getExtendedOperandRecurrence()
1126       SE->getSCEV(DU.NarrowUse->getOperand(ExtendOperIdx)), WideType);  in getExtendedOperandRecurrence()
1135   const SCEV *lhs = SE->getSCEV(DU.WideDef);  in getExtendedOperandRecurrence()
1177 static void truncateIVUse(NarrowIVDefUse DU, DominatorTree *DT, LoopInfo *LI) {  in truncateIVUse()  argument
1178   DEBUG(dbgs() << "INDVARS: Truncate IV " << *DU.WideDef  in truncateIVUse()
1179         << " for user " << *DU.NarrowUse << "\n");  in truncateIVUse()
1181       getInsertPointForUses(DU.NarrowUse, DU.NarrowDef, DT, LI));  in truncateIVUse()
1182   Value *Trunc = Builder.CreateTrunc(DU.WideDef, DU.NarrowDef->getType());  in truncateIVUse()
1183   DU.NarrowUse->replaceUsesOfWith(DU.NarrowDef, Trunc);  in truncateIVUse()
1189 bool WidenIV::widenLoopCompare(NarrowIVDefUse DU) {  in widenLoopCompare()  argument
1190   ICmpInst *Cmp = dyn_cast<ICmpInst>(DU.NarrowUse);  in widenLoopCompare()
1209   if (!(DU.NeverNegative || IsSigned == Cmp->isSigned()))  in widenLoopCompare()
1212   Value *Op = Cmp->getOperand(Cmp->getOperand(0) == DU.NarrowDef ? 1 : 0);  in widenLoopCompare()
1219       getInsertPointForUses(DU.NarrowUse, DU.NarrowDef, DT, LI));  in widenLoopCompare()
1220   DU.NarrowUse->replaceUsesOfWith(DU.NarrowDef, DU.WideDef);  in widenLoopCompare()
1225     DU.NarrowUse->replaceUsesOfWith(Op, ExtOp);  in widenLoopCompare()
1232 Instruction *WidenIV::widenIVUse(NarrowIVDefUse DU, SCEVExpander &Rewriter) {  in widenIVUse()  argument
1235   if (PHINode *UsePhi = dyn_cast<PHINode>(DU.NarrowUse)) {  in widenIVUse()
1241         truncateIVUse(DU, DT, LI);  in widenIVUse()
1244           PHINode::Create(DU.WideDef->getType(), 1, UsePhi->getName() + ".wide",  in widenIVUse()
1246         WidePhi->addIncoming(DU.WideDef, UsePhi->getIncomingBlock(0));  in widenIVUse()
1248         Value *Trunc = Builder.CreateTrunc(WidePhi, DU.NarrowDef->getType());  in widenIVUse()
1258   if (IsSigned ? isa<SExtInst>(DU.NarrowUse) : isa<ZExtInst>(DU.NarrowUse)) {  in widenIVUse()
1259     Value *NewDef = DU.WideDef;  in widenIVUse()
1260     if (DU.NarrowUse->getType() != WideType) {  in widenIVUse()
1261       unsigned CastWidth = SE->getTypeSizeInBits(DU.NarrowUse->getType());  in widenIVUse()
1265         IRBuilder<> Builder(DU.NarrowUse);  in widenIVUse()
1266         NewDef = Builder.CreateTrunc(DU.WideDef, DU.NarrowUse->getType());  in widenIVUse()
1273               << " not wide enough to subsume " << *DU.NarrowUse << "\n");  in widenIVUse()
1274         DU.NarrowUse->replaceUsesOfWith(DU.NarrowDef, DU.WideDef);  in widenIVUse()
1275         NewDef = DU.NarrowUse;  in widenIVUse()
1278     if (NewDef != DU.NarrowUse) {  in widenIVUse()
1279       DEBUG(dbgs() << "INDVARS: eliminating " << *DU.NarrowUse  in widenIVUse()
1280             << " replaced by " << *DU.WideDef << "\n");  in widenIVUse()
1282       DU.NarrowUse->replaceAllUsesWith(NewDef);  in widenIVUse()
1283       DeadInsts.emplace_back(DU.NarrowUse);  in widenIVUse()
1296   const SCEVAddRecExpr *WideAddRec = getWideRecurrence(DU.NarrowUse);  in widenIVUse()
1298     WideAddRec = getExtendedOperandRecurrence(DU);  in widenIVUse()
1303     if (widenLoopCompare(DU))  in widenIVUse()
1309     truncateIVUse(DU, DT, LI);  in widenIVUse()
1314   assert(DU.NarrowUse != DU.NarrowUse->getParent()->getTerminator() &&  in widenIVUse()
1321       && Rewriter.hoistIVInc(WideInc, DU.NarrowUse))  in widenIVUse()
1324     WideUse = cloneIVUser(DU, WideAddRec);  in widenIVUse()
1425     NarrowIVDefUse DU = NarrowIVUsers.pop_back_val();  in createWideIV()  local
1429     Instruction *WideUse = widenIVUse(DU, Rewriter);  in createWideIV()
1433       pushNarrowIVUsers(DU.NarrowUse, WideUse);  in createWideIV()
1436     if (DU.NarrowDef->use_empty())  in createWideIV()
1437       DeadInsts.emplace_back(DU.NarrowDef);  in createWideIV()