Lines Matching refs:SCEV
137 typedef DenseMap<const SCEV *, RegSortData> RegUsesTy;
140 SmallVector<const SCEV *, 16> RegSequence;
143 void CountRegister(const SCEV *Reg, size_t LUIdx);
144 void DropRegister(const SCEV *Reg, size_t LUIdx);
147 bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const;
149 const SmallBitVector &getUsedByIndices(const SCEV *Reg) const;
153 typedef SmallVectorImpl<const SCEV *>::iterator iterator;
154 typedef SmallVectorImpl<const SCEV *>::const_iterator const_iterator;
164 RegUseTracker::CountRegister(const SCEV *Reg, size_t LUIdx) { in CountRegister()
175 RegUseTracker::DropRegister(const SCEV *Reg, size_t LUIdx) { in DropRegister()
200 RegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const { in isRegUsedByUsesOtherThan()
211 const SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const { in getUsedByIndices()
250 SmallVector<const SCEV *, 4> BaseRegs;
254 const SCEV *ScaledReg;
265 void InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE);
276 void DeleteBaseReg(const SCEV *&S);
278 bool referencesReg(const SCEV *S) const;
289 static void DoInitialMatch(const SCEV *S, Loop *L, in DoInitialMatch()
290 SmallVectorImpl<const SCEV *> &Good, in DoInitialMatch()
291 SmallVectorImpl<const SCEV *> &Bad, in DoInitialMatch()
314 AR->getLoop(), SCEV::FlagAnyWrap), in DoInitialMatch()
322 SmallVector<const SCEV *, 4> Ops(Mul->op_begin()+1, Mul->op_end()); in DoInitialMatch()
323 const SCEV *NewMul = SE.getMulExpr(Ops); in DoInitialMatch()
325 SmallVector<const SCEV *, 4> MyGood; in DoInitialMatch()
326 SmallVector<const SCEV *, 4> MyBad; in DoInitialMatch()
328 const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue( in DoInitialMatch()
330 for (SmallVectorImpl<const SCEV *>::const_iterator I = MyGood.begin(), in DoInitialMatch()
333 for (SmallVectorImpl<const SCEV *>::const_iterator I = MyBad.begin(), in DoInitialMatch()
347 void Formula::InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) { in InitialMatch()
348 SmallVector<const SCEV *, 4> Good; in InitialMatch()
349 SmallVector<const SCEV *, 4> Bad; in InitialMatch()
352 const SCEV *Sum = SE.getAddExpr(Good); in InitialMatch()
358 const SCEV *Sum = SE.getAddExpr(Bad); in InitialMatch()
428 void Formula::DeleteBaseReg(const SCEV *&S) { in DeleteBaseReg()
435 bool Formula::referencesReg(const SCEV *S) const { in referencesReg()
447 for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(), in hasRegsUsedByUsesOtherThan()
464 for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(), in print()
528 static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS, in getExactSDiv()
562 const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE, in getExactSDiv()
565 const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE, in getExactSDiv()
571 return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagAnyWrap); in getExactSDiv()
579 SmallVector<const SCEV *, 8> Ops; in getExactSDiv()
582 const SCEV *Op = getExactSDiv(*I, RHS, SE, in getExactSDiv()
595 SmallVector<const SCEV *, 4> Ops; in getExactSDiv()
599 const SCEV *S = *I; in getExactSDiv()
601 if (const SCEV *Q = getExactSDiv(S, RHS, SE, in getExactSDiv()
620 static int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) { in ExtractImmediate()
627 SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); in ExtractImmediate()
633 SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); in ExtractImmediate()
638 SCEV::FlagAnyWrap); in ExtractImmediate()
647 static GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) { in ExtractSymbol()
654 SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); in ExtractSymbol()
660 SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); in ExtractSymbol()
665 SCEV::FlagAnyWrap); in ExtractSymbol()
746 static bool isHighCostExpansion(const SCEV *S, in isHighCostExpansion()
747 SmallPtrSet<const SCEV*, 8> &Processed, in isHighCostExpansion() argument
895 SmallPtrSet<const SCEV *, 16> &Regs,
896 const DenseSet<const SCEV *> &VisitedRegs,
901 SmallPtrSet<const SCEV *, 16> *LoserRegs = nullptr);
907 void RateRegister(const SCEV *Reg,
908 SmallPtrSet<const SCEV *, 16> &Regs,
911 void RatePrimaryRegister(const SCEV *Reg,
912 SmallPtrSet<const SCEV *, 16> &Regs,
915 SmallPtrSet<const SCEV *, 16> *LoserRegs);
921 void Cost::RateRegister(const SCEV *Reg, in RateRegister()
922 SmallPtrSet<const SCEV *, 16> &Regs, in RateRegister() argument
969 void Cost::RatePrimaryRegister(const SCEV *Reg, in RatePrimaryRegister()
970 SmallPtrSet<const SCEV *, 16> &Regs, in RatePrimaryRegister() argument
973 SmallPtrSet<const SCEV *, 16> *LoserRegs) { in RatePrimaryRegister()
987 SmallPtrSet<const SCEV *, 16> &Regs, in RateFormula() argument
988 const DenseSet<const SCEV *> &VisitedRegs, in RateFormula()
993 SmallPtrSet<const SCEV *, 16> *LoserRegs) { in RateFormula()
996 if (const SCEV *ScaledReg = F.ScaledReg) { in RateFormula()
1005 for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), in RateFormula()
1007 const SCEV *BaseReg = *I; in RateFormula()
1179 static SmallVector<const SCEV *, 4> getEmptyKey() { in getEmptyKey()
1180 SmallVector<const SCEV *, 4> V; in getEmptyKey()
1181 V.push_back(reinterpret_cast<const SCEV *>(-1)); in getEmptyKey()
1185 static SmallVector<const SCEV *, 4> getTombstoneKey() { in getTombstoneKey()
1186 SmallVector<const SCEV *, 4> V; in getTombstoneKey()
1187 V.push_back(reinterpret_cast<const SCEV *>(-2)); in getTombstoneKey()
1191 static unsigned getHashValue(const SmallVector<const SCEV *, 4> &V) { in getHashValue()
1195 static bool isEqual(const SmallVector<const SCEV *, 4> &LHS, in isEqual()
1196 const SmallVector<const SCEV *, 4> &RHS) { in isEqual()
1207 DenseSet<SmallVector<const SCEV *, 4>, UniquifierDenseMapInfo> Uniquifier;
1220 typedef PointerIntPair<const SCEV *, 2, KindType> SCEVUseKindPair;
1253 SmallPtrSet<const SCEV *, 4> Regs;
1276 SmallVector<const SCEV *, 4> Key = F.BaseRegs; in HasFormulaWithSameRegs()
1292 SmallVector<const SCEV *, 4> Key = F.BaseRegs; in InsertFormula()
1304 for (SmallVectorImpl<const SCEV *>::const_iterator I = in InsertFormula()
1330 SmallPtrSet<const SCEV *, 4> OldRegs = Regs; in RecomputeRegs()
1340 for (SmallPtrSet<const SCEV *, 4>::iterator I = OldRegs.begin(), in RecomputeRegs()
1565 Type *AccessTy, const SCEV *S, bool HasBaseReg) { in isAlwaysFoldable()
1602 const SCEV *IncExpr;
1604 IVInc(Instruction *U, Value *O, const SCEV *E): in IVInc()
1612 const SCEV *ExprBase;
1616 IVChain(const IVInc &Head, const SCEV *Base) in IVChain()
1641 bool isProfitableIncrement(const SCEV *OperExpr,
1642 const SCEV *IncExpr,
1724 std::pair<size_t, int64_t> getUse(const SCEV *&Expr,
1732 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1733 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1774 const SmallPtrSet<const SCEV *, 16> &CurRegs,
1775 DenseSet<const SCEV *> &VisitedRegs) const;
1822 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); in OptimizeShadowIV()
2002 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); in OptimizeMax()
2005 const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1); in OptimizeMax()
2008 const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount); in OptimizeMax()
2035 const SCEV *MaxLHS = Max->getOperand(0); in OptimizeMax()
2036 const SCEV *MaxRHS = Max->getOperand(1); in OptimizeMax()
2046 const SCEV *IV = SE.getSCEV(Cond->getOperand(0)); in OptimizeMax()
2157 const SCEV *A = IU.getStride(*CondUse, L); in OptimizeLoopTermCond()
2158 const SCEV *B = IU.getStride(*UI, L); in OptimizeLoopTermCond()
2289 LSRInstance::getUse(const SCEV *&Expr, in getUse()
2291 const SCEV *Copy = Expr; in getUse()
2383 SmallSetVector<const SCEV *, 4> Strides; in CollectInterestingTypesAndFactors()
2386 SmallVector<const SCEV *, 4> Worklist; in CollectInterestingTypesAndFactors()
2388 const SCEV *Expr = IU.getExpr(*UI); in CollectInterestingTypesAndFactors()
2396 const SCEV *S = Worklist.pop_back_val(); in CollectInterestingTypesAndFactors()
2408 for (SmallSetVector<const SCEV *, 4>::const_iterator in CollectInterestingTypesAndFactors()
2410 for (SmallSetVector<const SCEV *, 4>::const_iterator NewStrideIter = in CollectInterestingTypesAndFactors()
2412 const SCEV *OldStride = *I; in CollectInterestingTypesAndFactors()
2413 const SCEV *NewStride = *NewStrideIter; in CollectInterestingTypesAndFactors()
2492 static const SCEV *getExprBase(const SCEV *S) { in getExprBase()
2511 const SCEV *SubExpr = *I; in getExprBase()
2530 bool IVChain::isProfitableIncrement(const SCEV *OperExpr, in isProfitableIncrement()
2531 const SCEV *IncExpr, in isProfitableIncrement()
2540 const SCEV *HeadExpr = SE.getSCEV(getWideOperand(Incs[0].IVOperand)); in isProfitableIncrement()
2545 SmallPtrSet<const SCEV*, 8> Processed; in isProfitableIncrement()
2588 const SCEV *LastIncExpr = nullptr; in isProfitableChain()
2641 const SCEV *const OperExpr = SE.getSCEV(NextIV); in ChainInstruction()
2642 const SCEV *const OperExprBase = getExprBase(OperExpr); in ChainInstruction()
2647 const SCEV *LastIncExpr = nullptr; in ChainInstruction()
2667 const SCEV *PrevExpr = SE.getSCEV(PrevIV); in ChainInstruction()
2668 const SCEV *IncExpr = SE.getMinusSCEV(OperExpr, PrevExpr); in ChainInstruction()
2856 static bool canFoldIVIncExpr(const SCEV *IncExpr, Instruction *UserInst, in canFoldIVIncExpr()
2912 const SCEV *LeftOverExpr = nullptr; in GenerateIVChain()
2926 const SCEV *IncExpr = SE.getNoopOrSignExtend(IncI->IncExpr, IntTy); in GenerateIVChain()
2934 const SCEV *IVOperExpr = SE.getAddExpr(SE.getUnknown(IVSrc), in GenerateIVChain()
3004 const SCEV *S = IU.getExpr(*UI); in CollectFixupsAndInitialFormulae()
3025 const SCEV *N = SE.getSCEV(NV); in CollectFixupsAndInitialFormulae()
3068 LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) { in InsertInitialFormula()
3082 LSRInstance::InsertSupplementalFormula(const SCEV *S, in InsertSupplementalFormula()
3096 for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), in CountRegisters()
3121 SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end()); in CollectLoopInvariantFixupsAndFormulae()
3122 SmallPtrSet<const SCEV *, 8> Inserted; in CollectLoopInvariantFixupsAndFormulae()
3125 const SCEV *S = Worklist.pop_back_val(); in CollectLoopInvariantFixupsAndFormulae()
3162 const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst)); in CollectLoopInvariantFixupsAndFormulae()
3205 static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C, in CollectSubexprs()
3206 SmallVectorImpl<const SCEV *> &Ops, in CollectSubexprs()
3218 const SCEV *Remainder = CollectSubexprs(*I, C, Ops, L, SE, Depth+1); in CollectSubexprs()
3228 const SCEV *Remainder = CollectSubexprs(AR->getStart(), in CollectSubexprs()
3243 SCEV::FlagAnyWrap); in CollectSubexprs()
3252 const SCEV *Remainder = in CollectSubexprs()
3267 const SCEV *BaseReg = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx]; in GenerateReassociationsImpl()
3268 SmallVector<const SCEV *, 8> AddOps; in GenerateReassociationsImpl()
3269 const SCEV *Remainder = CollectSubexprs(BaseReg, nullptr, AddOps, L, SE); in GenerateReassociationsImpl()
3276 for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(), in GenerateReassociationsImpl()
3292 SmallVector<const SCEV *, 8> InnerAddOps( in GenerateReassociationsImpl()
3293 ((const SmallVector<const SCEV *, 8> &)AddOps).begin(), J); in GenerateReassociationsImpl()
3295 ((const SmallVector<const SCEV *, 8> &)AddOps).end()); in GenerateReassociationsImpl()
3304 const SCEV *InnerSum = SE.getAddExpr(InnerAddOps); in GenerateReassociationsImpl()
3375 SmallVector<const SCEV *, 4> Ops; in GenerateCombinations()
3376 for (SmallVectorImpl<const SCEV *>::const_iterator in GenerateCombinations()
3378 const SCEV *BaseReg = *I; in GenerateCombinations()
3386 const SCEV *Sum = SE.getAddExpr(Ops); in GenerateCombinations()
3402 const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx]; in GenerateSymbolicOffsetsImpl()
3434 const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx]; in GenerateConstantOffsetsImpl()
3443 const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), *I), G); in GenerateConstantOffsetsImpl()
3546 const SCEV *FactorS = SE.getConstant(IntTy, Factor); in GenerateICmpZeroScales()
3624 const SCEV *FactorS = SE.getConstant(IntTy, Factor); in GenerateScales()
3629 if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true)) { in GenerateScales()
3662 for (SmallVectorImpl<const SCEV *>::iterator J = F.BaseRegs.begin(), in GenerateTruncates()
3684 const SCEV *OrigReg;
3686 WorkItem(size_t LI, int64_t I, const SCEV *R) in WorkItem()
3710 typedef std::map<int64_t, const SCEV *> ImmMapTy; in GenerateCrossUseConstantOffsets()
3711 typedef DenseMap<const SCEV *, ImmMapTy> RegMapTy; in GenerateCrossUseConstantOffsets()
3713 DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap; in GenerateCrossUseConstantOffsets()
3714 SmallVector<const SCEV *, 8> Sequence; in GenerateCrossUseConstantOffsets()
3717 const SCEV *Reg = *I; in GenerateCrossUseConstantOffsets()
3732 for (SmallVectorImpl<const SCEV *>::const_iterator I = Sequence.begin(), in GenerateCrossUseConstantOffsets()
3734 const SCEV *Reg = *I; in GenerateCrossUseConstantOffsets()
3750 const SCEV *OrigReg = J->second; in GenerateCrossUseConstantOffsets()
3795 const SCEV *OrigReg = WI.OrigReg; in GenerateCrossUseConstantOffsets()
3798 const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm)); in GenerateCrossUseConstantOffsets()
3839 const SCEV *BaseReg = F.BaseRegs[N]; in GenerateCrossUseConstantOffsets()
3856 for (SmallVectorImpl<const SCEV *>::const_iterator in GenerateCrossUseConstantOffsets()
3917 DenseSet<const SCEV *> VisitedRegs; in FilterOutUndesirableDedicatedRegisters()
3918 SmallPtrSet<const SCEV *, 16> Regs; in FilterOutUndesirableDedicatedRegisters()
3919 SmallPtrSet<const SCEV *, 16> LoserRegs; in FilterOutUndesirableDedicatedRegisters()
3926 typedef DenseMap<SmallVector<const SCEV *, 4>, size_t, UniquifierDenseMapInfo> in FilterOutUndesirableDedicatedRegisters()
3961 SmallVector<const SCEV *, 4> Key; in FilterOutUndesirableDedicatedRegisters()
3962 for (SmallVectorImpl<const SCEV *>::const_iterator J = F.BaseRegs.begin(), in FilterOutUndesirableDedicatedRegisters()
3964 const SCEV *Reg = *J; in FilterOutUndesirableDedicatedRegisters()
4060 for (SmallVectorImpl<const SCEV *>::const_iterator in NarrowSearchSpaceByDetectingSupersets()
4211 SmallPtrSet<const SCEV *, 4> Taken; in NarrowSearchSpaceByPickingWinnerRegs()
4219 const SCEV *Best = nullptr; in NarrowSearchSpaceByPickingWinnerRegs()
4223 const SCEV *Reg = *I; in NarrowSearchSpaceByPickingWinnerRegs()
4286 const SmallPtrSet<const SCEV *, 16> &CurRegs, in SolveRecurse() argument
4287 DenseSet<const SCEV *> &VisitedRegs) const { in SolveRecurse()
4304 SmallSetVector<const SCEV *, 4> ReqRegs; in SolveRecurse()
4305 for (SmallPtrSet<const SCEV *, 16>::const_iterator I = CurRegs.begin(), in SolveRecurse()
4310 SmallPtrSet<const SCEV *, 16> NewRegs; in SolveRecurse()
4320 for (SmallSetVector<const SCEV *, 4>::const_iterator J = ReqRegs.begin(), in SolveRecurse()
4322 const SCEV *Reg = *J; in SolveRecurse()
4353 for (SmallPtrSet<const SCEV *, 16>::const_iterator in SolveRecurse()
4373 SmallPtrSet<const SCEV *, 16> CurRegs; in Solve()
4374 DenseSet<const SCEV *> VisitedRegs; in Solve()
4554 SmallVector<const SCEV *, 8> Ops; in Expand()
4557 for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(), in Expand()
4559 const SCEV *Reg = *I; in Expand()
4574 const SCEV *ScaledS = F.ScaledReg; in Expand()
4662 const SCEV *FullS = Ops.empty() ? in Expand()