Lines Matching refs:LU
854 const LSRUse &LU, const Formula &F);
857 const LSRUse &LU, const Formula &F);
904 const LSRUse &LU,
996 const LSRUse &LU, in RateFormula() argument
1025 NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(TTI, LU, F))); in RateFormula()
1029 ScaleCost += getScalingFactorCost(TTI, LU, F); in RateFormula()
1484 const LSRUse &LU, const Formula &F) { in isAMCompletelyFolded() argument
1485 return isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, in isAMCompletelyFolded()
1486 LU.AccessTy, F.BaseGV, F.BaseOffset, F.HasBaseReg, in isAMCompletelyFolded()
1491 const LSRUse &LU, const Formula &F) { in getScalingFactorCost() argument
1497 if (!isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, in getScalingFactorCost()
1498 LU.AccessTy, F)) in getScalingFactorCost()
1501 switch (LU.Kind) { in getScalingFactorCost()
1505 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MinOffset, F.HasBaseReg, in getScalingFactorCost()
1506 F.Scale, LU.AccessTy.AddrSpace); in getScalingFactorCost()
1508 LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MaxOffset, F.HasBaseReg, in getScalingFactorCost()
1509 F.Scale, LU.AccessTy.AddrSpace); in getScalingFactorCost()
1706 bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg,
1712 void DeleteUse(LSRUse &LU, size_t LUIdx);
1716 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1717 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1719 bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
1723 void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
1726 void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
1729 void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
1730 void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
1733 void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
1734 void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
1738 void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
1739 void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
1740 void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
1741 void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
1768 const LSRUse &LU,
2222 bool LSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset, in reconcileNewOffset() argument
2225 int64_t NewMinOffset = LU.MinOffset; in reconcileNewOffset()
2226 int64_t NewMaxOffset = LU.MaxOffset; in reconcileNewOffset()
2232 if (LU.Kind != Kind) in reconcileNewOffset()
2239 if (AccessTy != LU.AccessTy) in reconcileNewOffset()
2244 if (NewOffset < LU.MinOffset) { in reconcileNewOffset()
2246 LU.MaxOffset - NewOffset, HasBaseReg)) in reconcileNewOffset()
2249 } else if (NewOffset > LU.MaxOffset) { in reconcileNewOffset()
2251 NewOffset - LU.MinOffset, HasBaseReg)) in reconcileNewOffset()
2257 LU.MinOffset = NewMinOffset; in reconcileNewOffset()
2258 LU.MaxOffset = NewMaxOffset; in reconcileNewOffset()
2259 LU.AccessTy = NewAccessTy; in reconcileNewOffset()
2260 if (NewOffset != LU.Offsets.back()) in reconcileNewOffset()
2261 LU.Offsets.push_back(NewOffset); in reconcileNewOffset()
2286 LSRUse &LU = Uses[LUIdx]; in getUse() local
2287 if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy)) in getUse()
2296 LSRUse &LU = Uses[LUIdx]; in getUse() local
2300 if (LU.Offsets.empty() || Offset != LU.Offsets.back()) in getUse()
2301 LU.Offsets.push_back(Offset); in getUse()
2303 LU.MinOffset = Offset; in getUse()
2304 LU.MaxOffset = Offset; in getUse()
2309 void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) { in DeleteUse() argument
2310 if (&LU != &Uses.back()) in DeleteUse()
2311 std::swap(LU, Uses.back()); in DeleteUse()
2325 LSRUse &LU = Uses[LUIdx]; in FindUseWithSimilarFormula() local
2331 if (&LU != &OrigLU && in FindUseWithSimilarFormula()
2332 LU.Kind != LSRUse::ICmpZero && in FindUseWithSimilarFormula()
2333 LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy && in FindUseWithSimilarFormula()
2334 LU.WidestFixupType == OrigLU.WidestFixupType && in FindUseWithSimilarFormula()
2335 LU.HasFormulaWithSameRegs(OrigF)) { in FindUseWithSimilarFormula()
2337 for (const Formula &F : LU.Formulae) { in FindUseWithSimilarFormula()
2346 return &LU; in FindUseWithSimilarFormula()
3016 LSRUse &LU = Uses[LF.LUIdx]; in CollectFixupsAndInitialFormulae() local
3017 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); in CollectFixupsAndInitialFormulae()
3018 if (!LU.WidestFixupType || in CollectFixupsAndInitialFormulae()
3019 SE.getTypeSizeInBits(LU.WidestFixupType) < in CollectFixupsAndInitialFormulae()
3021 LU.WidestFixupType = LF.OperandValToReplace->getType(); in CollectFixupsAndInitialFormulae()
3024 if (LU.Formulae.empty()) { in CollectFixupsAndInitialFormulae()
3025 InsertInitialFormula(S, LU, LF.LUIdx); in CollectFixupsAndInitialFormulae()
3026 CountRegisters(LU.Formulae.back(), LF.LUIdx); in CollectFixupsAndInitialFormulae()
3036 LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) { in InsertInitialFormula() argument
3039 LU.RigidFormula = true; in InsertInitialFormula()
3043 bool Inserted = InsertFormula(LU, LUIdx, F); in InsertInitialFormula()
3051 LSRUse &LU, size_t LUIdx) { in InsertSupplementalFormula() argument
3055 bool Inserted = InsertFormula(LU, LUIdx, F); in InsertSupplementalFormula()
3069 bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) { in InsertFormula() argument
3071 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) && in InsertFormula()
3073 if (!LU.InsertFormula(F)) in InsertFormula()
3159 LSRUse &LU = Uses[LF.LUIdx]; in CollectLoopInvariantFixupsAndFormulae() local
3160 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); in CollectLoopInvariantFixupsAndFormulae()
3161 if (!LU.WidestFixupType || in CollectLoopInvariantFixupsAndFormulae()
3162 SE.getTypeSizeInBits(LU.WidestFixupType) < in CollectLoopInvariantFixupsAndFormulae()
3164 LU.WidestFixupType = LF.OperandValToReplace->getType(); in CollectLoopInvariantFixupsAndFormulae()
3165 InsertSupplementalFormula(US, LU, LF.LUIdx); in CollectLoopInvariantFixupsAndFormulae()
3166 CountRegisters(LU.Formulae.back(), Uses.size() - 1); in CollectLoopInvariantFixupsAndFormulae()
3235 void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx, in GenerateReassociationsImpl() argument
3259 if (isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind, in GenerateReassociationsImpl()
3260 LU.AccessTy, *J, Base.getNumRegs() > 1)) in GenerateReassociationsImpl()
3272 isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind, in GenerateReassociationsImpl()
3273 LU.AccessTy, InnerAddOps[0], Base.getNumRegs() > 1)) in GenerateReassociationsImpl()
3310 if (InsertFormula(LU, LUIdx, F)) in GenerateReassociationsImpl()
3313 GenerateReassociations(LU, LUIdx, LU.Formulae.back(), Depth + 1); in GenerateReassociationsImpl()
3318 void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx, in GenerateReassociations() argument
3326 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, i); in GenerateReassociations()
3329 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, in GenerateReassociations()
3335 void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx, in GenerateCombinations() argument
3362 (void)InsertFormula(LU, LUIdx, F); in GenerateCombinations()
3368 void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx, in GenerateSymbolicOffsetsImpl() argument
3377 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) in GenerateSymbolicOffsetsImpl()
3383 (void)InsertFormula(LU, LUIdx, F); in GenerateSymbolicOffsetsImpl()
3387 void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, in GenerateSymbolicOffsets() argument
3393 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, i); in GenerateSymbolicOffsets()
3395 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1, in GenerateSymbolicOffsets()
3401 LSRUse &LU, unsigned LUIdx, const Formula &Base, in GenerateConstantOffsetsImpl() argument
3407 if (isLegalUse(TTI, LU.MinOffset - Offset, LU.MaxOffset - Offset, LU.Kind, in GenerateConstantOffsetsImpl()
3408 LU.AccessTy, F)) { in GenerateConstantOffsetsImpl()
3424 (void)InsertFormula(LU, LUIdx, F); in GenerateConstantOffsetsImpl()
3433 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) in GenerateConstantOffsetsImpl()
3439 (void)InsertFormula(LU, LUIdx, F); in GenerateConstantOffsetsImpl()
3443 void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, in GenerateConstantOffsets() argument
3448 Worklist.push_back(LU.MinOffset); in GenerateConstantOffsets()
3449 if (LU.MaxOffset != LU.MinOffset) in GenerateConstantOffsets()
3450 Worklist.push_back(LU.MaxOffset); in GenerateConstantOffsets()
3453 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, i); in GenerateConstantOffsets()
3455 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1, in GenerateConstantOffsets()
3461 void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, in GenerateICmpZeroScales() argument
3463 if (LU.Kind != LSRUse::ICmpZero) return; in GenerateICmpZeroScales()
3471 if (LU.MinOffset != LU.MaxOffset) return; in GenerateICmpZeroScales()
3489 int64_t Offset = LU.MinOffset; in GenerateICmpZeroScales()
3493 if (Offset / Factor != LU.MinOffset) in GenerateICmpZeroScales()
3504 if (!isLegalUse(TTI, Offset, Offset, LU.Kind, LU.AccessTy, F)) in GenerateICmpZeroScales()
3508 F.BaseOffset = (uint64_t)F.BaseOffset + Offset - LU.MinOffset; in GenerateICmpZeroScales()
3540 (void)InsertFormula(LU, LUIdx, F); in GenerateICmpZeroScales()
3547 void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) { in GenerateScales() argument
3564 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, in GenerateScales()
3568 if (LU.Kind == LSRUse::Basic && in GenerateScales()
3569 isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LSRUse::Special, in GenerateScales()
3570 LU.AccessTy, Base) && in GenerateScales()
3571 LU.AllFixupsOutsideLoop) in GenerateScales()
3572 LU.Kind = LSRUse::Special; in GenerateScales()
3578 if (LU.Kind == LSRUse::ICmpZero && in GenerateScales()
3600 (void)InsertFormula(LU, LUIdx, F); in GenerateScales()
3607 void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) { in GenerateTruncates() argument
3629 (void)InsertFormula(LU, LUIdx, F); in GenerateTruncates()
3742 LSRUse &LU = Uses[LUIdx]; in GenerateCrossUseConstantOffsets() local
3751 for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) { in GenerateCrossUseConstantOffsets()
3752 Formula F = LU.Formulae[L]; in GenerateCrossUseConstantOffsets()
3767 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, in GenerateCrossUseConstantOffsets()
3783 (void)InsertFormula(LU, LUIdx, NewF); in GenerateCrossUseConstantOffsets()
3792 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, in GenerateCrossUseConstantOffsets()
3793 LU.Kind, LU.AccessTy, NewF)) { in GenerateCrossUseConstantOffsets()
3815 (void)InsertFormula(LU, LUIdx, NewF); in GenerateCrossUseConstantOffsets()
3830 LSRUse &LU = Uses[LUIdx]; in GenerateAllReuseFormulae() local
3831 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
3832 GenerateReassociations(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
3833 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
3834 GenerateCombinations(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
3837 LSRUse &LU = Uses[LUIdx]; in GenerateAllReuseFormulae() local
3838 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
3839 GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
3840 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
3841 GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
3842 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
3843 GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
3844 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
3845 GenerateScales(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
3848 LSRUse &LU = Uses[LUIdx]; in GenerateAllReuseFormulae() local
3849 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) in GenerateAllReuseFormulae()
3850 GenerateTruncates(LU, LUIdx, LU.Formulae[i]); in GenerateAllReuseFormulae()
3877 LSRUse &LU = Uses[LUIdx]; in FilterOutUndesirableDedicatedRegisters() local
3878 DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs()); dbgs() << '\n'); in FilterOutUndesirableDedicatedRegisters()
3881 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); in FilterOutUndesirableDedicatedRegisters()
3883 Formula &F = LU.Formulae[FIdx]; in FilterOutUndesirableDedicatedRegisters()
3894 CostF.RateFormula(TTI, F, Regs, VisitedRegs, L, LU.Offsets, SE, DT, LU, in FilterOutUndesirableDedicatedRegisters()
3924 Formula &Best = LU.Formulae[P.first->second]; in FilterOutUndesirableDedicatedRegisters()
3928 CostBest.RateFormula(TTI, Best, Regs, VisitedRegs, L, LU.Offsets, SE, in FilterOutUndesirableDedicatedRegisters()
3929 DT, LU); in FilterOutUndesirableDedicatedRegisters()
3940 LU.DeleteFormula(F); in FilterOutUndesirableDedicatedRegisters()
3948 LU.RecomputeRegs(LUIdx, RegUses); in FilterOutUndesirableDedicatedRegisters()
3969 for (const LSRUse &LU : Uses) { in EstimateSearchSpaceComplexity() local
3970 size_t FSize = LU.Formulae.size(); in EstimateSearchSpaceComplexity()
3994 LSRUse &LU = Uses[LUIdx]; in NarrowSearchSpaceByDetectingSupersets() local
3996 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { in NarrowSearchSpaceByDetectingSupersets()
3997 Formula &F = LU.Formulae[i]; in NarrowSearchSpaceByDetectingSupersets()
4008 if (LU.HasFormulaWithSameRegs(NewF)) { in NarrowSearchSpaceByDetectingSupersets()
4010 LU.DeleteFormula(F); in NarrowSearchSpaceByDetectingSupersets()
4023 if (LU.HasFormulaWithSameRegs(NewF)) { in NarrowSearchSpaceByDetectingSupersets()
4026 LU.DeleteFormula(F); in NarrowSearchSpaceByDetectingSupersets()
4037 LU.RecomputeRegs(LUIdx, RegUses); in NarrowSearchSpaceByDetectingSupersets()
4058 LSRUse &LU = Uses[LUIdx]; in NarrowSearchSpaceByCollapsingUnrolledCode() local
4059 for (const Formula &F : LU.Formulae) { in NarrowSearchSpaceByCollapsingUnrolledCode()
4063 LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU); in NarrowSearchSpaceByCollapsingUnrolledCode()
4068 LU.Kind, LU.AccessTy)) in NarrowSearchSpaceByCollapsingUnrolledCode()
4071 DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << '\n'); in NarrowSearchSpaceByCollapsingUnrolledCode()
4073 LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop; in NarrowSearchSpaceByCollapsingUnrolledCode()
4113 DeleteUse(LU, LUIdx); in NarrowSearchSpaceByCollapsingUnrolledCode()
4177 LSRUse &LU = Uses[LUIdx]; in NarrowSearchSpaceByPickingWinnerRegs() local
4178 if (!LU.Regs.count(Best)) continue; in NarrowSearchSpaceByPickingWinnerRegs()
4181 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { in NarrowSearchSpaceByPickingWinnerRegs()
4182 Formula &F = LU.Formulae[i]; in NarrowSearchSpaceByPickingWinnerRegs()
4185 LU.DeleteFormula(F); in NarrowSearchSpaceByPickingWinnerRegs()
4195 LU.RecomputeRegs(LUIdx, RegUses); in NarrowSearchSpaceByPickingWinnerRegs()
4231 const LSRUse &LU = Uses[Workspace.size()]; in SolveRecurse() local
4239 if (LU.Regs.count(S)) in SolveRecurse()
4244 for (const Formula &F : LU.Formulae) { in SolveRecurse()
4268 NewCost.RateFormula(TTI, F, NewRegs, VisitedRegs, L, LU.Offsets, SE, DT, in SolveRecurse()
4269 LU); in SolveRecurse()
4383 const LSRUse &LU, in AdjustInsertPositionForExpand() argument
4391 if (LU.Kind == LSRUse::ICmpZero) in AdjustInsertPositionForExpand()
4450 const LSRUse &LU = Uses[LF.LUIdx]; in Expand() local
4451 if (LU.RigidFormula) in Expand()
4456 IP = AdjustInsertPositionForExpand(IP, LF, LU, Rewriter); in Expand()
4502 if (LU.Kind == LSRUse::ICmpZero) { in Expand()
4521 if (!Ops.empty() && LU.Kind == LSRUse::Address && in Expand()
4522 isAMCompletelyFolded(TTI, LU, F)) { in Expand()
4557 if (LU.Kind == LSRUse::ICmpZero) { in Expand()
4593 if (LU.Kind == LSRUse::ICmpZero) { in Expand()
4868 for (const LSRUse &LU : Uses) { in LSRInstance() local
4869 for (const Formula &F : LU.Formulae) in LSRInstance()
4870 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, in LSRInstance()
4910 for (const LSRUse &LU : Uses) { in print_uses() local
4912 LU.print(OS); in print_uses()
4914 for (const Formula &F : LU.Formulae) { in print_uses()