Lines Matching refs:SU
554 void ScheduleDAGMI::releaseSucc(SUnit *SU, SDep *SuccEdge) { in releaseSucc() argument
573 if (SuccSU->TopReadyCycle < SU->TopReadyCycle + SuccEdge->getLatency()) in releaseSucc()
574 SuccSU->TopReadyCycle = SU->TopReadyCycle + SuccEdge->getLatency(); in releaseSucc()
582 void ScheduleDAGMI::releaseSuccessors(SUnit *SU) { in releaseSuccessors() argument
583 for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); in releaseSuccessors()
585 releaseSucc(SU, &*I); in releaseSuccessors()
593 void ScheduleDAGMI::releasePred(SUnit *SU, SDep *PredEdge) { in releasePred() argument
612 if (PredSU->BotReadyCycle < SU->BotReadyCycle + PredEdge->getLatency()) in releasePred()
613 PredSU->BotReadyCycle = SU->BotReadyCycle + PredEdge->getLatency(); in releasePred()
621 void ScheduleDAGMI::releasePredecessors(SUnit *SU) { in releasePredecessors() argument
622 for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); in releasePredecessors()
624 releasePred(SU, &*I); in releasePredecessors()
705 SUnit *SU = SchedImpl->pickNode(IsTopNode); in schedule() local
706 if (!SU) break; in schedule()
708 assert(!SU->isScheduled && "Node already scheduled"); in schedule()
712 MachineInstr *MI = SU->getInstr(); in schedule()
714 assert(SU->isTopReady() && "node still has unscheduled dependencies"); in schedule()
720 assert(SU->isBottomReady() && "node still has unscheduled dependencies"); in schedule()
736 SchedImpl->schedNode(SU, IsTopNode); in schedule()
738 updateQueues(SU, IsTopNode); in schedule()
764 SUnit *SU = &(*I); in findRootsAndBiasEdges() local
765 assert(!SU->isBoundaryNode() && "Boundary node should not be in SUnits"); in findRootsAndBiasEdges()
768 SU->biasCriticalPath(); in findRootsAndBiasEdges()
772 TopRoots.push_back(SU); in findRootsAndBiasEdges()
775 BotRoots.push_back(SU); in findRootsAndBiasEdges()
812 void ScheduleDAGMI::updateQueues(SUnit *SU, bool IsTopNode) { in updateQueues() argument
815 releaseSuccessors(SU); in updateQueues()
817 releasePredecessors(SU); in updateQueues()
819 SU->isScheduled = true; in updateQueues()
848 if (SUnit *SU = getSUnit(&(*MI))) in dumpSchedule() local
849 SU->dump(this); in dumpSchedule()
961 updateScheduledPressure(const SUnit *SU, in updateScheduledPressure() argument
963 const PressureDiff &PDiff = getPressureDiff(SU); in updateScheduledPressure()
1006 SUnit &SU = *V2SU.SU; in updatePressureDiffs() local
1007 if (SU.isScheduled || &SU == &ExitSU) in updatePressureDiffs()
1010 PressureDiff &PDiff = getPressureDiff(&SU); in updatePressureDiffs()
1013 dbgs() << " UpdateRegP: SU(" << SU.NodeNum << ") " in updatePressureDiffs()
1015 << ' ' << *SU.getInstr(); in updatePressureDiffs()
1041 SUnit *SU = V2SU.SU; in updatePressureDiffs() local
1044 if (!SU->isScheduled && SU != &ExitSU) { in updatePressureDiffs()
1046 LI.Query(LIS->getInstructionIndex(*SU->getInstr())); in updatePressureDiffs()
1048 PressureDiff &PDiff = getPressureDiff(SU); in updatePressureDiffs()
1051 dbgs() << " UpdateRegP: SU(" << SU->NodeNum << ") " in updatePressureDiffs()
1052 << *SU->getInstr(); in updatePressureDiffs()
1090 for (const SUnit &SU : SUnits) { in schedule()
1091 SU.dumpAll(this); in schedule()
1094 getPressureDiff(&SU).dump(*TRI); in schedule()
1107 SUnit *SU = SchedImpl->pickNode(IsTopNode); in schedule() local
1108 if (!SU) break; in schedule()
1110 assert(!SU->isScheduled && "Node already scheduled"); in schedule()
1114 scheduleMI(SU, IsTopNode); in schedule()
1117 unsigned SubtreeID = DFSResult->getSubtreeID(SU); in schedule()
1126 SchedImpl->schedNode(SU, IsTopNode); in schedule()
1128 updateQueues(SU, IsTopNode); in schedule()
1228 SUnit *SU = V2SU.SU; in computeCyclicCriticalPath() local
1229 if (SU == &ExitSU) in computeCyclicCriticalPath()
1233 LiveQueryResult LRQ = LI.Query(LIS->getInstructionIndex(*SU->getInstr())); in computeCyclicCriticalPath()
1241 if (LiveOutDepth > SU->getDepth()) in computeCyclicCriticalPath()
1242 CyclicLatency = LiveOutDepth - SU->getDepth(); in computeCyclicCriticalPath()
1244 unsigned LiveInHeight = SU->getHeight() + DefSU->Latency; in computeCyclicCriticalPath()
1252 << SU->NodeNum << ") = " << CyclicLatency << "c\n"); in computeCyclicCriticalPath()
1273 void ScheduleDAGMILive::scheduleMI(SUnit *SU, bool IsTopNode) { in scheduleMI() argument
1275 MachineInstr *MI = SU->getInstr(); in scheduleMI()
1278 assert(SU->isTopReady() && "node still has unscheduled dependencies"); in scheduleMI()
1306 updateScheduledPressure(SU, TopRPTracker.getPressure().MaxSetPressure); in scheduleMI()
1309 assert(SU->isBottomReady() && "node still has unscheduled dependencies"); in scheduleMI()
1343 updateScheduledPressure(SU, BotRPTracker.getPressure().MaxSetPressure); in scheduleMI()
1358 SUnit *SU; member
1362 : SU(su), BaseReg(reg), Offset(ofs) {} in MemOpInfo()
1402 SUnit *SU = MemOps[Idx]; in clusterNeighboringMemOps() local
1405 if (TII->getMemOpBaseRegImmOfs(*SU->getInstr(), BaseReg, Offset, TRI)) in clusterNeighboringMemOps()
1406 MemOpRecords.push_back(MemOpInfo(SU, BaseReg, Offset)); in clusterNeighboringMemOps()
1419 SUnit *SUa = MemOpRecords[Idx].SU; in clusterNeighboringMemOps()
1420 SUnit *SUb = MemOpRecords[Idx+1].SU; in clusterNeighboringMemOps()
1453 SUnit *SU = &DAG->SUnits[Idx]; in apply() local
1454 if ((IsLoad && !SU->getInstr()->mayLoad()) || in apply()
1455 (!IsLoad && !SU->getInstr()->mayStore())) in apply()
1460 PI = SU->Preds.begin(), PE = SU->Preds.end(); PI != PE; ++PI) { in apply()
1473 StoreChainDependents[Result.first->second].push_back(SU); in apply()
1524 for (SUnit &SU : DAG->SUnits) { in apply()
1526 if (!SU.Succs.empty()) in apply()
1529 MachineInstr *Pred = SU.getInstr(); in apply()
1542 bool Success = DAG->addEdge(&ExitSU, SDep(&SU, SDep::Cluster)); in apply()
1546 DEBUG(dbgs() << "Macro Fuse SU(" << SU.NodeNum << ")\n"); in apply()
1735 SUnit *SU = &DAG->SUnits[Idx]; in apply() local
1736 if (!SU->getInstr()->isCopy()) in apply()
1739 constrainLocalCopy(SU, static_cast<ScheduleDAGMILive*>(DAG)); in apply()
1825 unsigned SchedBoundary::getLatencyStallCycles(SUnit *SU) { in getLatencyStallCycles() argument
1826 if (!SU->isUnbuffered) in getLatencyStallCycles()
1829 unsigned ReadyCycle = (isTop() ? SU->TopReadyCycle : SU->BotReadyCycle); in getLatencyStallCycles()
1862 bool SchedBoundary::checkHazard(SUnit *SU) { in checkHazard() argument
1864 && HazardRec->getHazardType(SU) != ScheduleHazardRecognizer::NoHazard) { in checkHazard()
1867 unsigned uops = SchedModel->getNumMicroOps(SU->getInstr()); in checkHazard()
1869 DEBUG(dbgs() << " SU(" << SU->NodeNum << ") uops=" in checkHazard()
1870 << SchedModel->getNumMicroOps(SU->getInstr()) << '\n'); in checkHazard()
1873 if (SchedModel->hasInstrSchedModel() && SU->hasReservedResource) { in checkHazard()
1874 const MCSchedClassDesc *SC = DAG->getSchedClass(SU); in checkHazard()
1883 DEBUG(dbgs() << " SU(" << SU->NodeNum << ") " in checkHazard()
1942 void SchedBoundary::releaseNode(SUnit *SU, unsigned ReadyCycle) { in releaseNode() argument
1943 assert(SU->getInstr() && "Scheduled SUnit must have instr"); in releaseNode()
1959 if ((!IsBuffered && ReadyCycle > CurrCycle) || checkHazard(SU) || in releaseNode()
1961 Pending.push(SU); in releaseNode()
1963 Available.push(SU); in releaseNode()
1966 NextSUs.insert(SU); in releaseNode()
1969 void SchedBoundary::releaseTopNode(SUnit *SU) { in releaseTopNode() argument
1970 if (SU->isScheduled) in releaseTopNode()
1973 releaseNode(SU, SU->TopReadyCycle); in releaseTopNode()
1976 void SchedBoundary::releaseBottomNode(SUnit *SU) { in releaseBottomNode() argument
1977 if (SU->isScheduled) in releaseBottomNode()
1980 releaseNode(SU, SU->BotReadyCycle); in releaseBottomNode()
2065 void SchedBoundary::bumpNode(SUnit *SU) { in bumpNode() argument
2068 if (!isTop() && SU->isCall) { in bumpNode()
2073 HazardRec->EmitInstruction(SU); in bumpNode()
2077 const MCSchedClassDesc *SC = DAG->getSchedClass(SU); in bumpNode()
2078 unsigned IncMOps = SchedModel->getNumMicroOps(SU->getInstr()); in bumpNode()
2083 unsigned ReadyCycle = (isTop() ? SU->TopReadyCycle : SU->BotReadyCycle); in bumpNode()
2102 if (SU->isUnbuffered && ReadyCycle > NextCycle) in bumpNode()
2135 if (SU->hasReservedResource) { in bumpNode()
2158 if (SU->getDepth() > TopLatency) { in bumpNode()
2159 TopLatency = SU->getDepth(); in bumpNode()
2161 << " TopLatency SU(" << SU->NodeNum << ") " << TopLatency << "c\n"); in bumpNode()
2163 if (SU->getHeight() > BotLatency) { in bumpNode()
2164 BotLatency = SU->getHeight(); in bumpNode()
2166 << " BotLatency SU(" << SU->NodeNum << ") " << BotLatency << "c\n"); in bumpNode()
2203 SUnit *SU = *(Pending.begin()+i); in releasePending() local
2204 unsigned ReadyCycle = isTop() ? SU->TopReadyCycle : SU->BotReadyCycle; in releasePending()
2212 if (checkHazard(SU)) in releasePending()
2218 Available.push(SU); in releasePending()
2226 void SchedBoundary::removeReady(SUnit *SU) { in removeReady() argument
2227 if (Available.isInQueue(SU)) in removeReady()
2228 Available.remove(Available.find(SU)); in removeReady()
2230 assert(Pending.isInQueue(SU) && "bad ready count"); in removeReady()
2231 Pending.remove(Pending.find(SU)); in removeReady()
2306 const MCSchedClassDesc *SC = DAG->getSchedClass(SU); in initResourceDelta()
2438 Latency = Cand.SU->getDepth(); in traceCandidate()
2441 Latency = Cand.SU->getHeight(); in traceCandidate()
2444 Latency = Cand.SU->getHeight(); in traceCandidate()
2447 Latency = Cand.SU->getDepth(); in traceCandidate()
2450 dbgs() << " Cand SU(" << Cand.SU->NodeNum << ") " << getReasonStr(Cand.Reason); in traceCandidate()
2505 if (Cand.SU->getDepth() > Zone.getScheduledLatency()) { in tryLatency()
2506 if (tryLess(TryCand.SU->getDepth(), Cand.SU->getDepth(), in tryLatency()
2510 if (tryGreater(TryCand.SU->getHeight(), Cand.SU->getHeight(), in tryLatency()
2514 if (Cand.SU->getHeight() > Zone.getScheduledLatency()) { in tryLatency()
2515 if (tryLess(TryCand.SU->getHeight(), Cand.SU->getHeight(), in tryLatency()
2519 if (tryGreater(TryCand.SU->getDepth(), Cand.SU->getDepth(), in tryLatency()
2561 TopCand.SU = nullptr; in initialize()
2562 BotCand.SU = nullptr; in initialize()
2716 static unsigned getWeakLeft(const SUnit *SU, bool isTop) { in getWeakLeft() argument
2717 return (isTop) ? SU->WeakPredsLeft : SU->WeakSuccsLeft; in getWeakLeft()
2727 static int biasPhysRegCopy(const SUnit *SU, bool isTop) { in biasPhysRegCopy() argument
2728 const MachineInstr *MI = SU->getInstr(); in biasPhysRegCopy()
2741 bool AtBoundary = isTop ? !SU->NumSuccsLeft : !SU->NumPredsLeft; in biasPhysRegCopy()
2748 void GenericScheduler::initCandidate(SchedCandidate &Cand, SUnit *SU, in initCandidate() argument
2752 Cand.SU = SU; in initCandidate()
2757 Cand.SU->getInstr(), in initCandidate()
2764 Cand.SU->getInstr(), in initCandidate()
2765 &DAG->getPressureDiff(Cand.SU), in initCandidate()
2771 Cand.SU->getInstr(), in initCandidate()
2772 DAG->getPressureDiff(Cand.SU), in initCandidate()
2780 dbgs() << " Try SU(" << Cand.SU->NodeNum << ") " in initCandidate()
2804 if (tryGreater(biasPhysRegCopy(TryCand.SU, TryCand.AtTop), in tryCandidate()
2805 biasPhysRegCopy(Cand.SU, Cand.AtTop), in tryCandidate()
2839 if (tryLess(Zone->getLatencyStallCycles(TryCand.SU), in tryCandidate()
2840 Zone->getLatencyStallCycles(Cand.SU), TryCand, Cand, Stall)) in tryCandidate()
2854 if (tryGreater(TryCand.SU == TryCandNextClusterSU, in tryCandidate()
2855 Cand.SU == CandNextClusterSU, in tryCandidate()
2861 if (tryLess(getWeakLeft(TryCand.SU, TryCand.AtTop), in tryCandidate()
2862 getWeakLeft(Cand.SU, Cand.AtTop), in tryCandidate()
2894 if (tryGreater(Zone->isNextSU(TryCand.SU), Zone->isNextSU(Cand.SU), in tryCandidate()
2899 if ((Zone->isTop() && TryCand.SU->NodeNum < Cand.SU->NodeNum) in tryCandidate()
2900 || (!Zone->isTop() && TryCand.SU->NodeNum > Cand.SU->NodeNum)) { in tryCandidate()
2940 if (SUnit *SU = Bot.pickOnlyChoice()) { in pickNodeBidirectional() local
2943 return SU; in pickNodeBidirectional()
2945 if (SUnit *SU = Top.pickOnlyChoice()) { in pickNodeBidirectional() local
2948 return SU; in pickNodeBidirectional()
2961 if (!BotCand.isValid() || BotCand.SU->isScheduled || in pickNodeBidirectional()
2973 assert(TCand.SU == BotCand.SU && in pickNodeBidirectional()
2981 if (!TopCand.isValid() || TopCand.SU->isScheduled || in pickNodeBidirectional()
2993 assert(TCand.SU == TopCand.SU && in pickNodeBidirectional()
3012 return Cand.SU; in pickNodeBidirectional()
3022 SUnit *SU; in pickNode() local
3025 SU = Top.pickOnlyChoice(); in pickNode()
3026 if (!SU) { in pickNode()
3032 SU = TopCand.SU; in pickNode()
3036 SU = Bot.pickOnlyChoice(); in pickNode()
3037 if (!SU) { in pickNode()
3043 SU = BotCand.SU; in pickNode()
3047 SU = pickNodeBidirectional(IsTopNode); in pickNode()
3049 } while (SU->isScheduled); in pickNode()
3051 if (SU->isTopReady()) in pickNode()
3052 Top.removeReady(SU); in pickNode()
3053 if (SU->isBottomReady()) in pickNode()
3054 Bot.removeReady(SU); in pickNode()
3056 DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") " << *SU->getInstr()); in pickNode()
3057 return SU; in pickNode()
3060 void GenericScheduler::reschedulePhysRegCopies(SUnit *SU, bool isTop) { in reschedulePhysRegCopies() argument
3062 MachineBasicBlock::iterator InsertPos = SU->getInstr(); in reschedulePhysRegCopies()
3065 SmallVectorImpl<SDep> &Deps = isTop ? SU->Preds : SU->Succs; in reschedulePhysRegCopies()
3092 void GenericScheduler::schedNode(SUnit *SU, bool IsTopNode) { in schedNode() argument
3094 SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle()); in schedNode()
3095 Top.bumpNode(SU); in schedNode()
3096 if (SU->hasPhysRegUses) in schedNode()
3097 reschedulePhysRegCopies(SU, true); in schedNode()
3099 SU->BotReadyCycle = std::max(SU->BotReadyCycle, Bot.getCurrCycle()); in schedNode()
3100 Bot.bumpNode(SU); in schedNode()
3101 if (SU->hasPhysRegDefs) in schedNode()
3102 reschedulePhysRegCopies(SU, false); in schedNode()
3184 if (tryLess(Top.getLatencyStallCycles(TryCand.SU), in tryCandidate()
3185 Top.getLatencyStallCycles(Cand.SU), TryCand, Cand, Stall)) in tryCandidate()
3203 if (TryCand.SU->NodeNum < Cand.SU->NodeNum) in tryCandidate()
3211 TryCand.SU = *I; in pickNodeFromQueue()
3228 SUnit *SU; in pickNode() local
3230 SU = Top.pickOnlyChoice(); in pickNode()
3231 if (SU) { in pickNode()
3242 SU = TopCand.SU; in pickNode()
3244 } while (SU->isScheduled); in pickNode()
3247 Top.removeReady(SU); in pickNode()
3249 DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") " << *SU->getInstr()); in pickNode()
3250 return SU; in pickNode()
3255 void PostGenericScheduler::schedNode(SUnit *SU, bool IsTopNode) { in schedNode() argument
3256 SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle()); in schedNode()
3257 Top.bumpNode(SU); in schedNode()
3334 SUnit *SU = ReadyQ.back(); in pickNode() local
3337 DEBUG(dbgs() << "Pick node " << "SU(" << SU->NodeNum << ") " in pickNode()
3338 << " ILP: " << DAG->getDFSResult()->getILP(SU) in pickNode()
3339 << " Tree: " << DAG->getDFSResult()->getSubtreeID(SU) << " @" in pickNode()
3341 DAG->getDFSResult()->getSubtreeID(SU)) << '\n' in pickNode()
3342 << "Scheduling " << *SU->getInstr()); in pickNode()
3343 return SU; in pickNode()
3353 void schedNode(SUnit *SU, bool IsTopNode) override { in schedNode() argument
3359 void releaseBottomNode(SUnit *SU) override { in releaseBottomNode() argument
3360 ReadyQ.push_back(SU); in releaseBottomNode()
3421 SUnit *SU; in pickNode() local
3425 SU = TopQ.top(); in pickNode()
3427 } while (SU->isScheduled); in pickNode()
3432 SU = BottomQ.top(); in pickNode()
3434 } while (SU->isScheduled); in pickNode()
3439 return SU; in pickNode()
3442 void schedNode(SUnit *SU, bool IsTopNode) override {} in schedNode() argument
3444 void releaseTopNode(SUnit *SU) override { in releaseTopNode() argument
3445 TopQ.push(SU); in releaseTopNode()
3447 void releaseBottomNode(SUnit *SU) override { in releaseBottomNode() argument
3448 BottomQ.push(SU); in releaseBottomNode()
3507 static std::string getNodeLabel(const SUnit *SU, const ScheduleDAG *G) { in getNodeLabel()
3513 SS << "SU:" << SU->NodeNum; in getNodeLabel()
3515 SS << " I:" << DFS->getNumInstrs(SU); in getNodeLabel()
3518 static std::string getNodeDescription(const SUnit *SU, const ScheduleDAG *G) { in getNodeDescription()
3519 return G->getGraphNodeLabel(SU); in getNodeDescription()