1 //===- GCNRegPressure.h -----------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
10 #define LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
11
12 #include "AMDGPUSubtarget.h"
13 #include "llvm/ADT/DenseMap.h"
14 #include "llvm/CodeGen/LiveIntervals.h"
15 #include "llvm/CodeGen/MachineBasicBlock.h"
16 #include "llvm/CodeGen/MachineInstr.h"
17 #include "llvm/CodeGen/SlotIndexes.h"
18 #include "llvm/MC/LaneBitmask.h"
19 #include "llvm/Support/Debug.h"
20 #include <algorithm>
21 #include <limits>
22
23 namespace llvm {
24
25 class MachineRegisterInfo;
26 class raw_ostream;
27
28 struct GCNRegPressure {
29 enum RegKind {
30 SGPR32,
31 SGPR_TUPLE,
32 VGPR32,
33 VGPR_TUPLE,
34 AGPR32,
35 AGPR_TUPLE,
36 TOTAL_KINDS
37 };
38
GCNRegPressureGCNRegPressure39 GCNRegPressure() {
40 clear();
41 }
42
emptyGCNRegPressure43 bool empty() const { return getSGPRNum() == 0 && getVGPRNum() == 0; }
44
clearGCNRegPressure45 void clear() { std::fill(&Value[0], &Value[TOTAL_KINDS], 0); }
46
getSGPRNumGCNRegPressure47 unsigned getSGPRNum() const { return Value[SGPR32]; }
getVGPRNumGCNRegPressure48 unsigned getVGPRNum() const { return std::max(Value[VGPR32], Value[AGPR32]); }
49
getVGPRTuplesWeightGCNRegPressure50 unsigned getVGPRTuplesWeight() const { return std::max(Value[VGPR_TUPLE],
51 Value[AGPR_TUPLE]); }
getSGPRTuplesWeightGCNRegPressure52 unsigned getSGPRTuplesWeight() const { return Value[SGPR_TUPLE]; }
53
getOccupancyGCNRegPressure54 unsigned getOccupancy(const GCNSubtarget &ST) const {
55 return std::min(ST.getOccupancyWithNumSGPRs(getSGPRNum()),
56 ST.getOccupancyWithNumVGPRs(getVGPRNum()));
57 }
58
59 void inc(unsigned Reg,
60 LaneBitmask PrevMask,
61 LaneBitmask NewMask,
62 const MachineRegisterInfo &MRI);
63
higherOccupancyGCNRegPressure64 bool higherOccupancy(const GCNSubtarget &ST, const GCNRegPressure& O) const {
65 return getOccupancy(ST) > O.getOccupancy(ST);
66 }
67
68 bool less(const GCNSubtarget &ST, const GCNRegPressure& O,
69 unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const;
70
71 bool operator==(const GCNRegPressure &O) const {
72 return std::equal(&Value[0], &Value[TOTAL_KINDS], O.Value);
73 }
74
75 bool operator!=(const GCNRegPressure &O) const {
76 return !(*this == O);
77 }
78
79 void print(raw_ostream &OS, const GCNSubtarget *ST = nullptr) const;
dumpGCNRegPressure80 void dump() const { print(dbgs()); }
81
82 private:
83 unsigned Value[TOTAL_KINDS];
84
85 static unsigned getRegKind(unsigned Reg, const MachineRegisterInfo &MRI);
86
87 friend GCNRegPressure max(const GCNRegPressure &P1,
88 const GCNRegPressure &P2);
89 };
90
max(const GCNRegPressure & P1,const GCNRegPressure & P2)91 inline GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2) {
92 GCNRegPressure Res;
93 for (unsigned I = 0; I < GCNRegPressure::TOTAL_KINDS; ++I)
94 Res.Value[I] = std::max(P1.Value[I], P2.Value[I]);
95 return Res;
96 }
97
98 class GCNRPTracker {
99 public:
100 using LiveRegSet = DenseMap<unsigned, LaneBitmask>;
101
102 protected:
103 const LiveIntervals &LIS;
104 LiveRegSet LiveRegs;
105 GCNRegPressure CurPressure, MaxPressure;
106 const MachineInstr *LastTrackedMI = nullptr;
107 mutable const MachineRegisterInfo *MRI = nullptr;
108
GCNRPTracker(const LiveIntervals & LIS_)109 GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {}
110
111 void reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy,
112 bool After);
113
114 public:
115 // live regs for the current state
decltype(LiveRegs)116 const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; }
getLastTrackedMI()117 const MachineInstr *getLastTrackedMI() const { return LastTrackedMI; }
118
clearMaxPressure()119 void clearMaxPressure() { MaxPressure.clear(); }
120
121 // returns MaxPressure, resetting it
moveMaxPressure()122 decltype(MaxPressure) moveMaxPressure() {
123 auto Res = MaxPressure;
124 MaxPressure.clear();
125 return Res;
126 }
127
moveLiveRegs()128 decltype(LiveRegs) moveLiveRegs() {
129 return std::move(LiveRegs);
130 }
131
132 static void printLiveRegs(raw_ostream &OS, const LiveRegSet& LiveRegs,
133 const MachineRegisterInfo &MRI);
134 };
135
136 class GCNUpwardRPTracker : public GCNRPTracker {
137 public:
GCNUpwardRPTracker(const LiveIntervals & LIS_)138 GCNUpwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
139
140 // reset tracker to the point just below MI
141 // filling live regs upon this point using LIS
142 void reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr);
143
144 // move to the state just above the MI
145 void recede(const MachineInstr &MI);
146
147 // checks whether the tracker's state after receding MI corresponds
148 // to reported by LIS
149 bool isValid() const;
150 };
151
152 class GCNDownwardRPTracker : public GCNRPTracker {
153 // Last position of reset or advanceBeforeNext
154 MachineBasicBlock::const_iterator NextMI;
155
156 MachineBasicBlock::const_iterator MBBEnd;
157
158 public:
GCNDownwardRPTracker(const LiveIntervals & LIS_)159 GCNDownwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
160
getNext()161 const MachineBasicBlock::const_iterator getNext() const { return NextMI; }
162
163 // Reset tracker to the point before the MI
164 // filling live regs upon this point using LIS.
165 // Returns false if block is empty except debug values.
166 bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr);
167
168 // Move to the state right before the next MI. Returns false if reached
169 // end of the block.
170 bool advanceBeforeNext();
171
172 // Move to the state at the MI, advanceBeforeNext has to be called first.
173 void advanceToNext();
174
175 // Move to the state at the next MI. Returns false if reached end of block.
176 bool advance();
177
178 // Advance instructions until before End.
179 bool advance(MachineBasicBlock::const_iterator End);
180
181 // Reset to Begin and advance to End.
182 bool advance(MachineBasicBlock::const_iterator Begin,
183 MachineBasicBlock::const_iterator End,
184 const LiveRegSet *LiveRegsCopy = nullptr);
185 };
186
187 LaneBitmask getLiveLaneMask(unsigned Reg,
188 SlotIndex SI,
189 const LiveIntervals &LIS,
190 const MachineRegisterInfo &MRI);
191
192 GCNRPTracker::LiveRegSet getLiveRegs(SlotIndex SI,
193 const LiveIntervals &LIS,
194 const MachineRegisterInfo &MRI);
195
196 /// creates a map MachineInstr -> LiveRegSet
197 /// R - range of iterators on instructions
198 /// After - upon entry or exit of every instruction
199 /// Note: there is no entry in the map for instructions with empty live reg set
200 /// Complexity = O(NumVirtRegs * averageLiveRangeSegmentsPerReg * lg(R))
201 template <typename Range>
202 DenseMap<MachineInstr*, GCNRPTracker::LiveRegSet>
getLiveRegMap(Range && R,bool After,LiveIntervals & LIS)203 getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS) {
204 std::vector<SlotIndex> Indexes;
205 Indexes.reserve(std::distance(R.begin(), R.end()));
206 auto &SII = *LIS.getSlotIndexes();
207 for (MachineInstr *I : R) {
208 auto SI = SII.getInstructionIndex(*I);
209 Indexes.push_back(After ? SI.getDeadSlot() : SI.getBaseIndex());
210 }
211 std::sort(Indexes.begin(), Indexes.end());
212
213 auto &MRI = (*R.begin())->getParent()->getParent()->getRegInfo();
214 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> LiveRegMap;
215 SmallVector<SlotIndex, 32> LiveIdxs, SRLiveIdxs;
216 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
217 auto Reg = Register::index2VirtReg(I);
218 if (!LIS.hasInterval(Reg))
219 continue;
220 auto &LI = LIS.getInterval(Reg);
221 LiveIdxs.clear();
222 if (!LI.findIndexesLiveAt(Indexes, std::back_inserter(LiveIdxs)))
223 continue;
224 if (!LI.hasSubRanges()) {
225 for (auto SI : LiveIdxs)
226 LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] =
227 MRI.getMaxLaneMaskForVReg(Reg);
228 } else
229 for (const auto &S : LI.subranges()) {
230 // constrain search for subranges by indexes live at main range
231 SRLiveIdxs.clear();
232 S.findIndexesLiveAt(LiveIdxs, std::back_inserter(SRLiveIdxs));
233 for (auto SI : SRLiveIdxs)
234 LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] |= S.LaneMask;
235 }
236 }
237 return LiveRegMap;
238 }
239
getLiveRegsAfter(const MachineInstr & MI,const LiveIntervals & LIS)240 inline GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI,
241 const LiveIntervals &LIS) {
242 return getLiveRegs(LIS.getInstructionIndex(MI).getDeadSlot(), LIS,
243 MI.getParent()->getParent()->getRegInfo());
244 }
245
getLiveRegsBefore(const MachineInstr & MI,const LiveIntervals & LIS)246 inline GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI,
247 const LiveIntervals &LIS) {
248 return getLiveRegs(LIS.getInstructionIndex(MI).getBaseIndex(), LIS,
249 MI.getParent()->getParent()->getRegInfo());
250 }
251
252 template <typename Range>
getRegPressure(const MachineRegisterInfo & MRI,Range && LiveRegs)253 GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI,
254 Range &&LiveRegs) {
255 GCNRegPressure Res;
256 for (const auto &RM : LiveRegs)
257 Res.inc(RM.first, LaneBitmask::getNone(), RM.second, MRI);
258 return Res;
259 }
260
261 bool isEqual(const GCNRPTracker::LiveRegSet &S1,
262 const GCNRPTracker::LiveRegSet &S2);
263
264 void printLivesAt(SlotIndex SI,
265 const LiveIntervals &LIS,
266 const MachineRegisterInfo &MRI);
267
268 } // end namespace llvm
269
270 #endif // LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
271