1 //===--------------------- R600MergeVectorRegisters.cpp -------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// This pass merges inputs of swizzeable instructions into vector sharing
12 /// common data and/or have enough undef subreg using swizzle abilities.
13 ///
14 /// For instance let's consider the following pseudo code :
15 /// vreg5<def> = REG_SEQ vreg1, sub0, vreg2, sub1, vreg3, sub2, undef, sub3
16 /// ...
17 /// vreg7<def> = REG_SEQ vreg1, sub0, vreg3, sub1, undef, sub2, vreg4, sub3
18 /// (swizzable Inst) vreg7, SwizzleMask : sub0, sub1, sub2, sub3
19 ///
20 /// is turned into :
21 /// vreg5<def> = REG_SEQ vreg1, sub0, vreg2, sub1, vreg3, sub2, undef, sub3
22 /// ...
23 /// vreg7<def> = INSERT_SUBREG vreg4, sub3
24 /// (swizzable Inst) vreg7, SwizzleMask : sub0, sub2, sub1, sub3
25 ///
26 /// This allow regalloc to reduce register pressure for vector registers and
27 /// to reduce MOV count.
28 //===----------------------------------------------------------------------===//
29
30 #define DEBUG_TYPE "vec-merger"
31 #include "llvm/Support/Debug.h"
32 #include "AMDGPU.h"
33 #include "R600InstrInfo.h"
34 #include "llvm/CodeGen/DFAPacketizer.h"
35 #include "llvm/CodeGen/MachineDominators.h"
36 #include "llvm/CodeGen/MachineFunctionPass.h"
37 #include "llvm/CodeGen/MachineLoopInfo.h"
38 #include "llvm/CodeGen/Passes.h"
39 #include "llvm/CodeGen/MachineInstrBuilder.h"
40 #include "llvm/Support/raw_ostream.h"
41 #include "llvm/CodeGen/MachineRegisterInfo.h"
42
43 using namespace llvm;
44
45 namespace {
46
47 static bool
isImplicitlyDef(MachineRegisterInfo & MRI,unsigned Reg)48 isImplicitlyDef(MachineRegisterInfo &MRI, unsigned Reg) {
49 for (MachineRegisterInfo::def_iterator It = MRI.def_begin(Reg),
50 E = MRI.def_end(); It != E; ++It) {
51 return (*It).isImplicitDef();
52 }
53 llvm_unreachable("Reg without a def");
54 return false;
55 }
56
57 class RegSeqInfo {
58 public:
59 MachineInstr *Instr;
60 DenseMap<unsigned, unsigned> RegToChan;
61 std::vector<unsigned> UndefReg;
RegSeqInfo(MachineRegisterInfo & MRI,MachineInstr * MI)62 RegSeqInfo(MachineRegisterInfo &MRI, MachineInstr *MI) : Instr(MI) {
63 assert (MI->getOpcode() == AMDGPU::REG_SEQUENCE);
64 for (unsigned i = 1, e = Instr->getNumOperands(); i < e; i+=2) {
65 MachineOperand &MO = Instr->getOperand(i);
66 unsigned Chan = Instr->getOperand(i + 1).getImm();
67 if (isImplicitlyDef(MRI, MO.getReg()))
68 UndefReg.push_back(Chan);
69 else
70 RegToChan[MO.getReg()] = Chan;
71 }
72 }
RegSeqInfo()73 RegSeqInfo() {}
74
operator ==(const RegSeqInfo & RSI) const75 bool operator==(const RegSeqInfo &RSI) const {
76 return RSI.Instr == Instr;
77 }
78 };
79
80 class R600VectorRegMerger : public MachineFunctionPass {
81 private:
82 MachineRegisterInfo *MRI;
83 const R600InstrInfo *TII;
84 bool canSwizzle(const MachineInstr &) const;
85 bool areAllUsesSwizzeable(unsigned Reg) const;
86 void SwizzleInput(MachineInstr &,
87 const std::vector<std::pair<unsigned, unsigned> > &) const;
88 bool tryMergeVector(const RegSeqInfo *, RegSeqInfo *,
89 std::vector<std::pair<unsigned, unsigned> > &Remap) const;
90 bool tryMergeUsingCommonSlot(RegSeqInfo &RSI, RegSeqInfo &CompatibleRSI,
91 std::vector<std::pair<unsigned, unsigned> > &RemapChan);
92 bool tryMergeUsingFreeSlot(RegSeqInfo &RSI, RegSeqInfo &CompatibleRSI,
93 std::vector<std::pair<unsigned, unsigned> > &RemapChan);
94 MachineInstr *RebuildVector(RegSeqInfo *MI,
95 const RegSeqInfo *BaseVec,
96 const std::vector<std::pair<unsigned, unsigned> > &RemapChan) const;
97 void RemoveMI(MachineInstr *);
98 void trackRSI(const RegSeqInfo &RSI);
99
100 typedef DenseMap<unsigned, std::vector<MachineInstr *> > InstructionSetMap;
101 DenseMap<MachineInstr *, RegSeqInfo> PreviousRegSeq;
102 InstructionSetMap PreviousRegSeqByReg;
103 InstructionSetMap PreviousRegSeqByUndefCount;
104 public:
105 static char ID;
R600VectorRegMerger(TargetMachine & tm)106 R600VectorRegMerger(TargetMachine &tm) : MachineFunctionPass(ID),
107 TII(0) { }
108
getAnalysisUsage(AnalysisUsage & AU) const109 void getAnalysisUsage(AnalysisUsage &AU) const {
110 AU.setPreservesCFG();
111 AU.addRequired<MachineDominatorTree>();
112 AU.addPreserved<MachineDominatorTree>();
113 AU.addRequired<MachineLoopInfo>();
114 AU.addPreserved<MachineLoopInfo>();
115 MachineFunctionPass::getAnalysisUsage(AU);
116 }
117
getPassName() const118 const char *getPassName() const {
119 return "R600 Vector Registers Merge Pass";
120 }
121
122 bool runOnMachineFunction(MachineFunction &Fn);
123 };
124
125 char R600VectorRegMerger::ID = 0;
126
canSwizzle(const MachineInstr & MI) const127 bool R600VectorRegMerger::canSwizzle(const MachineInstr &MI)
128 const {
129 if (TII->get(MI.getOpcode()).TSFlags & R600_InstFlag::TEX_INST)
130 return true;
131 switch (MI.getOpcode()) {
132 case AMDGPU::R600_ExportSwz:
133 case AMDGPU::EG_ExportSwz:
134 return true;
135 default:
136 return false;
137 }
138 }
139
tryMergeVector(const RegSeqInfo * Untouched,RegSeqInfo * ToMerge,std::vector<std::pair<unsigned,unsigned>> & Remap) const140 bool R600VectorRegMerger::tryMergeVector(const RegSeqInfo *Untouched,
141 RegSeqInfo *ToMerge, std::vector< std::pair<unsigned, unsigned> > &Remap)
142 const {
143 unsigned CurrentUndexIdx = 0;
144 for (DenseMap<unsigned, unsigned>::iterator It = ToMerge->RegToChan.begin(),
145 E = ToMerge->RegToChan.end(); It != E; ++It) {
146 DenseMap<unsigned, unsigned>::const_iterator PosInUntouched =
147 Untouched->RegToChan.find((*It).first);
148 if (PosInUntouched != Untouched->RegToChan.end()) {
149 Remap.push_back(std::pair<unsigned, unsigned>
150 ((*It).second, (*PosInUntouched).second));
151 continue;
152 }
153 if (CurrentUndexIdx >= Untouched->UndefReg.size())
154 return false;
155 Remap.push_back(std::pair<unsigned, unsigned>
156 ((*It).second, Untouched->UndefReg[CurrentUndexIdx++]));
157 }
158
159 return true;
160 }
161
162 static
getReassignedChan(const std::vector<std::pair<unsigned,unsigned>> & RemapChan,unsigned Chan)163 unsigned getReassignedChan(
164 const std::vector<std::pair<unsigned, unsigned> > &RemapChan,
165 unsigned Chan) {
166 for (unsigned j = 0, je = RemapChan.size(); j < je; j++) {
167 if (RemapChan[j].first == Chan)
168 return RemapChan[j].second;
169 }
170 llvm_unreachable("Chan wasn't reassigned");
171 }
172
RebuildVector(RegSeqInfo * RSI,const RegSeqInfo * BaseRSI,const std::vector<std::pair<unsigned,unsigned>> & RemapChan) const173 MachineInstr *R600VectorRegMerger::RebuildVector(
174 RegSeqInfo *RSI, const RegSeqInfo *BaseRSI,
175 const std::vector<std::pair<unsigned, unsigned> > &RemapChan) const {
176 unsigned Reg = RSI->Instr->getOperand(0).getReg();
177 MachineBasicBlock::iterator Pos = RSI->Instr;
178 MachineBasicBlock &MBB = *Pos->getParent();
179 DebugLoc DL = Pos->getDebugLoc();
180
181 unsigned SrcVec = BaseRSI->Instr->getOperand(0).getReg();
182 DenseMap<unsigned, unsigned> UpdatedRegToChan = BaseRSI->RegToChan;
183 std::vector<unsigned> UpdatedUndef = BaseRSI->UndefReg;
184 for (DenseMap<unsigned, unsigned>::iterator It = RSI->RegToChan.begin(),
185 E = RSI->RegToChan.end(); It != E; ++It) {
186 unsigned DstReg = MRI->createVirtualRegister(&AMDGPU::R600_Reg128RegClass);
187 unsigned SubReg = (*It).first;
188 unsigned Swizzle = (*It).second;
189 unsigned Chan = getReassignedChan(RemapChan, Swizzle);
190
191 MachineInstr *Tmp = BuildMI(MBB, Pos, DL, TII->get(AMDGPU::INSERT_SUBREG),
192 DstReg)
193 .addReg(SrcVec)
194 .addReg(SubReg)
195 .addImm(Chan);
196 UpdatedRegToChan[SubReg] = Chan;
197 std::vector<unsigned>::iterator ChanPos =
198 std::find(UpdatedUndef.begin(), UpdatedUndef.end(), Chan);
199 if (ChanPos != UpdatedUndef.end())
200 UpdatedUndef.erase(ChanPos);
201 assert(std::find(UpdatedUndef.begin(), UpdatedUndef.end(), Chan) ==
202 UpdatedUndef.end() &&
203 "UpdatedUndef shouldn't contain Chan more than once!");
204 DEBUG(dbgs() << " ->"; Tmp->dump(););
205 (void)Tmp;
206 SrcVec = DstReg;
207 }
208 Pos = BuildMI(MBB, Pos, DL, TII->get(AMDGPU::COPY), Reg)
209 .addReg(SrcVec);
210 DEBUG(dbgs() << " ->"; Pos->dump(););
211
212 DEBUG(dbgs() << " Updating Swizzle:\n");
213 for (MachineRegisterInfo::use_iterator It = MRI->use_begin(Reg),
214 E = MRI->use_end(); It != E; ++It) {
215 DEBUG(dbgs() << " ";(*It).dump(); dbgs() << " ->");
216 SwizzleInput(*It, RemapChan);
217 DEBUG((*It).dump());
218 }
219 RSI->Instr->eraseFromParent();
220
221 // Update RSI
222 RSI->Instr = Pos;
223 RSI->RegToChan = UpdatedRegToChan;
224 RSI->UndefReg = UpdatedUndef;
225
226 return Pos;
227 }
228
RemoveMI(MachineInstr * MI)229 void R600VectorRegMerger::RemoveMI(MachineInstr *MI) {
230 for (InstructionSetMap::iterator It = PreviousRegSeqByReg.begin(),
231 E = PreviousRegSeqByReg.end(); It != E; ++It) {
232 std::vector<MachineInstr *> &MIs = (*It).second;
233 MIs.erase(std::find(MIs.begin(), MIs.end(), MI), MIs.end());
234 }
235 for (InstructionSetMap::iterator It = PreviousRegSeqByUndefCount.begin(),
236 E = PreviousRegSeqByUndefCount.end(); It != E; ++It) {
237 std::vector<MachineInstr *> &MIs = (*It).second;
238 MIs.erase(std::find(MIs.begin(), MIs.end(), MI), MIs.end());
239 }
240 }
241
SwizzleInput(MachineInstr & MI,const std::vector<std::pair<unsigned,unsigned>> & RemapChan) const242 void R600VectorRegMerger::SwizzleInput(MachineInstr &MI,
243 const std::vector<std::pair<unsigned, unsigned> > &RemapChan) const {
244 unsigned Offset;
245 if (TII->get(MI.getOpcode()).TSFlags & R600_InstFlag::TEX_INST)
246 Offset = 2;
247 else
248 Offset = 3;
249 for (unsigned i = 0; i < 4; i++) {
250 unsigned Swizzle = MI.getOperand(i + Offset).getImm() + 1;
251 for (unsigned j = 0, e = RemapChan.size(); j < e; j++) {
252 if (RemapChan[j].first == Swizzle) {
253 MI.getOperand(i + Offset).setImm(RemapChan[j].second - 1);
254 break;
255 }
256 }
257 }
258 }
259
areAllUsesSwizzeable(unsigned Reg) const260 bool R600VectorRegMerger::areAllUsesSwizzeable(unsigned Reg) const {
261 for (MachineRegisterInfo::use_iterator It = MRI->use_begin(Reg),
262 E = MRI->use_end(); It != E; ++It) {
263 if (!canSwizzle(*It))
264 return false;
265 }
266 return true;
267 }
268
tryMergeUsingCommonSlot(RegSeqInfo & RSI,RegSeqInfo & CompatibleRSI,std::vector<std::pair<unsigned,unsigned>> & RemapChan)269 bool R600VectorRegMerger::tryMergeUsingCommonSlot(RegSeqInfo &RSI,
270 RegSeqInfo &CompatibleRSI,
271 std::vector<std::pair<unsigned, unsigned> > &RemapChan) {
272 for (MachineInstr::mop_iterator MOp = RSI.Instr->operands_begin(),
273 MOE = RSI.Instr->operands_end(); MOp != MOE; ++MOp) {
274 if (!MOp->isReg())
275 continue;
276 if (PreviousRegSeqByReg[MOp->getReg()].empty())
277 continue;
278 std::vector<MachineInstr *> MIs = PreviousRegSeqByReg[MOp->getReg()];
279 for (unsigned i = 0, e = MIs.size(); i < e; i++) {
280 CompatibleRSI = PreviousRegSeq[MIs[i]];
281 if (RSI == CompatibleRSI)
282 continue;
283 if (tryMergeVector(&CompatibleRSI, &RSI, RemapChan))
284 return true;
285 }
286 }
287 return false;
288 }
289
tryMergeUsingFreeSlot(RegSeqInfo & RSI,RegSeqInfo & CompatibleRSI,std::vector<std::pair<unsigned,unsigned>> & RemapChan)290 bool R600VectorRegMerger::tryMergeUsingFreeSlot(RegSeqInfo &RSI,
291 RegSeqInfo &CompatibleRSI,
292 std::vector<std::pair<unsigned, unsigned> > &RemapChan) {
293 unsigned NeededUndefs = 4 - RSI.UndefReg.size();
294 if (PreviousRegSeqByUndefCount[NeededUndefs].empty())
295 return false;
296 std::vector<MachineInstr *> &MIs =
297 PreviousRegSeqByUndefCount[NeededUndefs];
298 CompatibleRSI = PreviousRegSeq[MIs.back()];
299 tryMergeVector(&CompatibleRSI, &RSI, RemapChan);
300 return true;
301 }
302
trackRSI(const RegSeqInfo & RSI)303 void R600VectorRegMerger::trackRSI(const RegSeqInfo &RSI) {
304 for (DenseMap<unsigned, unsigned>::const_iterator
305 It = RSI.RegToChan.begin(), E = RSI.RegToChan.end(); It != E; ++It) {
306 PreviousRegSeqByReg[(*It).first].push_back(RSI.Instr);
307 }
308 PreviousRegSeqByUndefCount[RSI.UndefReg.size()].push_back(RSI.Instr);
309 PreviousRegSeq[RSI.Instr] = RSI;
310 }
311
runOnMachineFunction(MachineFunction & Fn)312 bool R600VectorRegMerger::runOnMachineFunction(MachineFunction &Fn) {
313 TII = static_cast<const R600InstrInfo *>(Fn.getTarget().getInstrInfo());
314 MRI = &(Fn.getRegInfo());
315 for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
316 MBB != MBBe; ++MBB) {
317 MachineBasicBlock *MB = MBB;
318 PreviousRegSeq.clear();
319 PreviousRegSeqByReg.clear();
320 PreviousRegSeqByUndefCount.clear();
321
322 for (MachineBasicBlock::iterator MII = MB->begin(), MIIE = MB->end();
323 MII != MIIE; ++MII) {
324 MachineInstr *MI = MII;
325 if (MI->getOpcode() != AMDGPU::REG_SEQUENCE) {
326 if (TII->get(MI->getOpcode()).TSFlags & R600_InstFlag::TEX_INST) {
327 unsigned Reg = MI->getOperand(1).getReg();
328 for (MachineRegisterInfo::def_iterator It = MRI->def_begin(Reg),
329 E = MRI->def_end(); It != E; ++It) {
330 RemoveMI(&(*It));
331 }
332 }
333 continue;
334 }
335
336
337 RegSeqInfo RSI(*MRI, MI);
338
339 // All uses of MI are swizzeable ?
340 unsigned Reg = MI->getOperand(0).getReg();
341 if (!areAllUsesSwizzeable(Reg))
342 continue;
343
344 DEBUG (dbgs() << "Trying to optimize ";
345 MI->dump();
346 );
347
348 RegSeqInfo CandidateRSI;
349 std::vector<std::pair<unsigned, unsigned> > RemapChan;
350 DEBUG(dbgs() << "Using common slots...\n";);
351 if (tryMergeUsingCommonSlot(RSI, CandidateRSI, RemapChan)) {
352 // Remove CandidateRSI mapping
353 RemoveMI(CandidateRSI.Instr);
354 MII = RebuildVector(&RSI, &CandidateRSI, RemapChan);
355 trackRSI(RSI);
356 continue;
357 }
358 DEBUG(dbgs() << "Using free slots...\n";);
359 RemapChan.clear();
360 if (tryMergeUsingFreeSlot(RSI, CandidateRSI, RemapChan)) {
361 RemoveMI(CandidateRSI.Instr);
362 MII = RebuildVector(&RSI, &CandidateRSI, RemapChan);
363 trackRSI(RSI);
364 continue;
365 }
366 //Failed to merge
367 trackRSI(RSI);
368 }
369 }
370 return false;
371 }
372
373 }
374
createR600VectorRegMerger(TargetMachine & tm)375 llvm::FunctionPass *llvm::createR600VectorRegMerger(TargetMachine &tm) {
376 return new R600VectorRegMerger(tm);
377 }
378