1 //===-- R600ControlFlowFinalizer.cpp - Finalize Control Flow Inst----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// This pass compute turns all control flow pseudo instructions into native one
12 /// computing their address on the fly ; it also sets STACK_SIZE info.
13 //===----------------------------------------------------------------------===//
14
15 #include "llvm/Support/Debug.h"
16 #include "AMDGPU.h"
17 #include "AMDGPUSubtarget.h"
18 #include "R600Defines.h"
19 #include "R600InstrInfo.h"
20 #include "R600MachineFunctionInfo.h"
21 #include "R600RegisterInfo.h"
22 #include "llvm/CodeGen/MachineFunctionPass.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/Support/raw_ostream.h"
26
27 using namespace llvm;
28
29 #define DEBUG_TYPE "r600cf"
30
31 namespace {
32
33 struct CFStack {
34
35 enum StackItem {
36 ENTRY = 0,
37 SUB_ENTRY = 1,
38 FIRST_NON_WQM_PUSH = 2,
39 FIRST_NON_WQM_PUSH_W_FULL_ENTRY = 3
40 };
41
42 const R600Subtarget *ST;
43 std::vector<StackItem> BranchStack;
44 std::vector<StackItem> LoopStack;
45 unsigned MaxStackSize;
46 unsigned CurrentEntries;
47 unsigned CurrentSubEntries;
48
CFStack__anonf8757cf90111::CFStack49 CFStack(const R600Subtarget *st, CallingConv::ID cc) : ST(st),
50 // We need to reserve a stack entry for CALL_FS in vertex shaders.
51 MaxStackSize(cc == CallingConv::AMDGPU_VS ? 1 : 0),
52 CurrentEntries(0), CurrentSubEntries(0) { }
53
54 unsigned getLoopDepth();
55 bool branchStackContains(CFStack::StackItem);
56 bool requiresWorkAroundForInst(unsigned Opcode);
57 unsigned getSubEntrySize(CFStack::StackItem Item);
58 void updateMaxStackSize();
59 void pushBranch(unsigned Opcode, bool isWQM = false);
60 void pushLoop();
61 void popBranch();
62 void popLoop();
63 };
64
getLoopDepth()65 unsigned CFStack::getLoopDepth() {
66 return LoopStack.size();
67 }
68
branchStackContains(CFStack::StackItem Item)69 bool CFStack::branchStackContains(CFStack::StackItem Item) {
70 for (std::vector<CFStack::StackItem>::const_iterator I = BranchStack.begin(),
71 E = BranchStack.end(); I != E; ++I) {
72 if (*I == Item)
73 return true;
74 }
75 return false;
76 }
77
requiresWorkAroundForInst(unsigned Opcode)78 bool CFStack::requiresWorkAroundForInst(unsigned Opcode) {
79 if (Opcode == AMDGPU::CF_ALU_PUSH_BEFORE && ST->hasCaymanISA() &&
80 getLoopDepth() > 1)
81 return true;
82
83 if (!ST->hasCFAluBug())
84 return false;
85
86 switch(Opcode) {
87 default: return false;
88 case AMDGPU::CF_ALU_PUSH_BEFORE:
89 case AMDGPU::CF_ALU_ELSE_AFTER:
90 case AMDGPU::CF_ALU_BREAK:
91 case AMDGPU::CF_ALU_CONTINUE:
92 if (CurrentSubEntries == 0)
93 return false;
94 if (ST->getWavefrontSize() == 64) {
95 // We are being conservative here. We only require this work-around if
96 // CurrentSubEntries > 3 &&
97 // (CurrentSubEntries % 4 == 3 || CurrentSubEntries % 4 == 0)
98 //
99 // We have to be conservative, because we don't know for certain that
100 // our stack allocation algorithm for Evergreen/NI is correct. Applying this
101 // work-around when CurrentSubEntries > 3 allows us to over-allocate stack
102 // resources without any problems.
103 return CurrentSubEntries > 3;
104 } else {
105 assert(ST->getWavefrontSize() == 32);
106 // We are being conservative here. We only require the work-around if
107 // CurrentSubEntries > 7 &&
108 // (CurrentSubEntries % 8 == 7 || CurrentSubEntries % 8 == 0)
109 // See the comment on the wavefront size == 64 case for why we are
110 // being conservative.
111 return CurrentSubEntries > 7;
112 }
113 }
114 }
115
getSubEntrySize(CFStack::StackItem Item)116 unsigned CFStack::getSubEntrySize(CFStack::StackItem Item) {
117 switch(Item) {
118 default:
119 return 0;
120 case CFStack::FIRST_NON_WQM_PUSH:
121 assert(!ST->hasCaymanISA());
122 if (ST->getGeneration() <= R600Subtarget::R700) {
123 // +1 For the push operation.
124 // +2 Extra space required.
125 return 3;
126 } else {
127 // Some documentation says that this is not necessary on Evergreen,
128 // but experimentation has show that we need to allocate 1 extra
129 // sub-entry for the first non-WQM push.
130 // +1 For the push operation.
131 // +1 Extra space required.
132 return 2;
133 }
134 case CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY:
135 assert(ST->getGeneration() >= R600Subtarget::EVERGREEN);
136 // +1 For the push operation.
137 // +1 Extra space required.
138 return 2;
139 case CFStack::SUB_ENTRY:
140 return 1;
141 }
142 }
143
updateMaxStackSize()144 void CFStack::updateMaxStackSize() {
145 unsigned CurrentStackSize =
146 CurrentEntries + (alignTo(CurrentSubEntries, 4) / 4);
147 MaxStackSize = std::max(CurrentStackSize, MaxStackSize);
148 }
149
pushBranch(unsigned Opcode,bool isWQM)150 void CFStack::pushBranch(unsigned Opcode, bool isWQM) {
151 CFStack::StackItem Item = CFStack::ENTRY;
152 switch(Opcode) {
153 case AMDGPU::CF_PUSH_EG:
154 case AMDGPU::CF_ALU_PUSH_BEFORE:
155 if (!isWQM) {
156 if (!ST->hasCaymanISA() &&
157 !branchStackContains(CFStack::FIRST_NON_WQM_PUSH))
158 Item = CFStack::FIRST_NON_WQM_PUSH; // May not be required on Evergreen/NI
159 // See comment in
160 // CFStack::getSubEntrySize()
161 else if (CurrentEntries > 0 &&
162 ST->getGeneration() > R600Subtarget::EVERGREEN &&
163 !ST->hasCaymanISA() &&
164 !branchStackContains(CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY))
165 Item = CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY;
166 else
167 Item = CFStack::SUB_ENTRY;
168 } else
169 Item = CFStack::ENTRY;
170 break;
171 }
172 BranchStack.push_back(Item);
173 if (Item == CFStack::ENTRY)
174 CurrentEntries++;
175 else
176 CurrentSubEntries += getSubEntrySize(Item);
177 updateMaxStackSize();
178 }
179
pushLoop()180 void CFStack::pushLoop() {
181 LoopStack.push_back(CFStack::ENTRY);
182 CurrentEntries++;
183 updateMaxStackSize();
184 }
185
popBranch()186 void CFStack::popBranch() {
187 CFStack::StackItem Top = BranchStack.back();
188 if (Top == CFStack::ENTRY)
189 CurrentEntries--;
190 else
191 CurrentSubEntries-= getSubEntrySize(Top);
192 BranchStack.pop_back();
193 }
194
popLoop()195 void CFStack::popLoop() {
196 CurrentEntries--;
197 LoopStack.pop_back();
198 }
199
200 class R600ControlFlowFinalizer : public MachineFunctionPass {
201
202 private:
203 typedef std::pair<MachineInstr *, std::vector<MachineInstr *> > ClauseFile;
204
205 enum ControlFlowInstruction {
206 CF_TC,
207 CF_VC,
208 CF_CALL_FS,
209 CF_WHILE_LOOP,
210 CF_END_LOOP,
211 CF_LOOP_BREAK,
212 CF_LOOP_CONTINUE,
213 CF_JUMP,
214 CF_ELSE,
215 CF_POP,
216 CF_END
217 };
218
219 static char ID;
220 const R600InstrInfo *TII;
221 const R600RegisterInfo *TRI;
222 unsigned MaxFetchInst;
223 const R600Subtarget *ST;
224
IsTrivialInst(MachineInstr & MI) const225 bool IsTrivialInst(MachineInstr &MI) const {
226 switch (MI.getOpcode()) {
227 case AMDGPU::KILL:
228 case AMDGPU::RETURN:
229 return true;
230 default:
231 return false;
232 }
233 }
234
getHWInstrDesc(ControlFlowInstruction CFI) const235 const MCInstrDesc &getHWInstrDesc(ControlFlowInstruction CFI) const {
236 unsigned Opcode = 0;
237 bool isEg = (ST->getGeneration() >= R600Subtarget::EVERGREEN);
238 switch (CFI) {
239 case CF_TC:
240 Opcode = isEg ? AMDGPU::CF_TC_EG : AMDGPU::CF_TC_R600;
241 break;
242 case CF_VC:
243 Opcode = isEg ? AMDGPU::CF_VC_EG : AMDGPU::CF_VC_R600;
244 break;
245 case CF_CALL_FS:
246 Opcode = isEg ? AMDGPU::CF_CALL_FS_EG : AMDGPU::CF_CALL_FS_R600;
247 break;
248 case CF_WHILE_LOOP:
249 Opcode = isEg ? AMDGPU::WHILE_LOOP_EG : AMDGPU::WHILE_LOOP_R600;
250 break;
251 case CF_END_LOOP:
252 Opcode = isEg ? AMDGPU::END_LOOP_EG : AMDGPU::END_LOOP_R600;
253 break;
254 case CF_LOOP_BREAK:
255 Opcode = isEg ? AMDGPU::LOOP_BREAK_EG : AMDGPU::LOOP_BREAK_R600;
256 break;
257 case CF_LOOP_CONTINUE:
258 Opcode = isEg ? AMDGPU::CF_CONTINUE_EG : AMDGPU::CF_CONTINUE_R600;
259 break;
260 case CF_JUMP:
261 Opcode = isEg ? AMDGPU::CF_JUMP_EG : AMDGPU::CF_JUMP_R600;
262 break;
263 case CF_ELSE:
264 Opcode = isEg ? AMDGPU::CF_ELSE_EG : AMDGPU::CF_ELSE_R600;
265 break;
266 case CF_POP:
267 Opcode = isEg ? AMDGPU::POP_EG : AMDGPU::POP_R600;
268 break;
269 case CF_END:
270 if (ST->hasCaymanISA()) {
271 Opcode = AMDGPU::CF_END_CM;
272 break;
273 }
274 Opcode = isEg ? AMDGPU::CF_END_EG : AMDGPU::CF_END_R600;
275 break;
276 }
277 assert (Opcode && "No opcode selected");
278 return TII->get(Opcode);
279 }
280
isCompatibleWithClause(const MachineInstr & MI,std::set<unsigned> & DstRegs) const281 bool isCompatibleWithClause(const MachineInstr &MI,
282 std::set<unsigned> &DstRegs) const {
283 unsigned DstMI, SrcMI;
284 for (MachineInstr::const_mop_iterator I = MI.operands_begin(),
285 E = MI.operands_end();
286 I != E; ++I) {
287 const MachineOperand &MO = *I;
288 if (!MO.isReg())
289 continue;
290 if (MO.isDef()) {
291 unsigned Reg = MO.getReg();
292 if (AMDGPU::R600_Reg128RegClass.contains(Reg))
293 DstMI = Reg;
294 else
295 DstMI = TRI->getMatchingSuperReg(Reg,
296 TRI->getSubRegFromChannel(TRI->getHWRegChan(Reg)),
297 &AMDGPU::R600_Reg128RegClass);
298 }
299 if (MO.isUse()) {
300 unsigned Reg = MO.getReg();
301 if (AMDGPU::R600_Reg128RegClass.contains(Reg))
302 SrcMI = Reg;
303 else
304 SrcMI = TRI->getMatchingSuperReg(Reg,
305 TRI->getSubRegFromChannel(TRI->getHWRegChan(Reg)),
306 &AMDGPU::R600_Reg128RegClass);
307 }
308 }
309 if ((DstRegs.find(SrcMI) == DstRegs.end())) {
310 DstRegs.insert(DstMI);
311 return true;
312 } else
313 return false;
314 }
315
316 ClauseFile
MakeFetchClause(MachineBasicBlock & MBB,MachineBasicBlock::iterator & I) const317 MakeFetchClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
318 const {
319 MachineBasicBlock::iterator ClauseHead = I;
320 std::vector<MachineInstr *> ClauseContent;
321 unsigned AluInstCount = 0;
322 bool IsTex = TII->usesTextureCache(*ClauseHead);
323 std::set<unsigned> DstRegs;
324 for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) {
325 if (IsTrivialInst(*I))
326 continue;
327 if (AluInstCount >= MaxFetchInst)
328 break;
329 if ((IsTex && !TII->usesTextureCache(*I)) ||
330 (!IsTex && !TII->usesVertexCache(*I)))
331 break;
332 if (!isCompatibleWithClause(*I, DstRegs))
333 break;
334 AluInstCount ++;
335 ClauseContent.push_back(&*I);
336 }
337 MachineInstr *MIb = BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead),
338 getHWInstrDesc(IsTex?CF_TC:CF_VC))
339 .addImm(0) // ADDR
340 .addImm(AluInstCount - 1); // COUNT
341 return ClauseFile(MIb, std::move(ClauseContent));
342 }
343
getLiteral(MachineInstr & MI,std::vector<MachineOperand * > & Lits) const344 void getLiteral(MachineInstr &MI, std::vector<MachineOperand *> &Lits) const {
345 static const unsigned LiteralRegs[] = {
346 AMDGPU::ALU_LITERAL_X,
347 AMDGPU::ALU_LITERAL_Y,
348 AMDGPU::ALU_LITERAL_Z,
349 AMDGPU::ALU_LITERAL_W
350 };
351 const SmallVector<std::pair<MachineOperand *, int64_t>, 3> Srcs =
352 TII->getSrcs(MI);
353 for (const auto &Src:Srcs) {
354 if (Src.first->getReg() != AMDGPU::ALU_LITERAL_X)
355 continue;
356 int64_t Imm = Src.second;
357 std::vector<MachineOperand*>::iterator It =
358 std::find_if(Lits.begin(), Lits.end(),
359 [&](MachineOperand* val)
360 { return val->isImm() && (val->getImm() == Imm);});
361
362 // Get corresponding Operand
363 MachineOperand &Operand = MI.getOperand(
364 TII->getOperandIdx(MI.getOpcode(), AMDGPU::OpName::literal));
365
366 if (It != Lits.end()) {
367 // Reuse existing literal reg
368 unsigned Index = It - Lits.begin();
369 Src.first->setReg(LiteralRegs[Index]);
370 } else {
371 // Allocate new literal reg
372 assert(Lits.size() < 4 && "Too many literals in Instruction Group");
373 Src.first->setReg(LiteralRegs[Lits.size()]);
374 Lits.push_back(&Operand);
375 }
376 }
377 }
378
insertLiterals(MachineBasicBlock::iterator InsertPos,const std::vector<unsigned> & Literals) const379 MachineBasicBlock::iterator insertLiterals(
380 MachineBasicBlock::iterator InsertPos,
381 const std::vector<unsigned> &Literals) const {
382 MachineBasicBlock *MBB = InsertPos->getParent();
383 for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
384 unsigned LiteralPair0 = Literals[i];
385 unsigned LiteralPair1 = (i + 1 < e)?Literals[i + 1]:0;
386 InsertPos = BuildMI(MBB, InsertPos->getDebugLoc(),
387 TII->get(AMDGPU::LITERALS))
388 .addImm(LiteralPair0)
389 .addImm(LiteralPair1);
390 }
391 return InsertPos;
392 }
393
394 ClauseFile
MakeALUClause(MachineBasicBlock & MBB,MachineBasicBlock::iterator & I) const395 MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
396 const {
397 MachineInstr &ClauseHead = *I;
398 std::vector<MachineInstr *> ClauseContent;
399 I++;
400 for (MachineBasicBlock::instr_iterator E = MBB.instr_end(); I != E;) {
401 if (IsTrivialInst(*I)) {
402 ++I;
403 continue;
404 }
405 if (!I->isBundle() && !TII->isALUInstr(I->getOpcode()))
406 break;
407 std::vector<MachineOperand *>Literals;
408 if (I->isBundle()) {
409 MachineInstr &DeleteMI = *I;
410 MachineBasicBlock::instr_iterator BI = I.getInstrIterator();
411 while (++BI != E && BI->isBundledWithPred()) {
412 BI->unbundleFromPred();
413 for (MachineOperand &MO : BI->operands()) {
414 if (MO.isReg() && MO.isInternalRead())
415 MO.setIsInternalRead(false);
416 }
417 getLiteral(*BI, Literals);
418 ClauseContent.push_back(&*BI);
419 }
420 I = BI;
421 DeleteMI.eraseFromParent();
422 } else {
423 getLiteral(*I, Literals);
424 ClauseContent.push_back(&*I);
425 I++;
426 }
427 for (unsigned i = 0, e = Literals.size(); i < e; i += 2) {
428 MachineInstrBuilder MILit = BuildMI(MBB, I, I->getDebugLoc(),
429 TII->get(AMDGPU::LITERALS));
430 if (Literals[i]->isImm()) {
431 MILit.addImm(Literals[i]->getImm());
432 } else {
433 MILit.addGlobalAddress(Literals[i]->getGlobal(),
434 Literals[i]->getOffset());
435 }
436 if (i + 1 < e) {
437 if (Literals[i + 1]->isImm()) {
438 MILit.addImm(Literals[i + 1]->getImm());
439 } else {
440 MILit.addGlobalAddress(Literals[i + 1]->getGlobal(),
441 Literals[i + 1]->getOffset());
442 }
443 } else
444 MILit.addImm(0);
445 ClauseContent.push_back(MILit);
446 }
447 }
448 assert(ClauseContent.size() < 128 && "ALU clause is too big");
449 ClauseHead.getOperand(7).setImm(ClauseContent.size() - 1);
450 return ClauseFile(&ClauseHead, std::move(ClauseContent));
451 }
452
453 void
EmitFetchClause(MachineBasicBlock::iterator InsertPos,ClauseFile & Clause,unsigned & CfCount)454 EmitFetchClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
455 unsigned &CfCount) {
456 CounterPropagateAddr(*Clause.first, CfCount);
457 MachineBasicBlock *BB = Clause.first->getParent();
458 BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::FETCH_CLAUSE))
459 .addImm(CfCount);
460 for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
461 BB->splice(InsertPos, BB, Clause.second[i]);
462 }
463 CfCount += 2 * Clause.second.size();
464 }
465
466 void
EmitALUClause(MachineBasicBlock::iterator InsertPos,ClauseFile & Clause,unsigned & CfCount)467 EmitALUClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
468 unsigned &CfCount) {
469 Clause.first->getOperand(0).setImm(0);
470 CounterPropagateAddr(*Clause.first, CfCount);
471 MachineBasicBlock *BB = Clause.first->getParent();
472 BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::ALU_CLAUSE))
473 .addImm(CfCount);
474 for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
475 BB->splice(InsertPos, BB, Clause.second[i]);
476 }
477 CfCount += Clause.second.size();
478 }
479
CounterPropagateAddr(MachineInstr & MI,unsigned Addr) const480 void CounterPropagateAddr(MachineInstr &MI, unsigned Addr) const {
481 MI.getOperand(0).setImm(Addr + MI.getOperand(0).getImm());
482 }
CounterPropagateAddr(const std::set<MachineInstr * > & MIs,unsigned Addr) const483 void CounterPropagateAddr(const std::set<MachineInstr *> &MIs,
484 unsigned Addr) const {
485 for (MachineInstr *MI : MIs) {
486 CounterPropagateAddr(*MI, Addr);
487 }
488 }
489
490 public:
R600ControlFlowFinalizer(TargetMachine & tm)491 R600ControlFlowFinalizer(TargetMachine &tm)
492 : MachineFunctionPass(ID), TII(nullptr), TRI(nullptr), ST(nullptr) {}
493
runOnMachineFunction(MachineFunction & MF)494 bool runOnMachineFunction(MachineFunction &MF) override {
495 ST = &MF.getSubtarget<R600Subtarget>();
496 MaxFetchInst = ST->getTexVTXClauseSize();
497 TII = ST->getInstrInfo();
498 TRI = ST->getRegisterInfo();
499
500 R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
501
502 CFStack CFStack(ST, MF.getFunction()->getCallingConv());
503 for (MachineFunction::iterator MB = MF.begin(), ME = MF.end(); MB != ME;
504 ++MB) {
505 MachineBasicBlock &MBB = *MB;
506 unsigned CfCount = 0;
507 std::vector<std::pair<unsigned, std::set<MachineInstr *> > > LoopStack;
508 std::vector<MachineInstr * > IfThenElseStack;
509 if (MF.getFunction()->getCallingConv() == CallingConv::AMDGPU_VS) {
510 BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()),
511 getHWInstrDesc(CF_CALL_FS));
512 CfCount++;
513 }
514 std::vector<ClauseFile> FetchClauses, AluClauses;
515 std::vector<MachineInstr *> LastAlu(1);
516 std::vector<MachineInstr *> ToPopAfter;
517
518 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
519 I != E;) {
520 if (TII->usesTextureCache(*I) || TII->usesVertexCache(*I)) {
521 DEBUG(dbgs() << CfCount << ":"; I->dump(););
522 FetchClauses.push_back(MakeFetchClause(MBB, I));
523 CfCount++;
524 LastAlu.back() = nullptr;
525 continue;
526 }
527
528 MachineBasicBlock::iterator MI = I;
529 if (MI->getOpcode() != AMDGPU::ENDIF)
530 LastAlu.back() = nullptr;
531 if (MI->getOpcode() == AMDGPU::CF_ALU)
532 LastAlu.back() = &*MI;
533 I++;
534 bool RequiresWorkAround =
535 CFStack.requiresWorkAroundForInst(MI->getOpcode());
536 switch (MI->getOpcode()) {
537 case AMDGPU::CF_ALU_PUSH_BEFORE:
538 if (RequiresWorkAround) {
539 DEBUG(dbgs() << "Applying bug work-around for ALU_PUSH_BEFORE\n");
540 BuildMI(MBB, MI, MBB.findDebugLoc(MI), TII->get(AMDGPU::CF_PUSH_EG))
541 .addImm(CfCount + 1)
542 .addImm(1);
543 MI->setDesc(TII->get(AMDGPU::CF_ALU));
544 CfCount++;
545 CFStack.pushBranch(AMDGPU::CF_PUSH_EG);
546 } else
547 CFStack.pushBranch(AMDGPU::CF_ALU_PUSH_BEFORE);
548
549 case AMDGPU::CF_ALU:
550 I = MI;
551 AluClauses.push_back(MakeALUClause(MBB, I));
552 DEBUG(dbgs() << CfCount << ":"; MI->dump(););
553 CfCount++;
554 break;
555 case AMDGPU::WHILELOOP: {
556 CFStack.pushLoop();
557 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
558 getHWInstrDesc(CF_WHILE_LOOP))
559 .addImm(1);
560 std::pair<unsigned, std::set<MachineInstr *> > Pair(CfCount,
561 std::set<MachineInstr *>());
562 Pair.second.insert(MIb);
563 LoopStack.push_back(std::move(Pair));
564 MI->eraseFromParent();
565 CfCount++;
566 break;
567 }
568 case AMDGPU::ENDLOOP: {
569 CFStack.popLoop();
570 std::pair<unsigned, std::set<MachineInstr *> > Pair =
571 std::move(LoopStack.back());
572 LoopStack.pop_back();
573 CounterPropagateAddr(Pair.second, CfCount);
574 BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END_LOOP))
575 .addImm(Pair.first + 1);
576 MI->eraseFromParent();
577 CfCount++;
578 break;
579 }
580 case AMDGPU::IF_PREDICATE_SET: {
581 LastAlu.push_back(nullptr);
582 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
583 getHWInstrDesc(CF_JUMP))
584 .addImm(0)
585 .addImm(0);
586 IfThenElseStack.push_back(MIb);
587 DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
588 MI->eraseFromParent();
589 CfCount++;
590 break;
591 }
592 case AMDGPU::ELSE: {
593 MachineInstr * JumpInst = IfThenElseStack.back();
594 IfThenElseStack.pop_back();
595 CounterPropagateAddr(*JumpInst, CfCount);
596 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
597 getHWInstrDesc(CF_ELSE))
598 .addImm(0)
599 .addImm(0);
600 DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
601 IfThenElseStack.push_back(MIb);
602 MI->eraseFromParent();
603 CfCount++;
604 break;
605 }
606 case AMDGPU::ENDIF: {
607 CFStack.popBranch();
608 if (LastAlu.back()) {
609 ToPopAfter.push_back(LastAlu.back());
610 } else {
611 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
612 getHWInstrDesc(CF_POP))
613 .addImm(CfCount + 1)
614 .addImm(1);
615 (void)MIb;
616 DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
617 CfCount++;
618 }
619
620 MachineInstr *IfOrElseInst = IfThenElseStack.back();
621 IfThenElseStack.pop_back();
622 CounterPropagateAddr(*IfOrElseInst, CfCount);
623 IfOrElseInst->getOperand(1).setImm(1);
624 LastAlu.pop_back();
625 MI->eraseFromParent();
626 break;
627 }
628 case AMDGPU::BREAK: {
629 CfCount ++;
630 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
631 getHWInstrDesc(CF_LOOP_BREAK))
632 .addImm(0);
633 LoopStack.back().second.insert(MIb);
634 MI->eraseFromParent();
635 break;
636 }
637 case AMDGPU::CONTINUE: {
638 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
639 getHWInstrDesc(CF_LOOP_CONTINUE))
640 .addImm(0);
641 LoopStack.back().second.insert(MIb);
642 MI->eraseFromParent();
643 CfCount++;
644 break;
645 }
646 case AMDGPU::RETURN: {
647 BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END));
648 CfCount++;
649 if (CfCount % 2) {
650 BuildMI(MBB, I, MBB.findDebugLoc(MI), TII->get(AMDGPU::PAD));
651 CfCount++;
652 }
653 MI->eraseFromParent();
654 for (unsigned i = 0, e = FetchClauses.size(); i < e; i++)
655 EmitFetchClause(I, FetchClauses[i], CfCount);
656 for (unsigned i = 0, e = AluClauses.size(); i < e; i++)
657 EmitALUClause(I, AluClauses[i], CfCount);
658 break;
659 }
660 default:
661 if (TII->isExport(MI->getOpcode())) {
662 DEBUG(dbgs() << CfCount << ":"; MI->dump(););
663 CfCount++;
664 }
665 break;
666 }
667 }
668 for (unsigned i = 0, e = ToPopAfter.size(); i < e; ++i) {
669 MachineInstr *Alu = ToPopAfter[i];
670 BuildMI(MBB, Alu, MBB.findDebugLoc((MachineBasicBlock::iterator)Alu),
671 TII->get(AMDGPU::CF_ALU_POP_AFTER))
672 .addImm(Alu->getOperand(0).getImm())
673 .addImm(Alu->getOperand(1).getImm())
674 .addImm(Alu->getOperand(2).getImm())
675 .addImm(Alu->getOperand(3).getImm())
676 .addImm(Alu->getOperand(4).getImm())
677 .addImm(Alu->getOperand(5).getImm())
678 .addImm(Alu->getOperand(6).getImm())
679 .addImm(Alu->getOperand(7).getImm())
680 .addImm(Alu->getOperand(8).getImm());
681 Alu->eraseFromParent();
682 }
683 MFI->StackSize = CFStack.MaxStackSize;
684 }
685
686 return false;
687 }
688
getPassName() const689 const char *getPassName() const override {
690 return "R600 Control Flow Finalizer Pass";
691 }
692 };
693
694 char R600ControlFlowFinalizer::ID = 0;
695
696 } // end anonymous namespace
697
698
createR600ControlFlowFinalizer(TargetMachine & TM)699 llvm::FunctionPass *llvm::createR600ControlFlowFinalizer(TargetMachine &TM) {
700 return new R600ControlFlowFinalizer(TM);
701 }
702