1 //===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ---------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Expand VFP / NEON floating point MLA / MLS instructions (each to a pair of
11 // multiple and add / sub instructions) when special VMLx hazards are detected.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #define DEBUG_TYPE "mlx-expansion"
16 #include "ARM.h"
17 #include "ARMBaseInstrInfo.h"
18 #include "ARMSubtarget.h"
19 #include "llvm/ADT/SmallPtrSet.h"
20 #include "llvm/ADT/Statistic.h"
21 #include "llvm/CodeGen/MachineFunctionPass.h"
22 #include "llvm/CodeGen/MachineInstr.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/Support/CommandLine.h"
26 #include "llvm/Support/Debug.h"
27 #include "llvm/Support/raw_ostream.h"
28 #include "llvm/Target/TargetRegisterInfo.h"
29 using namespace llvm;
30
31 static cl::opt<bool>
32 ForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden);
33 static cl::opt<unsigned>
34 ExpandLimit("expand-limit", cl::init(~0U), cl::Hidden);
35
36 STATISTIC(NumExpand, "Number of fp MLA / MLS instructions expanded");
37
38 namespace {
39 struct MLxExpansion : public MachineFunctionPass {
40 static char ID;
MLxExpansion__anonb9695a9e0111::MLxExpansion41 MLxExpansion() : MachineFunctionPass(ID) {}
42
43 virtual bool runOnMachineFunction(MachineFunction &Fn);
44
getPassName__anonb9695a9e0111::MLxExpansion45 virtual const char *getPassName() const {
46 return "ARM MLA / MLS expansion pass";
47 }
48
49 private:
50 const ARMBaseInstrInfo *TII;
51 const TargetRegisterInfo *TRI;
52 MachineRegisterInfo *MRI;
53
54 bool isLikeA9;
55 bool isSwift;
56 unsigned MIIdx;
57 MachineInstr* LastMIs[4];
58 SmallPtrSet<MachineInstr*, 4> IgnoreStall;
59
60 void clearStack();
61 void pushStack(MachineInstr *MI);
62 MachineInstr *getAccDefMI(MachineInstr *MI) const;
63 unsigned getDefReg(MachineInstr *MI) const;
64 bool hasLoopHazard(MachineInstr *MI) const;
65 bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const;
66 bool FindMLxHazard(MachineInstr *MI);
67 void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
68 unsigned MulOpc, unsigned AddSubOpc,
69 bool NegAcc, bool HasLane);
70 bool ExpandFPMLxInstructions(MachineBasicBlock &MBB);
71 };
72 char MLxExpansion::ID = 0;
73 }
74
clearStack()75 void MLxExpansion::clearStack() {
76 std::fill(LastMIs, LastMIs + 4, (MachineInstr*)0);
77 MIIdx = 0;
78 }
79
pushStack(MachineInstr * MI)80 void MLxExpansion::pushStack(MachineInstr *MI) {
81 LastMIs[MIIdx] = MI;
82 if (++MIIdx == 4)
83 MIIdx = 0;
84 }
85
getAccDefMI(MachineInstr * MI) const86 MachineInstr *MLxExpansion::getAccDefMI(MachineInstr *MI) const {
87 // Look past COPY and INSERT_SUBREG instructions to find the
88 // real definition MI. This is important for _sfp instructions.
89 unsigned Reg = MI->getOperand(1).getReg();
90 if (TargetRegisterInfo::isPhysicalRegister(Reg))
91 return 0;
92
93 MachineBasicBlock *MBB = MI->getParent();
94 MachineInstr *DefMI = MRI->getVRegDef(Reg);
95 while (true) {
96 if (DefMI->getParent() != MBB)
97 break;
98 if (DefMI->isCopyLike()) {
99 Reg = DefMI->getOperand(1).getReg();
100 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
101 DefMI = MRI->getVRegDef(Reg);
102 continue;
103 }
104 } else if (DefMI->isInsertSubreg()) {
105 Reg = DefMI->getOperand(2).getReg();
106 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
107 DefMI = MRI->getVRegDef(Reg);
108 continue;
109 }
110 }
111 break;
112 }
113 return DefMI;
114 }
115
getDefReg(MachineInstr * MI) const116 unsigned MLxExpansion::getDefReg(MachineInstr *MI) const {
117 unsigned Reg = MI->getOperand(0).getReg();
118 if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
119 !MRI->hasOneNonDBGUse(Reg))
120 return Reg;
121
122 MachineBasicBlock *MBB = MI->getParent();
123 MachineInstr *UseMI = &*MRI->use_nodbg_begin(Reg);
124 if (UseMI->getParent() != MBB)
125 return Reg;
126
127 while (UseMI->isCopy() || UseMI->isInsertSubreg()) {
128 Reg = UseMI->getOperand(0).getReg();
129 if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
130 !MRI->hasOneNonDBGUse(Reg))
131 return Reg;
132 UseMI = &*MRI->use_nodbg_begin(Reg);
133 if (UseMI->getParent() != MBB)
134 return Reg;
135 }
136
137 return Reg;
138 }
139
140 /// hasLoopHazard - Check whether an MLx instruction is chained to itself across
141 /// a single-MBB loop.
hasLoopHazard(MachineInstr * MI) const142 bool MLxExpansion::hasLoopHazard(MachineInstr *MI) const {
143 unsigned Reg = MI->getOperand(1).getReg();
144 if (TargetRegisterInfo::isPhysicalRegister(Reg))
145 return false;
146
147 MachineBasicBlock *MBB = MI->getParent();
148 MachineInstr *DefMI = MRI->getVRegDef(Reg);
149 while (true) {
150 outer_continue:
151 if (DefMI->getParent() != MBB)
152 break;
153
154 if (DefMI->isPHI()) {
155 for (unsigned i = 1, e = DefMI->getNumOperands(); i < e; i += 2) {
156 if (DefMI->getOperand(i + 1).getMBB() == MBB) {
157 unsigned SrcReg = DefMI->getOperand(i).getReg();
158 if (TargetRegisterInfo::isVirtualRegister(SrcReg)) {
159 DefMI = MRI->getVRegDef(SrcReg);
160 goto outer_continue;
161 }
162 }
163 }
164 } else if (DefMI->isCopyLike()) {
165 Reg = DefMI->getOperand(1).getReg();
166 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
167 DefMI = MRI->getVRegDef(Reg);
168 continue;
169 }
170 } else if (DefMI->isInsertSubreg()) {
171 Reg = DefMI->getOperand(2).getReg();
172 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
173 DefMI = MRI->getVRegDef(Reg);
174 continue;
175 }
176 }
177
178 break;
179 }
180
181 return DefMI == MI;
182 }
183
hasRAWHazard(unsigned Reg,MachineInstr * MI) const184 bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
185 // FIXME: Detect integer instructions properly.
186 const MCInstrDesc &MCID = MI->getDesc();
187 unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
188 if (MI->mayStore())
189 return false;
190 unsigned Opcode = MCID.getOpcode();
191 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
192 return false;
193 if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
194 return MI->readsRegister(Reg, TRI);
195 return false;
196 }
197
isFpMulInstruction(unsigned Opcode)198 static bool isFpMulInstruction(unsigned Opcode) {
199 switch (Opcode) {
200 case ARM::VMULS:
201 case ARM::VMULfd:
202 case ARM::VMULfq:
203 case ARM::VMULD:
204 case ARM::VMULslfd:
205 case ARM::VMULslfq:
206 return true;
207 default:
208 return false;
209 }
210 }
211
FindMLxHazard(MachineInstr * MI)212 bool MLxExpansion::FindMLxHazard(MachineInstr *MI) {
213 if (NumExpand >= ExpandLimit)
214 return false;
215
216 if (ForceExapnd)
217 return true;
218
219 MachineInstr *DefMI = getAccDefMI(MI);
220 if (TII->isFpMLxInstruction(DefMI->getOpcode())) {
221 // r0 = vmla
222 // r3 = vmla r0, r1, r2
223 // takes 16 - 17 cycles
224 //
225 // r0 = vmla
226 // r4 = vmul r1, r2
227 // r3 = vadd r0, r4
228 // takes about 14 - 15 cycles even with vmul stalling for 4 cycles.
229 IgnoreStall.insert(DefMI);
230 return true;
231 }
232
233 // On Swift, we mostly care about hazards from multiplication instructions
234 // writing the accumulator and the pipelining of loop iterations by out-of-
235 // order execution.
236 if (isSwift)
237 return isFpMulInstruction(DefMI->getOpcode()) || hasLoopHazard(MI);
238
239 if (IgnoreStall.count(MI))
240 return false;
241
242 // If a VMLA.F is followed by an VADD.F or VMUL.F with no RAW hazard, the
243 // VADD.F or VMUL.F will stall 4 cycles before issue. The 4 cycle stall
244 // preserves the in-order retirement of the instructions.
245 // Look at the next few instructions, if *most* of them can cause hazards,
246 // then the scheduler can't *fix* this, we'd better break up the VMLA.
247 unsigned Limit1 = isLikeA9 ? 1 : 4;
248 unsigned Limit2 = isLikeA9 ? 1 : 4;
249 for (unsigned i = 1; i <= 4; ++i) {
250 int Idx = ((int)MIIdx - i + 4) % 4;
251 MachineInstr *NextMI = LastMIs[Idx];
252 if (!NextMI)
253 continue;
254
255 if (TII->canCauseFpMLxStall(NextMI->getOpcode())) {
256 if (i <= Limit1)
257 return true;
258 }
259
260 // Look for VMLx RAW hazard.
261 if (i <= Limit2 && hasRAWHazard(getDefReg(MI), NextMI))
262 return true;
263 }
264
265 return false;
266 }
267
268 /// ExpandFPMLxInstructions - Expand a MLA / MLS instruction into a pair
269 /// of MUL + ADD / SUB instructions.
270 void
ExpandFPMLxInstruction(MachineBasicBlock & MBB,MachineInstr * MI,unsigned MulOpc,unsigned AddSubOpc,bool NegAcc,bool HasLane)271 MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
272 unsigned MulOpc, unsigned AddSubOpc,
273 bool NegAcc, bool HasLane) {
274 unsigned DstReg = MI->getOperand(0).getReg();
275 bool DstDead = MI->getOperand(0).isDead();
276 unsigned AccReg = MI->getOperand(1).getReg();
277 unsigned Src1Reg = MI->getOperand(2).getReg();
278 unsigned Src2Reg = MI->getOperand(3).getReg();
279 bool Src1Kill = MI->getOperand(2).isKill();
280 bool Src2Kill = MI->getOperand(3).isKill();
281 unsigned LaneImm = HasLane ? MI->getOperand(4).getImm() : 0;
282 unsigned NextOp = HasLane ? 5 : 4;
283 ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NextOp).getImm();
284 unsigned PredReg = MI->getOperand(++NextOp).getReg();
285
286 const MCInstrDesc &MCID1 = TII->get(MulOpc);
287 const MCInstrDesc &MCID2 = TII->get(AddSubOpc);
288 const MachineFunction &MF = *MI->getParent()->getParent();
289 unsigned TmpReg = MRI->createVirtualRegister(
290 TII->getRegClass(MCID1, 0, TRI, MF));
291
292 MachineInstrBuilder MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID1, TmpReg)
293 .addReg(Src1Reg, getKillRegState(Src1Kill))
294 .addReg(Src2Reg, getKillRegState(Src2Kill));
295 if (HasLane)
296 MIB.addImm(LaneImm);
297 MIB.addImm(Pred).addReg(PredReg);
298
299 MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID2)
300 .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead));
301
302 if (NegAcc) {
303 bool AccKill = MRI->hasOneNonDBGUse(AccReg);
304 MIB.addReg(TmpReg, getKillRegState(true))
305 .addReg(AccReg, getKillRegState(AccKill));
306 } else {
307 MIB.addReg(AccReg).addReg(TmpReg, getKillRegState(true));
308 }
309 MIB.addImm(Pred).addReg(PredReg);
310
311 DEBUG({
312 dbgs() << "Expanding: " << *MI;
313 dbgs() << " to:\n";
314 MachineBasicBlock::iterator MII = MI;
315 MII = llvm::prior(MII);
316 MachineInstr &MI2 = *MII;
317 MII = llvm::prior(MII);
318 MachineInstr &MI1 = *MII;
319 dbgs() << " " << MI1;
320 dbgs() << " " << MI2;
321 });
322
323 MI->eraseFromParent();
324 ++NumExpand;
325 }
326
ExpandFPMLxInstructions(MachineBasicBlock & MBB)327 bool MLxExpansion::ExpandFPMLxInstructions(MachineBasicBlock &MBB) {
328 bool Changed = false;
329
330 clearStack();
331 IgnoreStall.clear();
332
333 unsigned Skip = 0;
334 MachineBasicBlock::reverse_iterator MII = MBB.rbegin(), E = MBB.rend();
335 while (MII != E) {
336 MachineInstr *MI = &*MII;
337
338 if (MI->isLabel() || MI->isImplicitDef() || MI->isCopy()) {
339 ++MII;
340 continue;
341 }
342
343 const MCInstrDesc &MCID = MI->getDesc();
344 if (MI->isBarrier()) {
345 clearStack();
346 Skip = 0;
347 ++MII;
348 continue;
349 }
350
351 unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
352 if (Domain == ARMII::DomainGeneral) {
353 if (++Skip == 2)
354 // Assume dual issues of non-VFP / NEON instructions.
355 pushStack(0);
356 } else {
357 Skip = 0;
358
359 unsigned MulOpc, AddSubOpc;
360 bool NegAcc, HasLane;
361 if (!TII->isFpMLxInstruction(MCID.getOpcode(),
362 MulOpc, AddSubOpc, NegAcc, HasLane) ||
363 !FindMLxHazard(MI))
364 pushStack(MI);
365 else {
366 ExpandFPMLxInstruction(MBB, MI, MulOpc, AddSubOpc, NegAcc, HasLane);
367 E = MBB.rend(); // May have changed if MI was the 1st instruction.
368 Changed = true;
369 continue;
370 }
371 }
372
373 ++MII;
374 }
375
376 return Changed;
377 }
378
runOnMachineFunction(MachineFunction & Fn)379 bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) {
380 TII = static_cast<const ARMBaseInstrInfo*>(Fn.getTarget().getInstrInfo());
381 TRI = Fn.getTarget().getRegisterInfo();
382 MRI = &Fn.getRegInfo();
383 const ARMSubtarget *STI = &Fn.getTarget().getSubtarget<ARMSubtarget>();
384 isLikeA9 = STI->isLikeA9() || STI->isSwift();
385 isSwift = STI->isSwift();
386
387 bool Modified = false;
388 for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E;
389 ++MFI) {
390 MachineBasicBlock &MBB = *MFI;
391 Modified |= ExpandFPMLxInstructions(MBB);
392 }
393
394 return Modified;
395 }
396
createMLxExpansionPass()397 FunctionPass *llvm::createMLxExpansionPass() {
398 return new MLxExpansion();
399 }
400