1 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file This file contains the AArch64 implementation of the DAG scheduling
11 /// mutation to pair instructions back to back.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "AArch64Subtarget.h"
16 #include "llvm/CodeGen/MacroFusion.h"
17 #include "llvm/CodeGen/TargetInstrInfo.h"
18
19 using namespace llvm;
20
21 namespace {
22
23 // Fuse CMN, CMP, TST followed by Bcc.
isArithmeticBccPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)24 static bool isArithmeticBccPair(const MachineInstr *FirstMI,
25 const MachineInstr &SecondMI) {
26 if (SecondMI.getOpcode() == AArch64::Bcc) {
27 // Assume the 1st instr to be a wildcard if it is unspecified.
28 if (!FirstMI)
29 return true;
30
31 switch (FirstMI->getOpcode()) {
32 case AArch64::ADDSWri:
33 case AArch64::ADDSWrr:
34 case AArch64::ADDSXri:
35 case AArch64::ADDSXrr:
36 case AArch64::ANDSWri:
37 case AArch64::ANDSWrr:
38 case AArch64::ANDSXri:
39 case AArch64::ANDSXrr:
40 case AArch64::SUBSWri:
41 case AArch64::SUBSWrr:
42 case AArch64::SUBSXri:
43 case AArch64::SUBSXrr:
44 case AArch64::BICSWrr:
45 case AArch64::BICSXrr:
46 return true;
47 case AArch64::ADDSWrs:
48 case AArch64::ADDSXrs:
49 case AArch64::ANDSWrs:
50 case AArch64::ANDSXrs:
51 case AArch64::SUBSWrs:
52 case AArch64::SUBSXrs:
53 case AArch64::BICSWrs:
54 case AArch64::BICSXrs:
55 // Shift value can be 0 making these behave like the "rr" variant...
56 return (!AArch64InstrInfo::hasShiftedReg(*FirstMI));
57 }
58 }
59 return false;
60 }
61
62 // Fuse ALU operations followed by CBZ/CBNZ.
isArithmeticCbzPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)63 static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
64 const MachineInstr &SecondMI) {
65 unsigned SecondOpcode = SecondMI.getOpcode();
66
67 if (SecondOpcode == AArch64::CBNZW || SecondOpcode == AArch64::CBNZX ||
68 SecondOpcode == AArch64::CBZW || SecondOpcode == AArch64::CBZX) {
69 // Assume the 1st instr to be a wildcard if it is unspecified.
70 if (!FirstMI)
71 return true;
72
73 switch (FirstMI->getOpcode()) {
74 case AArch64::ADDWri:
75 case AArch64::ADDWrr:
76 case AArch64::ADDXri:
77 case AArch64::ADDXrr:
78 case AArch64::ANDWri:
79 case AArch64::ANDWrr:
80 case AArch64::ANDXri:
81 case AArch64::ANDXrr:
82 case AArch64::EORWri:
83 case AArch64::EORWrr:
84 case AArch64::EORXri:
85 case AArch64::EORXrr:
86 case AArch64::ORRWri:
87 case AArch64::ORRWrr:
88 case AArch64::ORRXri:
89 case AArch64::ORRXrr:
90 case AArch64::SUBWri:
91 case AArch64::SUBWrr:
92 case AArch64::SUBXri:
93 case AArch64::SUBXrr:
94 return true;
95 case AArch64::ADDWrs:
96 case AArch64::ADDXrs:
97 case AArch64::ANDWrs:
98 case AArch64::ANDXrs:
99 case AArch64::SUBWrs:
100 case AArch64::SUBXrs:
101 case AArch64::BICWrs:
102 case AArch64::BICXrs:
103 // Shift value can be 0 making these behave like the "rr" variant...
104 return (!AArch64InstrInfo::hasShiftedReg(*FirstMI));
105 }
106 }
107 return false;
108 }
109
110 // Fuse AES crypto encoding or decoding.
isAESPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)111 static bool isAESPair(const MachineInstr *FirstMI,
112 const MachineInstr &SecondMI) {
113 // Assume the 1st instr to be a wildcard if it is unspecified.
114 unsigned FirstOpcode =
115 FirstMI ? FirstMI->getOpcode()
116 : static_cast<unsigned>(AArch64::INSTRUCTION_LIST_END);
117 unsigned SecondOpcode = SecondMI.getOpcode();
118
119 // AES encode.
120 if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
121 FirstOpcode == AArch64::AESErr) &&
122 (SecondOpcode == AArch64::AESMCrr ||
123 SecondOpcode == AArch64::AESMCrrTied))
124 return true;
125 // AES decode.
126 else if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
127 FirstOpcode == AArch64::AESDrr) &&
128 (SecondOpcode == AArch64::AESIMCrr ||
129 SecondOpcode == AArch64::AESIMCrrTied))
130 return true;
131
132 return false;
133 }
134
135 // Fuse literal generation.
isLiteralsPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)136 static bool isLiteralsPair(const MachineInstr *FirstMI,
137 const MachineInstr &SecondMI) {
138 // Assume the 1st instr to be a wildcard if it is unspecified.
139 unsigned FirstOpcode =
140 FirstMI ? FirstMI->getOpcode()
141 : static_cast<unsigned>(AArch64::INSTRUCTION_LIST_END);
142 unsigned SecondOpcode = SecondMI.getOpcode();
143
144 // PC relative address.
145 if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
146 FirstOpcode == AArch64::ADRP) &&
147 SecondOpcode == AArch64::ADDXri)
148 return true;
149 // 32 bit immediate.
150 else if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
151 FirstOpcode == AArch64::MOVZWi) &&
152 (SecondOpcode == AArch64::MOVKWi &&
153 SecondMI.getOperand(3).getImm() == 16))
154 return true;
155 // Lower half of 64 bit immediate.
156 else if((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
157 FirstOpcode == AArch64::MOVZXi) &&
158 (SecondOpcode == AArch64::MOVKXi &&
159 SecondMI.getOperand(3).getImm() == 16))
160 return true;
161 // Upper half of 64 bit immediate.
162 else if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
163 (FirstOpcode == AArch64::MOVKXi &&
164 FirstMI->getOperand(3).getImm() == 32)) &&
165 (SecondOpcode == AArch64::MOVKXi &&
166 SecondMI.getOperand(3).getImm() == 48))
167 return true;
168
169 return false;
170 }
171
172 // Fuse address generation and loads or stores.
isAddressLdStPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)173 static bool isAddressLdStPair(const MachineInstr *FirstMI,
174 const MachineInstr &SecondMI) {
175 unsigned SecondOpcode = SecondMI.getOpcode();
176
177 switch (SecondOpcode) {
178 case AArch64::STRBBui:
179 case AArch64::STRBui:
180 case AArch64::STRDui:
181 case AArch64::STRHHui:
182 case AArch64::STRHui:
183 case AArch64::STRQui:
184 case AArch64::STRSui:
185 case AArch64::STRWui:
186 case AArch64::STRXui:
187 case AArch64::LDRBBui:
188 case AArch64::LDRBui:
189 case AArch64::LDRDui:
190 case AArch64::LDRHHui:
191 case AArch64::LDRHui:
192 case AArch64::LDRQui:
193 case AArch64::LDRSui:
194 case AArch64::LDRWui:
195 case AArch64::LDRXui:
196 case AArch64::LDRSBWui:
197 case AArch64::LDRSBXui:
198 case AArch64::LDRSHWui:
199 case AArch64::LDRSHXui:
200 case AArch64::LDRSWui:
201 // Assume the 1st instr to be a wildcard if it is unspecified.
202 if (!FirstMI)
203 return true;
204
205 switch (FirstMI->getOpcode()) {
206 case AArch64::ADR:
207 return (SecondMI.getOperand(2).getImm() == 0);
208 case AArch64::ADRP:
209 return true;
210 }
211 }
212 return false;
213 }
214
215 // Fuse compare and conditional select.
isCCSelectPair(const MachineInstr * FirstMI,const MachineInstr & SecondMI)216 static bool isCCSelectPair(const MachineInstr *FirstMI,
217 const MachineInstr &SecondMI) {
218 unsigned SecondOpcode = SecondMI.getOpcode();
219
220 // 32 bits
221 if (SecondOpcode == AArch64::CSELWr) {
222 // Assume the 1st instr to be a wildcard if it is unspecified.
223 if (!FirstMI)
224 return true;
225
226 if (FirstMI->definesRegister(AArch64::WZR))
227 switch (FirstMI->getOpcode()) {
228 case AArch64::SUBSWrs:
229 return (!AArch64InstrInfo::hasShiftedReg(*FirstMI));
230 case AArch64::SUBSWrx:
231 return (!AArch64InstrInfo::hasExtendedReg(*FirstMI));
232 case AArch64::SUBSWrr:
233 case AArch64::SUBSWri:
234 return true;
235 }
236 }
237 // 64 bits
238 else if (SecondOpcode == AArch64::CSELXr) {
239 // Assume the 1st instr to be a wildcard if it is unspecified.
240 if (!FirstMI)
241 return true;
242
243 if (FirstMI->definesRegister(AArch64::XZR))
244 switch (FirstMI->getOpcode()) {
245 case AArch64::SUBSXrs:
246 return (!AArch64InstrInfo::hasShiftedReg(*FirstMI));
247 case AArch64::SUBSXrx:
248 case AArch64::SUBSXrx64:
249 return (!AArch64InstrInfo::hasExtendedReg(*FirstMI));
250 case AArch64::SUBSXrr:
251 case AArch64::SUBSXri:
252 return true;
253 }
254 }
255 return false;
256 }
257
258 /// Check if the instr pair, FirstMI and SecondMI, should be fused
259 /// together. Given SecondMI, when FirstMI is unspecified, then check if
260 /// SecondMI may be part of a fused pair at all.
shouldScheduleAdjacent(const TargetInstrInfo & TII,const TargetSubtargetInfo & TSI,const MachineInstr * FirstMI,const MachineInstr & SecondMI)261 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
262 const TargetSubtargetInfo &TSI,
263 const MachineInstr *FirstMI,
264 const MachineInstr &SecondMI) {
265 const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
266
267 if (ST.hasArithmeticBccFusion() && isArithmeticBccPair(FirstMI, SecondMI))
268 return true;
269 if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
270 return true;
271 if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
272 return true;
273 if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
274 return true;
275 if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
276 return true;
277 if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
278 return true;
279
280 return false;
281 }
282
283 } // end namespace
284
285
286 namespace llvm {
287
createAArch64MacroFusionDAGMutation()288 std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation () {
289 return createMacroFusionDAGMutation(shouldScheduleAdjacent);
290 }
291
292 } // end namespace llvm
293