1 /*
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "aarch64_peep.h"
17 #include "aarch64_cg.h"
18
19 namespace maplebe {
20 #define CG_PEEP_DUMP CG_DEBUG_FUNC(*cgFunc)
21
IsZeroRegister(const Operand & opnd)22 static bool IsZeroRegister(const Operand &opnd)
23 {
24 if (!opnd.IsRegister()) {
25 return false;
26 }
27 const RegOperand *regOpnd = static_cast<const RegOperand *>(&opnd);
28 return regOpnd->GetRegisterNumber() == RZR;
29 }
30
Run()31 void AArch64CGPeepHole::Run()
32 {
33 FOR_ALL_BB(bb, cgFunc)
34 {
35 FOR_BB_INSNS_SAFE(insn, bb, nextInsn)
36 {
37 if (!insn->IsMachineInstruction()) {
38 continue;
39 }
40 DoNormalOptimize(*bb, *insn);
41 }
42 }
43 }
44
DoNormalOptimize(BB & bb,Insn & insn)45 void AArch64CGPeepHole::DoNormalOptimize(BB &bb, Insn &insn)
46 {
47 MOperator thisMop = insn.GetMachineOpcode();
48 manager = peepMemPool->New<PeepOptimizeManager>(*cgFunc, bb, insn);
49 switch (thisMop) {
50 /*
51 * e.g.
52 * execute before & after RA: manager->NormalPatternOpt<>(true)
53 * execute before RA: manager->NormalPatternOpt<>(!cgFunc->IsAfterRegAlloc())
54 * execute after RA: manager->NormalPatternOpt<>(cgFunc->IsAfterRegAlloc())
55 */
56 case MOP_wmovrr:
57 case MOP_xmovrr:
58 case MOP_xvmovs:
59 case MOP_xvmovd: {
60 manager->NormalPatternOpt<RemoveMovingtoSameRegPattern>(cgFunc->IsAfterRegAlloc());
61 break;
62 }
63 case MOP_wstrb:
64 case MOP_wldrb: {
65 // only strb ldrb can do this pattern, other patterns still need to be done, so there is no break here.
66 break;
67 }
68 case MOP_wstrh:
69 case MOP_wldrh:
70 case MOP_xldr:
71 case MOP_xstr:
72 case MOP_wldr:
73 case MOP_wstr:
74 case MOP_dldr:
75 case MOP_dstr:
76 case MOP_sldr:
77 case MOP_sstr:
78 case MOP_qldr:
79 case MOP_qstr: {
80 manager->NormalPatternOpt<CombineContiLoadAndStorePattern>(cgFunc->IsAfterRegAlloc());
81 if (!manager->OptSuccess()) {
82 manager->NormalPatternOpt<ContiLDRorSTRToSameMEMPattern>(cgFunc->IsAfterRegAlloc());
83 }
84 break;
85 }
86 case MOP_xvmovrv:
87 case MOP_xvmovrd: {
88 manager->NormalPatternOpt<FmovRegPattern>(cgFunc->IsAfterRegAlloc());
89 break;
90 }
91 case MOP_xsbfxrri6i6: {
92 manager->NormalPatternOpt<SbfxOptPattern>(cgFunc->IsAfterRegAlloc());
93 break;
94 }
95 case MOP_wcbz:
96 case MOP_xcbz:
97 case MOP_wcbnz:
98 case MOP_xcbnz: {
99 manager->NormalPatternOpt<CbnzToCbzPattern>(cgFunc->IsAfterRegAlloc());
100 break;
101 }
102 default:
103 break;
104 }
105 }
106 /* ======== CGPeepPattern End ======== */
107
InitOpts()108 void AArch64PeepHole::InitOpts()
109 {
110 optimizations.resize(kPeepholeOptsNum);
111 optimizations[kEliminateSpecifcSXTOpt] = optOwnMemPool->New<EliminateSpecifcSXTAArch64>(cgFunc);
112 optimizations[kEliminateSpecifcUXTOpt] = optOwnMemPool->New<EliminateSpecifcUXTAArch64>(cgFunc);
113 optimizations[kCsetCbzToBeqOpt] = optOwnMemPool->New<CsetCbzToBeqOptAArch64>(cgFunc);
114 optimizations[kAndCmpBranchesToCsetOpt] = optOwnMemPool->New<AndCmpBranchesToCsetAArch64>(cgFunc);
115 optimizations[kAndCmpBranchesToTstOpt] = optOwnMemPool->New<AndCmpBranchesToTstAArch64>(cgFunc);
116 optimizations[kAndCbzBranchesToTstOpt] = optOwnMemPool->New<AndCbzBranchesToTstAArch64>(cgFunc);
117 optimizations[kZeroCmpBranchesOpt] = optOwnMemPool->New<ZeroCmpBranchesAArch64>(cgFunc);
118 optimizations[kCselZeroOneToCsetOpt] = optOwnMemPool->New<CselZeroOneToCsetOpt>(cgFunc);
119 optimizations[kAndCmpCsetEorCbzOpt] = optOwnMemPool->New<AndCmpCsetEorCbzOpt>(cgFunc);
120 optimizations[kAddLdrOpt] = optOwnMemPool->New<AddLdrOpt>(cgFunc);
121 optimizations[kCsetEorOpt] = optOwnMemPool->New<CsetEorOpt>(cgFunc);
122 optimizations[kMoveCmpOpt] = optOwnMemPool->New<MoveCmpOpt>(cgFunc);
123 optimizations[kCmpZeroBranch] = optOwnMemPool->New<CmpZeroBranch>(cgFunc);
124 }
125
Run(BB & bb,Insn & insn)126 void AArch64PeepHole::Run(BB &bb, Insn &insn)
127 {
128 MOperator thisMop = insn.GetMachineOpcode();
129 switch (thisMop) {
130 case MOP_xsxtb32:
131 case MOP_xsxth32:
132 case MOP_xsxtb64:
133 case MOP_xsxth64:
134 case MOP_xsxtw64: {
135 (static_cast<EliminateSpecifcSXTAArch64 *>(optimizations[kEliminateSpecifcSXTOpt]))->Run(bb, insn);
136 break;
137 }
138 case MOP_xuxtb32:
139 case MOP_xuxth32:
140 case MOP_xuxtw64: {
141 (static_cast<EliminateSpecifcUXTAArch64 *>(optimizations[kEliminateSpecifcUXTOpt]))->Run(bb, insn);
142 break;
143 }
144 case MOP_wcmpri:
145 case MOP_xcmpri: {
146 (static_cast<CmpZeroBranch *>(optimizations[kCmpZeroBranch]))->Run(bb, insn);
147 break;
148 }
149 case MOP_wcbnz:
150 case MOP_xcbnz: {
151 (static_cast<CsetCbzToBeqOptAArch64 *>(optimizations[kCsetCbzToBeqOpt]))->Run(bb, insn);
152 break;
153 }
154 case MOP_wcbz:
155 case MOP_xcbz: {
156 (static_cast<CsetCbzToBeqOptAArch64 *>(optimizations[kCsetCbzToBeqOpt]))->Run(bb, insn);
157 break;
158 }
159 case MOP_xandrrr:
160 case MOP_wandrrr:
161 case MOP_wandrri12:
162 case MOP_xandrri13: {
163 (static_cast<AndCmpCsetEorCbzOpt *>(optimizations[kAndCmpCsetEorCbzOpt]))->Run(bb, insn);
164 (static_cast<AndCmpBranchesToTstAArch64 *>(optimizations[kAndCmpBranchesToTstOpt]))->Run(bb, insn);
165 (static_cast<AndCbzBranchesToTstAArch64 *>(optimizations[kAndCbzBranchesToTstOpt]))->Run(bb, insn);
166 break;
167 }
168 case MOP_wcsetrc:
169 case MOP_xcsetrc: {
170 (static_cast<CsetEorOpt *>(optimizations[kCsetEorOpt]))->Run(bb, insn);
171 (static_cast<AndCmpBranchesToCsetAArch64 *>(optimizations[kAndCmpBranchesToCsetOpt]))->Run(bb, insn);
172 break;
173 }
174 case MOP_xmovri64:
175 case MOP_wmovri32: {
176 static_cast<MoveCmpOpt *>(optimizations[kMoveCmpOpt])->Run(bb, insn);
177 break;
178 }
179 case MOP_xaddrrr: {
180 (static_cast<AddLdrOpt *>(optimizations[kAddLdrOpt]))->Run(bb, insn);
181 break;
182 }
183 case MOP_wcselrrrc:
184 case MOP_xcselrrrc: {
185 (static_cast<CselZeroOneToCsetOpt *>(optimizations[kCselZeroOneToCsetOpt]))->Run(bb, insn);
186 break;
187 }
188 default:
189 break;
190 }
191 if (&insn == bb.GetLastInsn()) {
192 (static_cast<ZeroCmpBranchesAArch64 *>(optimizations[kZeroCmpBranchesOpt]))->Run(bb, insn);
193 }
194 }
195
InitOpts()196 void AArch64PeepHole0::InitOpts()
197 {
198 optimizations.resize(kPeepholeOptsNum);
199 optimizations[kRemoveIdenticalLoadAndStoreOpt] = optOwnMemPool->New<RemoveIdenticalLoadAndStoreAArch64>(cgFunc);
200 optimizations[kCmpCsetOpt] = optOwnMemPool->New<CmpCsetAArch64>(cgFunc);
201 optimizations[kComplexMemOperandOptAdd] = optOwnMemPool->New<ComplexMemOperandAddAArch64>(cgFunc);
202 optimizations[kRemoveSxtBeforeStrOpt] = optOwnMemPool->New<RemoveSxtBeforeStrAArch64>(cgFunc);
203 optimizations[kRedundantMovAArch64Opt] = optOwnMemPool->New<RedundantMovAArch64>(cgFunc);
204 optimizations[kRemoveMovingtoSameRegOpt] = optOwnMemPool->New<RemoveMovingtoSameRegAArch64>(cgFunc);
205 optimizations[kEnhanceStrLdrAArch64Opt] = optOwnMemPool->New<EnhanceStrLdrAArch64>(cgFunc);
206 optimizations[kAddImmZeroToMov] = optOwnMemPool->New<AddImmZeroToMov>(cgFunc);
207 }
208
Run(BB & bb,Insn & insn)209 void AArch64PeepHole0::Run(BB &bb, Insn &insn)
210 {
211 MOperator thisMop = insn.GetMachineOpcode();
212 switch (thisMop) {
213 case MOP_wcmpri:
214 case MOP_xcmpri: {
215 (static_cast<CmpCsetAArch64 *>(optimizations[kCmpCsetOpt]))->Run(bb, insn);
216 break;
217 }
218 case MOP_xaddrrr: {
219 (static_cast<ComplexMemOperandAddAArch64 *>(optimizations[kComplexMemOperandOptAdd]))->Run(bb, insn);
220 break;
221 }
222 case MOP_xaddrri12: {
223 (static_cast<AddImmZeroToMov *>(optimizations[kAddImmZeroToMov]))->Run(bb, insn);
224 break;
225 }
226 case MOP_wstrh:
227 case MOP_wstrb: {
228 (static_cast<RemoveSxtBeforeStrAArch64 *>(optimizations[kRemoveSxtBeforeStrOpt]))->Run(bb, insn);
229 break;
230 }
231 case MOP_wmovrr:
232 case MOP_xmovrr:
233 case MOP_xvmovs:
234 case MOP_xvmovd: {
235 (static_cast<RedundantMovAArch64 *>(optimizations[kRedundantMovAArch64Opt]))->Run(bb, insn);
236 (static_cast<RemoveMovingtoSameRegAArch64 *>(optimizations[kRemoveMovingtoSameRegOpt]))->Run(bb, insn);
237 break;
238 }
239 case MOP_xldr:
240 case MOP_xstr:
241 case MOP_wldr:
242 case MOP_wstr:
243 case MOP_dldr:
244 case MOP_dstr:
245 case MOP_sldr:
246 case MOP_sstr: {
247 if (thisMop == MOP_wstr || thisMop == MOP_xstr) {
248 (static_cast<RemoveIdenticalLoadAndStoreAArch64 *>(optimizations[kRemoveIdenticalLoadAndStoreOpt]))
249 ->Run(bb, insn);
250 }
251 (static_cast<EnhanceStrLdrAArch64 *>(optimizations[kEnhanceStrLdrAArch64Opt]))->Run(bb, insn);
252 break;
253 }
254 default:
255 break;
256 }
257 }
258
IsMemOperandsIdentical(const Insn & insn1,const Insn & insn2) const259 bool RemoveIdenticalLoadAndStoreAArch64::IsMemOperandsIdentical(const Insn &insn1, const Insn &insn2) const
260 {
261 regno_t regNO1 = static_cast<RegOperand &>(insn1.GetOperand(kInsnFirstOpnd)).GetRegisterNumber();
262 regno_t regNO2 = static_cast<RegOperand &>(insn2.GetOperand(kInsnFirstOpnd)).GetRegisterNumber();
263 if (regNO1 != regNO2) {
264 return false;
265 }
266 /* Match only [base + offset] */
267 auto &memOpnd1 = static_cast<MemOperand &>(insn1.GetOperand(kInsnSecondOpnd));
268 if (memOpnd1.GetAddrMode() != MemOperand::kAddrModeBOi || !memOpnd1.IsIntactIndexed()) {
269 return false;
270 }
271 auto &memOpnd2 = static_cast<MemOperand &>(insn2.GetOperand(kInsnSecondOpnd));
272 if (memOpnd2.GetAddrMode() != MemOperand::kAddrModeBOi || !memOpnd1.IsIntactIndexed()) {
273 return false;
274 }
275 Operand *base1 = memOpnd1.GetBaseRegister();
276 Operand *base2 = memOpnd2.GetBaseRegister();
277 if (!((base1 != nullptr) && base1->IsRegister()) || !((base2 != nullptr) && base2->IsRegister())) {
278 return false;
279 }
280
281 regno_t baseRegNO1 = static_cast<RegOperand *>(base1)->GetRegisterNumber();
282 /* First insn re-write base addr reg1 <- [ reg1 + offset ] */
283 if (baseRegNO1 == regNO1) {
284 return false;
285 }
286
287 regno_t baseRegNO2 = static_cast<RegOperand *>(base2)->GetRegisterNumber();
288 if (baseRegNO1 != baseRegNO2) {
289 return false;
290 }
291
292 return memOpnd1.GetOffsetImmediate()->GetOffsetValue() == memOpnd2.GetOffsetImmediate()->GetOffsetValue();
293 }
294
Run(BB & bb,Insn & insn)295 void RemoveIdenticalLoadAndStoreAArch64::Run(BB &bb, Insn &insn)
296 {
297 Insn *nextInsn = insn.GetNextMachineInsn();
298 if (nextInsn == nullptr) {
299 return;
300 }
301 MOperator mop1 = insn.GetMachineOpcode();
302 MOperator mop2 = nextInsn->GetMachineOpcode();
303 if ((mop1 == MOP_wstr && mop2 == MOP_wstr) || (mop1 == MOP_xstr && mop2 == MOP_xstr)) {
304 if (IsMemOperandsIdentical(insn, *nextInsn)) {
305 bb.RemoveInsn(insn);
306 }
307 } else if ((mop1 == MOP_wstr && mop2 == MOP_wldr) || (mop1 == MOP_xstr && mop2 == MOP_xldr)) {
308 if (IsMemOperandsIdentical(insn, *nextInsn)) {
309 bb.RemoveInsn(*nextInsn);
310 }
311 }
312 }
313
CheckCondition(Insn & insn)314 bool RemoveMovingtoSameRegPattern::CheckCondition(Insn &insn)
315 {
316 DEBUG_ASSERT(insn.GetOperand(kInsnFirstOpnd).IsRegister(), "expects registers");
317 DEBUG_ASSERT(insn.GetOperand(kInsnSecondOpnd).IsRegister(), "expects registers");
318 auto ®1 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
319 auto ®2 = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
320 /* remove mov x0,x0 when it cast i32 to i64 */
321 if ((reg1.GetRegisterNumber() == reg2.GetRegisterNumber()) && (reg1.GetSize() >= reg2.GetSize())) {
322 return true;
323 }
324 return false;
325 }
326
Run(BB & bb,Insn & insn)327 void RemoveMovingtoSameRegPattern::Run(BB &bb, Insn &insn)
328 {
329 /* remove mov x0,x0 when it cast i32 to i64 */
330 if (CheckCondition(insn)) {
331 bb.RemoveInsn(insn);
332 }
333 }
334
Run(BB & bb,Insn & insn)335 void RedundantMovAArch64::Run(BB &bb, Insn &insn)
336 {
337 auto *prevInsn = insn.GetPreviousMachineInsn();
338 if (prevInsn == nullptr) {
339 return;
340 }
341 auto *prevOpndDes = prevInsn->GetDesc()->GetOpndDes(kInsnFirstOpnd);
342 auto *srcOpndDesc = insn.GetDesc()->GetOpndDes(kInsnSecondOpnd);
343 if (!prevOpndDes->IsRegDef() || prevOpndDes->IsUse() || prevOpndDes->GetSize() > srcOpndDesc->GetSize()) {
344 return;
345 }
346 auto &srcOpnd = static_cast<RegOperand&>(insn.GetOperand(kInsnSecondOpnd));
347 auto &prevDestOpnd = static_cast<RegOperand&>(prevInsn->GetOperand(kInsnFirstOpnd));
348 if (srcOpnd.GetRegisterNumber() != prevDestOpnd.GetRegisterNumber()) {
349 return;
350 }
351 if (IfOperandIsLiveAfterInsn(srcOpnd, insn)) {
352 return;
353 }
354 auto &desOpnd = static_cast<RegOperand&>(insn.GetOperand(kInsnFirstOpnd));
355 prevInsn->SetOperand(kInsnFirstOpnd, desOpnd);
356 bb.RemoveInsn(insn);
357 }
358
Run(BB & bb,Insn & insn)359 void RemoveMovingtoSameRegAArch64::Run(BB &bb, Insn &insn)
360 {
361 DEBUG_ASSERT(insn.GetOperand(kInsnFirstOpnd).IsRegister(), "expects registers");
362 DEBUG_ASSERT(insn.GetOperand(kInsnSecondOpnd).IsRegister(), "expects registers");
363 auto ®1 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
364 auto ®2 = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
365 /* remove mov x0,x0 when it cast i32 to i64 */
366 if ((reg1.GetRegisterNumber() == reg2.GetRegisterNumber()) && (reg1.GetSize() >= reg2.GetSize())) {
367 bb.RemoveInsn(insn);
368 }
369 }
370
CheckOperandIsDeadFromInsn(const RegOperand & regOpnd,Insn & insn)371 bool EnhanceStrLdrAArch64::CheckOperandIsDeadFromInsn(const RegOperand ®Opnd, Insn &insn)
372 {
373 for (uint32 i = 0; i < insn.GetOperandSize(); ++i) {
374 auto &opnd = insn.GetOperand(i);
375 if (!insn.GetDesc()->GetOpndDes(i)->IsRegDef()) {
376 continue;
377 }
378 // regOpnd is redefined at curInsn
379 if (static_cast<RegOperand &>(opnd).GetRegisterNumber() == regOpnd.GetRegisterNumber()) {
380 return true;
381 }
382 }
383 return !IfOperandIsLiveAfterInsn(regOpnd, insn);
384 }
385
GetInsnAddOrSubNewOffset(Insn & insn,ImmOperand & offset)386 ImmOperand *EnhanceStrLdrAArch64::GetInsnAddOrSubNewOffset(Insn &insn, ImmOperand &offset)
387 {
388 int64 val = 0;
389 VaryType vary = offset.GetVary();
390 auto mOp = insn.GetMachineOpcode();
391 if (mOp == MOP_xaddrri12 || mOp == MOP_xsubrri12) {
392 auto &immOpnd = static_cast<ImmOperand &>(insn.GetOperand(kInsnThirdOpnd));
393 val = immOpnd.GetValue();
394 CHECK_FATAL(!(vary == kUnAdjustVary && immOpnd.GetVary() == kUnAdjustVary), "NIY, can not deal this case!");
395 vary = immOpnd.GetVary();
396 } else {
397 auto &immOpnd = static_cast<ImmOperand &>(insn.GetOperand(kInsnThirdOpnd));
398 auto &shiftOpnd = static_cast<BitShiftOperand &>(insn.GetOperand(kInsnFourthOpnd));
399 CHECK_FATAL(shiftOpnd.GetShiftAmount() == 12, "invalid shiftAmount"); // 12: invalid shiftAmount
400 val = (immOpnd.GetValue() << shiftOpnd.GetShiftAmount());
401 }
402
403 if (mOp == MOP_xsubrri12 || mOp == MOP_xsubrri24) {
404 val = -val;
405 }
406 val += offset.GetValue();
407 auto &newImm = static_cast<AArch64CGFunc &>(cgFunc).GetOrCreateOfstOpnd(val, k64BitSize);
408 newImm.SetVary(vary);
409 return &newImm;
410 }
411
OptimizeAddrBOI(Insn & insn,MemOperand & memOpnd,Insn & prevInsn)412 void EnhanceStrLdrAArch64::OptimizeAddrBOI(Insn &insn, MemOperand &memOpnd, Insn &prevInsn)
413 {
414 auto *oriBase = memOpnd.GetBaseRegister();
415 auto *oriOffset = memOpnd.GetOffsetOperand();
416 auto &defOpnd = static_cast<RegOperand &>(prevInsn.GetOperand(kInsnFirstOpnd));
417 if (defOpnd.GetRegisterNumber() != oriBase->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) {
418 return;
419 }
420 auto *newBase = static_cast<RegOperand *>(&prevInsn.GetOperand(kInsnSecondOpnd));
421 auto *newOffset = GetInsnAddOrSubNewOffset(prevInsn, *memOpnd.GetOffsetOperand());
422 if (newOffset->GetValue() < 0) {
423 return; // obj dump cannot deal str x19, [x29,#-16]
424 }
425
426 memOpnd.SetBaseRegister(*newBase);
427 memOpnd.SetOffsetOperand(*newOffset);
428 if (!static_cast<AArch64CGFunc &>(cgFunc).IsOperandImmValid(insn.GetMachineOpcode(), &memOpnd, kInsnSecondOpnd)) {
429 // If new offset is invalid, undo it
430 memOpnd.SetBaseRegister(*oriBase);
431 memOpnd.SetOffsetOperand(*oriOffset);
432 return;
433 }
434 memOpnd.SetAddrMode(MemOperand::kAddrModeBOi);
435 prevInsn.GetBB()->RemoveInsn(prevInsn);
436 }
437
OptimizeAddrBOrXShiftExtend(Insn & insn,MemOperand & memOpnd,Insn & shiftExtendInsn)438 void EnhanceStrLdrAArch64::OptimizeAddrBOrXShiftExtend(Insn &insn, MemOperand &memOpnd, Insn &shiftExtendInsn)
439 {
440 auto mOp = shiftExtendInsn.GetMachineOpcode();
441 if (mOp != MOP_xuxtw64 && mOp != MOP_xsxtw64 && mOp != MOP_xlslrri6) {
442 return;
443 }
444 auto *oriIndex = memOpnd.GetIndexRegister();
445 auto &defOpnd = static_cast<RegOperand &>(shiftExtendInsn.GetOperand(kInsnFirstOpnd));
446 if (defOpnd.GetRegisterNumber() != oriIndex->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) {
447 return;
448 }
449 auto &newIndex = static_cast<RegOperand &>(shiftExtendInsn.GetOperand(kInsnSecondOpnd));
450 bool isSigned = (mOp == MOP_xsxtw64);
451 uint32 shift = 0;
452 if (mOp == MOP_xlslrri6) {
453 shift = static_cast<uint32>(static_cast<ImmOperand &>(shiftExtendInsn.GetOperand(kInsnThirdOpnd)).GetValue());
454 }
455 const uint32 regSize = insn.GetDesc()->GetOpndDes(kInsnFirstOpnd)->GetSize();
456 // lsl extend insn shift amount can only be 0 or 1(16-bit def opnd) or 2(32-bit def opnd) or
457 // 3(64-bit def opnd) or 4(128-bit def opnd) in ldr/str insn, and in this pattern we only have
458 // 32-bit & 64-bit situation now
459 if ((shift == k0BitSize) || (regSize == k32BitSize && shift == k2BitSize) ||
460 (regSize == k64BitSize && shift == k3BitSize)) {
461 auto *newMemOpnd = static_cast<AArch64CGFunc &>(cgFunc).CreateMemOperand(
462 MemOperand::kAddrModeBOrX, memOpnd.GetSize(), *memOpnd.GetBaseRegister(), newIndex, shift, isSigned);
463 insn.SetOperand(kInsnSecondOpnd, *newMemOpnd);
464 shiftExtendInsn.GetBB()->RemoveInsn(shiftExtendInsn);
465 }
466 }
467
OptimizeAddrBOrX(Insn & insn,MemOperand & memOpnd,Insn & prevInsn)468 void EnhanceStrLdrAArch64::OptimizeAddrBOrX(Insn &insn, MemOperand &memOpnd, Insn &prevInsn)
469 {
470 if (memOpnd.GetOffsetOperand()->GetValue() != 0 || memOpnd.GetOffsetOperand()->GetVary() == kUnAdjustVary) {
471 return;
472 }
473 auto *oriBase = memOpnd.GetBaseRegister();
474 auto &defOpnd = static_cast<RegOperand &>(prevInsn.GetOperand(kInsnFirstOpnd));
475 if (defOpnd.GetRegisterNumber() != oriBase->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) {
476 return;
477 }
478 auto *newBase = static_cast<RegOperand *>(&prevInsn.GetOperand(kInsnSecondOpnd));
479 auto *newIndex = static_cast<RegOperand *>(&prevInsn.GetOperand(kInsnThirdOpnd));
480
481 memOpnd.SetBaseRegister(*newBase);
482 memOpnd.SetIndexRegister(*newIndex);
483 memOpnd.SetAddrMode(MemOperand::kAddrModeBOrX);
484 auto *prevShiftExtendInsn = prevInsn.GetPreviousMachineInsn();
485 if (prevShiftExtendInsn != nullptr) {
486 OptimizeAddrBOrXShiftExtend(insn, memOpnd, *prevShiftExtendInsn);
487 }
488 prevInsn.GetBB()->RemoveInsn(prevInsn);
489 }
490
OptimizeWithAddrrrs(Insn & insn,MemOperand & memOpnd,Insn & addInsn)491 void EnhanceStrLdrAArch64::OptimizeWithAddrrrs(Insn &insn, MemOperand &memOpnd, Insn &addInsn)
492 {
493 if (memOpnd.GetOffsetOperand()->GetValue() != 0 || memOpnd.GetOffsetOperand()->GetVary() != kNotVary) {
494 return;
495 }
496 auto *oriBase = memOpnd.GetBaseRegister();
497 auto &defOpnd = static_cast<RegOperand &>(addInsn.GetOperand(kInsnFirstOpnd));
498 if (defOpnd.GetRegisterNumber() != oriBase->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) {
499 return;
500 }
501 auto &newBase = static_cast<RegOperand &>(addInsn.GetOperand(kInsnSecondOpnd));
502 auto &newIndex = static_cast<RegOperand &>(addInsn.GetOperand(kInsnThirdOpnd));
503 auto &shift = static_cast<BitShiftOperand &>(addInsn.GetOperand(kInsnFourthOpnd));
504 if (shift.GetShiftOp() != BitShiftOperand::kLSL) {
505 return;
506 }
507 auto *newMemOpnd = static_cast<AArch64CGFunc &>(cgFunc).CreateMemOperand(
508 MemOperand::kAddrModeBOrX, memOpnd.GetSize(), newBase, newIndex, shift.GetShiftAmount());
509 insn.SetOperand(kInsnSecondOpnd, *newMemOpnd);
510 addInsn.GetBB()->RemoveInsn(addInsn);
511 }
512
OptimizeAddrBOrXShift(MemOperand & memOpnd,Insn & insn)513 void EnhanceStrLdrAArch64::OptimizeAddrBOrXShift(MemOperand &memOpnd, Insn &insn)
514 {
515 auto *prev = insn.GetPreviousMachineInsn();
516 if (prev == nullptr || prev->GetMachineOpcode() != MOP_xlslrri6) {
517 return;
518 }
519 auto *indexOpnd = memOpnd.GetIndexRegister();
520 if (indexOpnd == nullptr) {
521 return;
522 }
523 auto &lslInsnDefOpnd = static_cast<RegOperand &>(prev->GetOperand(kInsnFirstOpnd));
524 if (lslInsnDefOpnd.GetRegisterNumber() != indexOpnd->GetRegisterNumber() ||
525 !CheckOperandIsDeadFromInsn(lslInsnDefOpnd, insn)) {
526 return;
527 }
528
529 auto &newIndex = static_cast<RegOperand &>(prev->GetOperand(kInsnSecondOpnd));
530 uint32 shift = static_cast<uint32>(static_cast<ImmOperand &>(prev->GetOperand(kInsnThirdOpnd)).GetValue());
531 const uint32 regSize = insn.GetDesc()->GetOpndDes(kInsnFirstOpnd)->GetSize();
532 // lsl extend insn shift amount can only be 0 or 1(16-bit def opnd) or 2(32-bit def opnd) or
533 // 3(64-bit def opnd) or 4(128-bit def opnd) in ldr/str insn, and in this pattern we only have
534 // 32-bit & 64-bit situation now
535 if ((shift == k0BitSize) || (regSize == k32BitSize && shift == k2BitSize) ||
536 (regSize == k64BitSize && shift == k3BitSize)) {
537 auto *newMemOpnd = static_cast<AArch64CGFunc &>(cgFunc).CreateMemOperand(
538 MemOperand::kAddrModeBOrX, memOpnd.GetSize(), *memOpnd.GetBaseRegister(), newIndex, shift, false);
539 insn.SetOperand(kInsnSecondOpnd, *newMemOpnd);
540 prev->GetBB()->RemoveInsn(*prev);
541 }
542 return;
543 }
544
Run(BB & bb,Insn & insn)545 void EnhanceStrLdrAArch64::Run(BB &bb, Insn &insn)
546 {
547 Operand &opnd = insn.GetOperand(kInsnSecondOpnd);
548 CHECK_FATAL(opnd.IsMemoryAccessOperand(), "Unexpected operand in EnhanceStrLdrAArch64");
549 auto &memOpnd = static_cast<MemOperand &>(opnd);
550 if (memOpnd.GetAddrMode() == MemOperand::kAddrModeBOrX) {
551 OptimizeAddrBOrXShift(memOpnd, insn);
552 return;
553 }
554 if (memOpnd.GetAddrMode() != MemOperand::kAddrModeBOi || !memOpnd.GetOffsetImmediate()->IsImmOffset()) {
555 return;
556 }
557
558 auto *prev = insn.GetPreviousMachineInsn();
559 while (prev != nullptr) {
560 if (prev->GetMachineOpcode() == MOP_xmovrr) {
561 auto &defOpnd = static_cast<RegOperand &>(prev->GetOperand(kInsnFirstOpnd));
562 if (defOpnd.GetRegisterNumber() != memOpnd.GetBaseRegister()->GetRegisterNumber() ||
563 !CheckOperandIsDeadFromInsn(defOpnd, insn)) {
564 return;
565 }
566 memOpnd.SetBaseRegister(static_cast<RegOperand &>(prev->GetOperand(kInsnSecondOpnd)));
567 auto *tmpInsn = prev;
568 prev = prev->GetPreviousMachineInsn();
569 tmpInsn->GetBB()->RemoveInsn(*tmpInsn);
570 continue;
571 }
572 break;
573 }
574 if (prev == nullptr) {
575 return;
576 }
577 auto prevMop = prev->GetMachineOpcode();
578 if (prevMop == MOP_xaddrri12 || prevMop == MOP_xsubrri12 || prevMop == MOP_xaddrri24 || prevMop == MOP_xsubrri24) {
579 OptimizeAddrBOI(insn, memOpnd, *prev);
580 } else if (prevMop == MOP_xaddrrr) {
581 OptimizeAddrBOrX(insn, memOpnd, *prev);
582 } else if (prevMop == MOP_xaddrrrs) {
583 OptimizeWithAddrrrs(insn, memOpnd, *prev);
584 }
585 }
586
IsSameRegisterOperation(const RegOperand & desMovOpnd,const RegOperand & uxtDestOpnd,const RegOperand & uxtFromOpnd)587 bool IsSameRegisterOperation(const RegOperand &desMovOpnd, const RegOperand &uxtDestOpnd, const RegOperand &uxtFromOpnd)
588 {
589 return ((desMovOpnd.GetRegisterNumber() == uxtDestOpnd.GetRegisterNumber()) &&
590 (uxtDestOpnd.GetRegisterNumber() == uxtFromOpnd.GetRegisterNumber()));
591 }
592
IsRegNotSameMemUseInInsn(const Insn & checkInsn,const Insn & curInsn,regno_t curBaseRegNO,bool isCurStore,int64 curBaseOfst,int64 curMemRange) const593 bool CombineContiLoadAndStorePattern::IsRegNotSameMemUseInInsn(const Insn &checkInsn, const Insn &curInsn,
594 regno_t curBaseRegNO, bool isCurStore, int64 curBaseOfst,
595 int64 curMemRange) const
596 {
597 uint32 opndNum = checkInsn.GetOperandSize();
598 for (uint32 i = 0; i < opndNum; ++i) {
599 Operand &opnd = checkInsn.GetOperand(i);
600 if (opnd.IsList()) {
601 auto &listOpnd = static_cast<const ListOperand &>(opnd);
602 for (auto &listElem : listOpnd.GetOperands()) {
603 auto *regOpnd = static_cast<RegOperand *>(listElem);
604 DEBUG_ASSERT(regOpnd != nullptr, "parameter operand must be RegOperand");
605 if (curBaseRegNO == regOpnd->GetRegisterNumber()) {
606 return true;
607 }
608 }
609 } else if (opnd.IsMemoryAccessOperand()) {
610 auto &memOperand = static_cast<MemOperand &>(opnd);
611 RegOperand *checkBaseReg = memOperand.GetBaseRegister();
612 // Check memory overlap
613 if ((isCurStore || checkInsn.IsStore()) && checkBaseReg != nullptr &&
614 memOperand.GetAddrMode() == MemOperand::kAddrModeBOi && memOperand.GetOffsetImmediate() != nullptr) {
615 // If memInsn is split with x16, we need to find the actual base register
616 int64 checkOffset = memOperand.GetOffsetImmediate()->GetOffsetValue();
617 regno_t checkRegNO = checkBaseReg->GetRegisterNumber();
618 if (checkRegNO == R16) {
619 const Insn *prevInsn = checkInsn.GetPrev();
620 // Before cgaggressiveopt, the def and use of R16 must be adjacent, and the def of R16 must be
621 // addrri, otherwise, the process is conservative and the mem insn that can be combined is not
622 // search forward.
623 if (prevInsn == nullptr || prevInsn->GetMachineOpcode() != MOP_xaddrri12 ||
624 static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd)).GetRegisterNumber() != R16) {
625 return true;
626 }
627 checkOffset += static_cast<ImmOperand &>(prevInsn->GetOperand(kInsnThirdOpnd)).GetValue();
628 }
629 auto checkMemRange = static_cast<int64>(checkInsn.GetMemoryByteSize());
630 // curOfst curOfst+curMemRange
631 // |______|_/_/_/_/_/_/_/_/_/_/_|____________|
632 if ((curBaseOfst >= checkOffset && curBaseOfst < (checkOffset + checkMemRange)) ||
633 (checkOffset >= curBaseOfst && checkOffset < (curBaseOfst + curMemRange))) {
634 return true;
635 }
636 }
637 } else if (opnd.IsConditionCode()) {
638 auto &rflagOpnd = static_cast<RegOperand &>(cgFunc->GetOrCreateRflag());
639 if (rflagOpnd.GetRegisterNumber() == curBaseRegNO) {
640 return true;
641 }
642 } else if (opnd.IsRegister()) {
643 if (!isCurStore && static_cast<RegOperand &>(opnd).GetRegisterNumber() == curBaseRegNO) {
644 return true;
645 }
646 }
647 }
648 return false;
649 }
650
FindPrevStrLdr(Insn & insn,regno_t destRegNO,regno_t memBaseRegNO,int64 baseOfst) const651 std::vector<Insn *> CombineContiLoadAndStorePattern::FindPrevStrLdr(Insn &insn, regno_t destRegNO, regno_t memBaseRegNO,
652 int64 baseOfst) const
653 {
654 std::vector<Insn *> prevContiInsns;
655 for (Insn *curInsn = insn.GetPrev(); curInsn != nullptr; curInsn = curInsn->GetPrev()) {
656 if (!curInsn->IsMachineInstruction()) {
657 continue;
658 }
659 if (curInsn->IsRegDefined(memBaseRegNO)) {
660 return prevContiInsns;
661 }
662 DEBUG_ASSERT(insn.GetOperand(kInsnSecondOpnd).IsMemoryAccessOperand(), "invalid mem insn");
663 auto baseMemRange = static_cast<int64>(insn.GetMemoryByteSize());
664 if (IsRegNotSameMemUseInInsn(*curInsn, insn, memBaseRegNO, insn.IsStore(), static_cast<int32>(baseOfst),
665 baseMemRange)) {
666 return prevContiInsns;
667 }
668 // record continuous STD/LDR insn
669 if (!curInsn->IsLoadStorePair() &&
670 ((insn.IsStore() && curInsn->IsStore()) || (insn.IsLoad() && curInsn->IsLoad()))) {
671 auto *memOperand = static_cast<MemOperand *>(curInsn->GetMemOpnd());
672 /* do not combine ldr r0, label */
673 if (memOperand != nullptr) {
674 auto *baseRegOpnd = static_cast<RegOperand *>(memOperand->GetBaseRegister());
675 DEBUG_ASSERT(baseRegOpnd == nullptr || !baseRegOpnd->IsVirtualRegister(),
676 "physical register has not been allocated?");
677 if (memOperand->GetAddrMode() == MemOperand::kAddrModeBOi &&
678 baseRegOpnd->GetRegisterNumber() == memBaseRegNO) {
679 prevContiInsns.emplace_back(curInsn);
680 }
681 }
682 }
683 /* check insn that changes the data flow */
684 /* ldr x8, [x21, #8]
685 * call foo()
686 * ldr x9, [x21, #16]
687 * although x21 is a calleeSave register, there is no guarantee data in memory [x21] is not changed
688 */
689 if (curInsn->IsCall() || curInsn->GetMachineOpcode() == MOP_asm) {
690 return prevContiInsns;
691 }
692 /* Check regOpnd for mem access:
693 * 1. if the destRegNO is RZR, we do not need to check define and use for destRegNO between PREVINSN and INSN;
694 * 2. for load insn, we forbid both use and define destRegNO between PREVINSN and INSN;
695 * 3. for store insn, we only forbit define destRegNO between PREVINSN and INSN;
696 * e.g.1
697 * ldr x2, [sp, #16]
698 * add x3, x1, #5 & add x1, x3, #5 ---\--> all [x1] use and define can not across
699 * ldr x1, [sp, #8]
700 * e.g.2
701 * str x2, [sp, #16]
702 * add x1, x3, #5 ---\---> only [x1] define can not across
703 * str x1, [sp, #8]
704 */
705 /* store opt should not cross call due to stack args */
706 if (destRegNO != RZR &&
707 ((insn.IsLoad() && curInsn->ScanReg(destRegNO)) || (insn.IsStore() && curInsn->IsRegDefined(destRegNO)))) {
708 return prevContiInsns;
709 }
710 if (curInsn->ScanReg(destRegNO)) {
711 return prevContiInsns;
712 }
713 }
714 return prevContiInsns;
715 }
716
CheckCondition(Insn & insn)717 bool CombineContiLoadAndStorePattern::CheckCondition(Insn &insn)
718 {
719 MOperator mop = insn.GetMachineOpcode();
720 if (mop == MOP_wldrb || mop == MOP_wldrh) {
721 return false;
722 }
723 auto *curMemOpnd = static_cast<MemOperand *>(insn.GetMemOpnd());
724 DEBUG_ASSERT(curMemOpnd != nullptr, "get mem operand failed");
725 if (!doAggressiveCombine || curMemOpnd->GetAddrMode() != MemOperand::kAddrModeBOi) {
726 return false;
727 }
728 return true;
729 }
730
731 /* Combining 2 STRs into 1 stp or 2 LDRs into 1 ldp */
Run(BB & bb,Insn & insn)732 void CombineContiLoadAndStorePattern::Run(BB &bb, Insn &insn)
733 {
734 if (!CheckCondition(insn)) {
735 return;
736 }
737
738 auto *curMemOpnd = static_cast<MemOperand *>(insn.GetMemOpnd());
739 CHECK_FATAL(curMemOpnd != nullptr, "nullptr check");
740 DEBUG_ASSERT(curMemOpnd->GetAddrMode() == MemOperand::kAddrModeBOi, "invalid continues mem insn");
741 OfstOperand *curOfstOpnd = curMemOpnd->GetOffsetImmediate();
742 int64 curOfstVal = curOfstOpnd ? curOfstOpnd->GetOffsetValue() : 0;
743
744 auto *baseRegOpnd = static_cast<RegOperand *>(curMemOpnd->GetBaseRegister());
745 DEBUG_ASSERT(baseRegOpnd == nullptr || !baseRegOpnd->IsVirtualRegister(),
746 "physical register has not been allocated?");
747 auto &curDestOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
748 std::vector<Insn *> prevContiInsnVec =
749 FindPrevStrLdr(insn, curDestOpnd.GetRegisterNumber(), baseRegOpnd->GetRegisterNumber(), curOfstVal);
750 for (auto prevContiInsn : prevContiInsnVec) {
751 DEBUG_ASSERT(prevContiInsn != nullptr, "get previous consecutive instructions failed");
752 auto *prevMemOpnd = static_cast<MemOperand *>(prevContiInsn->GetMemOpnd());
753 DEBUG_ASSERT(prevMemOpnd->GetAddrMode() == MemOperand::kAddrModeBOi, "invalid continues mem insn");
754 OfstOperand *prevOfstOpnd = prevMemOpnd->GetOffsetImmediate();
755 int64 prevOfstVal = prevOfstOpnd ? prevOfstOpnd->GetOffsetValue() : 0;
756 auto &prevDestOpnd = static_cast<RegOperand &>(prevContiInsn->GetOperand(kInsnFirstOpnd));
757 if (prevDestOpnd.GetRegisterType() != curDestOpnd.GetRegisterType()) {
758 continue;
759 }
760
761 MemOperand *combineMemOpnd = (curOfstVal < prevOfstVal) ? curMemOpnd : prevMemOpnd;
762 if (IsValidNormalLoadOrStorePattern(insn, *prevContiInsn, *curMemOpnd, curOfstVal, prevOfstVal)) {
763 // Process normal mem pair
764 MOperator newMop = GetMopPair(insn.GetMachineOpcode(), true);
765 Insn *combineInsn =
766 GenerateMemPairInsn(newMop, curDestOpnd, prevDestOpnd, *combineMemOpnd, curOfstVal < prevOfstVal);
767 DEBUG_ASSERT(combineInsn != nullptr, "create combineInsn failed");
768 bb.InsertInsnAfter(*prevContiInsn, *combineInsn);
769 if (!(static_cast<AArch64CGFunc &>(*cgFunc).IsOperandImmValid(
770 newMop, combineMemOpnd, isPairAfterCombine ? kInsnThirdOpnd : kInsnSecondOpnd))) {
771 if (FindUseX16AfterInsn(*prevContiInsn)) {
772 // Do not combine Insns when x16 was used after curInsn
773 bb.RemoveInsn(*combineInsn);
774 return;
775 }
776 SPLIT_INSN(combineInsn, cgFunc);
777 }
778 RemoveInsnAndKeepComment(bb, insn, *prevContiInsn);
779 SetCurrInsn(combineInsn);
780 optSuccess = true;
781 return;
782 } else if (IsValidStackArgLoadOrStorePattern(insn, *prevContiInsn, *curMemOpnd, *prevMemOpnd, curOfstVal,
783 prevOfstVal)) {
784 // Process stack-arg mem pair
785 regno_t curDestRegNo = curDestOpnd.GetRegisterNumber();
786 regno_t prevDestRegNo = prevDestOpnd.GetRegisterNumber();
787 RegOperand &newDest = static_cast<AArch64CGFunc *>(cgFunc)->GetOrCreatePhysicalRegisterOperand(
788 static_cast<AArch64reg>(curDestRegNo), k64BitSize, curDestOpnd.GetRegisterType());
789 RegOperand &newPrevDest = static_cast<AArch64CGFunc *>(cgFunc)->GetOrCreatePhysicalRegisterOperand(
790 static_cast<AArch64reg>(prevDestRegNo), k64BitSize, prevDestOpnd.GetRegisterType());
791 MOperator newMop = (curDestOpnd.GetRegisterType() == kRegTyInt) ? MOP_xstp : MOP_dstp;
792 if (!(static_cast<AArch64CGFunc &>(*cgFunc).IsOperandImmValid(newMop, combineMemOpnd, kInsnThirdOpnd))) {
793 return;
794 }
795 Insn *combineInsn =
796 GenerateMemPairInsn(newMop, newDest, newPrevDest, *combineMemOpnd, curOfstVal < prevOfstVal);
797 bb.InsertInsnAfter(*prevContiInsn, *combineInsn);
798 RemoveInsnAndKeepComment(bb, insn, *prevContiInsn);
799 SetCurrInsn(combineInsn);
800 optSuccess = true;
801 return;
802 }
803 }
804 }
805
FindUseX16AfterInsn(const Insn & curInsn) const806 bool CombineContiLoadAndStorePattern::FindUseX16AfterInsn(const Insn &curInsn) const
807 {
808 for (Insn *cursor = curInsn.GetNext(); cursor != nullptr; cursor = cursor->GetNext()) {
809 if (!cursor->IsMachineInstruction()) {
810 continue;
811 }
812 for (uint32 defRegNo : cursor->GetDefRegs()) {
813 if (defRegNo == R16) {
814 return false;
815 }
816 }
817 if ((!cursor->IsLoad() && !cursor->IsStore() && !cursor->IsLoadStorePair()) || cursor->IsAtomic()) {
818 continue;
819 }
820 const InsnDesc *md = &AArch64CG::kMd[cursor->GetMachineOpcode()];
821 if (cursor->IsLoadLabel() || md->IsLoadAddress()) {
822 continue;
823 }
824 uint32 memIdx = (cursor->IsLoadStorePair() ? kInsnThirdOpnd : kInsnSecondOpnd);
825 auto &curMemOpnd = static_cast<MemOperand &>(cursor->GetOperand(memIdx));
826 RegOperand *baseOpnd = curMemOpnd.GetBaseRegister();
827 if (baseOpnd != nullptr && baseOpnd->GetRegisterNumber() == R16) {
828 return true;
829 }
830 }
831 return false;
832 }
833
GenerateMemPairInsn(MOperator newMop,RegOperand & curDestOpnd,RegOperand & prevDestOpnd,MemOperand & combineMemOpnd,bool isCurDestFirst)834 Insn *CombineContiLoadAndStorePattern::GenerateMemPairInsn(MOperator newMop, RegOperand &curDestOpnd,
835 RegOperand &prevDestOpnd, MemOperand &combineMemOpnd,
836 bool isCurDestFirst)
837 {
838 DEBUG_ASSERT(newMop != MOP_undef, "invalid MOperator");
839 Insn *combineInsn = nullptr;
840 if (isPairAfterCombine) { // for ldr/str --> ldp/stp
841 combineInsn = (isCurDestFirst)
842 ? &cgFunc->GetInsnBuilder()->BuildInsn(newMop, curDestOpnd, prevDestOpnd, combineMemOpnd)
843 : &cgFunc->GetInsnBuilder()->BuildInsn(newMop, prevDestOpnd, curDestOpnd, combineMemOpnd);
844 } else { // for strb/strh --> strh/str, curDestOpnd == preDestOpnd
845 combineInsn = &cgFunc->GetInsnBuilder()->BuildInsn(newMop, curDestOpnd, combineMemOpnd);
846 combineMemOpnd.SetSize(newMop == MOP_wstrh ? maplebe::k16BitSize : maplebe::k32BitSize);
847 }
848 return combineInsn;
849 }
850
IsValidNormalLoadOrStorePattern(const Insn & insn,const Insn & prevInsn,const MemOperand & memOpnd,int64 curOfstVal,int64 prevOfstVal)851 bool CombineContiLoadAndStorePattern::IsValidNormalLoadOrStorePattern(const Insn &insn, const Insn &prevInsn,
852 const MemOperand &memOpnd, int64 curOfstVal,
853 int64 prevOfstVal)
854 {
855 if (memOpnd.IsStackArgMem()) {
856 return false;
857 }
858 DEBUG_ASSERT(insn.GetOperand(kInsnFirstOpnd).IsRegister(), "unexpect operand");
859 DEBUG_ASSERT(prevInsn.GetOperand(kInsnFirstOpnd).IsRegister(), "unexpect operand");
860 auto &curDestOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
861 auto &prevDestOpnd = static_cast<RegOperand &>(prevInsn.GetOperand(kInsnFirstOpnd));
862 if (prevDestOpnd.GetRegisterType() != curDestOpnd.GetRegisterType() ||
863 curDestOpnd.GetSize() != prevDestOpnd.GetSize()) {
864 return false;
865 }
866 uint32 memSize = insn.GetMemoryByteSize();
867 uint32 prevMemSize = prevInsn.GetMemoryByteSize();
868 if (memSize != prevMemSize) {
869 return false;
870 }
871
872 int64 diffVal = std::abs(curOfstVal - prevOfstVal);
873 if ((memSize == k1ByteSize && diffVal == k1BitSize) || (memSize == k2ByteSize && diffVal == k2BitSize) ||
874 (memSize == k4ByteSize && diffVal == k4BitSize) || (memSize == k8ByteSize && diffVal == k8BitSize)) {
875 MOperator curMop = insn.GetMachineOpcode();
876 DEBUG_ASSERT(curMop != MOP_wldrb && curMop != MOP_wldrh, "invalid mem insn that cannot be combined");
877 if (curMop == MOP_wstrb || curMop == MOP_wstrh) {
878 isPairAfterCombine = false;
879 }
880
881 regno_t destRegNO = curDestOpnd.GetRegisterNumber();
882 regno_t prevDestRegNO = prevDestOpnd.GetRegisterNumber();
883 if (destRegNO == RZR && prevDestRegNO == RZR) {
884 return true;
885 }
886
887 if (insn.IsLoad() && destRegNO == prevDestRegNO) {
888 return false;
889 }
890
891 if ((curMop == MOP_wstrb || curMop == MOP_wstrh) && (destRegNO != RZR || prevDestRegNO != RZR)) {
892 return false;
893 }
894
895 return true;
896 }
897
898 return false;
899 }
900
IsValidStackArgLoadOrStorePattern(const Insn & curInsn,const Insn & prevInsn,const MemOperand & curMemOpnd,const MemOperand & prevMemOpnd,int64 curOfstVal,int64 prevOfstVal) const901 bool CombineContiLoadAndStorePattern::IsValidStackArgLoadOrStorePattern(const Insn &curInsn, const Insn &prevInsn,
902 const MemOperand &curMemOpnd,
903 const MemOperand &prevMemOpnd, int64 curOfstVal,
904 int64 prevOfstVal) const
905 {
906 if (!curInsn.IsStore()) {
907 return false;
908 }
909 if (!curMemOpnd.IsStackArgMem() || !prevMemOpnd.IsStackArgMem()) {
910 return false;
911 }
912 auto &curDestOpnd = static_cast<RegOperand &>(curInsn.GetOperand(kInsnFirstOpnd));
913 auto &prevDestOpnd = static_cast<RegOperand &>(prevInsn.GetOperand(kInsnFirstOpnd));
914 uint32 memSize = curInsn.GetMemoryByteSize();
915 uint32 prevMemSize = prevInsn.GetMemoryByteSize();
916 auto diffVal = std::abs(curOfstVal - prevOfstVal);
917 if ((memSize == k4ByteSize || memSize == k8ByteSize) && (prevMemSize == k4ByteSize || prevMemSize == k8ByteSize) &&
918 (diffVal == k8BitSize) && (curDestOpnd.GetValidBitsNum() == memSize * k8BitSize) &&
919 (prevDestOpnd.GetValidBitsNum() == prevMemSize * k8BitSize)) {
920 return true;
921 }
922 return false;
923 }
924
RemoveInsnAndKeepComment(BB & bb,Insn & insn,Insn & prevInsn) const925 void CombineContiLoadAndStorePattern::RemoveInsnAndKeepComment(BB &bb, Insn &insn, Insn &prevInsn) const
926 {
927 /* keep the comment */
928 Insn *nn = prevInsn.GetNextMachineInsn();
929 std::string newComment = "";
930 MapleString comment = insn.GetComment();
931 if (comment.c_str() != nullptr && strlen(comment.c_str()) > 0) {
932 newComment += comment.c_str();
933 }
934 comment = prevInsn.GetComment();
935 if (comment.c_str() != nullptr && strlen(comment.c_str()) > 0) {
936 newComment = newComment + " " + comment.c_str();
937 }
938 if (newComment.c_str() != nullptr && strlen(newComment.c_str()) > 0) {
939 DEBUG_ASSERT(nn != nullptr, "nn should not be nullptr");
940 nn->SetComment(newComment);
941 }
942 bb.RemoveInsn(insn);
943 bb.RemoveInsn(prevInsn);
944 }
945
Run(BB & bb,Insn & insn)946 void CmpZeroBranch::Run(BB &bb, Insn &insn)
947 {
948 bool is64Bit = insn.GetMachineOpcode() == MOP_xcmpri;
949 RegOperand &cmpInsnSecondOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
950 ImmOperand &cmpInsnThirdOpnd = static_cast<ImmOperand &>(insn.GetOperand(kInsnThirdOpnd));
951 if (cmpInsnThirdOpnd.GetValue() != 0) {
952 return;
953 }
954 Insn *nextInsn = insn.GetNextMachineInsn();
955 if (nextInsn == nullptr || (nextInsn->GetMachineOpcode() != MOP_xcsetrc &&
956 nextInsn->GetMachineOpcode() != MOP_wcsetrc)) {
957 return;
958 }
959 auto condCode = static_cast<CondOperand &>(nextInsn->GetOperand(kInsnSecondOpnd)).GetCode();
960 if (condCode != CC_EQ && condCode != CC_NE) {
961 return;
962 }
963 Insn *thirdInsn = nextInsn->GetNextMachineInsn();
964 MOperator cbzMop = is64Bit ? MOP_xcbz : MOP_wcbz;
965 MOperator cbnzMop = is64Bit ? MOP_xcbnz : MOP_wcbnz;
966 if (thirdInsn == nullptr || (thirdInsn->GetMachineOpcode() != MOP_xcbz &&
967 thirdInsn->GetMachineOpcode() != MOP_wcbz && thirdInsn->GetMachineOpcode() != MOP_xcbnz &&
968 thirdInsn->GetMachineOpcode() != MOP_wcbnz)) {
969 return;
970 }
971 auto &tmpRegOp1 = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnFirstOpnd));
972 regno_t baseRegNO1 = tmpRegOp1.GetRegisterNumber();
973 auto &tmpRegOp2 = static_cast<RegOperand &>(thirdInsn->GetOperand(kInsnFirstOpnd));
974 regno_t baseRegNO2 = tmpRegOp2.GetRegisterNumber();
975 if (baseRegNO1 != baseRegNO2) {
976 return;
977 }
978 if (IfOperandIsLiveAfterInsn(tmpRegOp2, *thirdInsn)) {
979 return;
980 }
981 bool isCbz = thirdInsn->GetMachineOpcode() == MOP_xcbz || thirdInsn->GetMachineOpcode() == MOP_wcbz;
982 if ((condCode == CC_EQ && isCbz) || (condCode == CC_NE && !isCbz)) {
983 thirdInsn->SetMOP(AArch64CG::kMd[cbnzMop]);
984 } else {
985 thirdInsn->SetMOP(AArch64CG::kMd[cbzMop]);
986 }
987 thirdInsn->SetOperand(kInsnFirstOpnd, cmpInsnSecondOpnd);
988 bb.RemoveInsn(insn);
989 bb.RemoveInsn(*nextInsn);
990 }
991
Run(BB & bb,Insn & insn)992 void EliminateSpecifcSXTAArch64::Run(BB &bb, Insn &insn)
993 {
994 MOperator thisMop = insn.GetMachineOpcode();
995 Insn *prevInsn = insn.GetPrev();
996 while (prevInsn != nullptr && !prevInsn->GetMachineOpcode()) {
997 prevInsn = prevInsn->GetPrev();
998 }
999 if (prevInsn == nullptr) {
1000 return;
1001 }
1002 auto ®Opnd0 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1003 auto ®Opnd1 = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
1004 if (&insn != bb.GetFirstInsn() && regOpnd0.GetRegisterNumber() == regOpnd1.GetRegisterNumber() &&
1005 prevInsn->IsMachineInstruction()) {
1006 if (prevInsn->GetMachineOpcode() == MOP_wmovri32 || prevInsn->GetMachineOpcode() == MOP_xmovri64) {
1007 auto &dstMovOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1008 if (dstMovOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) {
1009 return;
1010 }
1011 Operand &opnd = prevInsn->GetOperand(kInsnSecondOpnd);
1012 if (opnd.IsIntImmediate()) {
1013 auto &immOpnd = static_cast<ImmOperand &>(opnd);
1014 int64 value = immOpnd.GetValue();
1015 if (thisMop == MOP_xsxtb32) {
1016 /* value should in range between -127 and 127 */
1017 if (value >= static_cast<int64>(0xFFFFFFFFFFFFFF80) && value <= 0x7F &&
1018 immOpnd.IsSingleInstructionMovable(regOpnd0.GetSize())) {
1019 bb.RemoveInsn(insn);
1020 }
1021 } else if (thisMop == MOP_xsxth32) {
1022 /* value should in range between -32678 and 32678 */
1023 if (value >= static_cast<int64>(0xFFFFFFFFFFFF8000) && value <= 0x7FFF &&
1024 immOpnd.IsSingleInstructionMovable(regOpnd0.GetSize())) {
1025 bb.RemoveInsn(insn);
1026 }
1027 } else {
1028 uint64 flag = 0xFFFFFFFFFFFFFF80; /* initialize the flag with fifty-nine 1s at top */
1029 if (thisMop == MOP_xsxth64) {
1030 flag = 0xFFFFFFFFFFFF8000; /* specify the flag with forty-nine 1s at top in this case */
1031 } else if (thisMop == MOP_xsxtw64) {
1032 flag = 0xFFFFFFFF80000000; /* specify the flag with thirty-three 1s at top in this case */
1033 }
1034 if (!(static_cast<uint64>(value) & flag) &&
1035 immOpnd.IsSingleInstructionMovable(regOpnd0.GetSize())) {
1036 auto *aarch64CGFunc = static_cast<AArch64CGFunc *>(&cgFunc);
1037 RegOperand &dstOpnd = aarch64CGFunc->GetOrCreatePhysicalRegisterOperand(
1038 static_cast<AArch64reg>(dstMovOpnd.GetRegisterNumber()), k64BitSize,
1039 dstMovOpnd.GetRegisterType());
1040 prevInsn->SetOperand(kInsnFirstOpnd, dstOpnd);
1041 prevInsn->SetMOP(AArch64CG::kMd[MOP_xmovri64]);
1042 bb.RemoveInsn(insn);
1043 }
1044 }
1045 }
1046 } else if (prevInsn->GetMachineOpcode() == MOP_wldrsb) {
1047 auto &dstMovOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1048 if (dstMovOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) {
1049 return;
1050 }
1051 if (thisMop == MOP_xsxtb32) {
1052 bb.RemoveInsn(insn);
1053 }
1054 } else if (prevInsn->GetMachineOpcode() == MOP_wldrsh) {
1055 auto &dstMovOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1056 if (dstMovOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) {
1057 return;
1058 }
1059 if (thisMop == MOP_xsxth32) {
1060 bb.RemoveInsn(insn);
1061 }
1062 }
1063 }
1064 }
1065
Run(BB & bb,Insn & insn)1066 void EliminateSpecifcUXTAArch64::Run(BB &bb, Insn &insn)
1067 {
1068 MOperator thisMop = insn.GetMachineOpcode();
1069 Insn *prevInsn = insn.GetPreviousMachineInsn();
1070 if (prevInsn == nullptr) {
1071 return;
1072 }
1073 auto ®Opnd0 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1074 auto ®Opnd1 = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
1075 if (prevInsn->IsCall() && prevInsn->GetIsCallReturnUnsigned() &&
1076 regOpnd0.GetRegisterNumber() == regOpnd1.GetRegisterNumber() &&
1077 (regOpnd1.GetRegisterNumber() == R0 || regOpnd1.GetRegisterNumber() == V0)) {
1078 uint32 retSize = prevInsn->GetRetSize();
1079 if (retSize > 0 &&
1080 ((thisMop == MOP_xuxtb32 && retSize <= k1ByteSize) || (thisMop == MOP_xuxth32 && retSize <= k2ByteSize) ||
1081 (thisMop == MOP_xuxtw64 && retSize <= k4ByteSize))) {
1082 bb.RemoveInsn(insn);
1083 }
1084 return;
1085 }
1086 if (&insn == bb.GetFirstInsn() || regOpnd0.GetRegisterNumber() != regOpnd1.GetRegisterNumber() ||
1087 !prevInsn->IsMachineInstruction()) {
1088 return;
1089 }
1090 if (cgFunc.GetMirModule().GetSrcLang() == kSrcLangC && prevInsn->IsCall() && prevInsn->GetIsCallReturnSigned()) {
1091 return;
1092 }
1093 if (thisMop == MOP_xuxtb32) {
1094 if (prevInsn->GetMachineOpcode() == MOP_wmovri32 || prevInsn->GetMachineOpcode() == MOP_xmovri64) {
1095 auto &dstMovOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1096 if (!IsSameRegisterOperation(dstMovOpnd, regOpnd1, regOpnd0)) {
1097 return;
1098 }
1099 Operand &opnd = prevInsn->GetOperand(kInsnSecondOpnd);
1100 if (opnd.IsIntImmediate()) {
1101 auto &immOpnd = static_cast<ImmOperand &>(opnd);
1102 int64 value = immOpnd.GetValue();
1103 /* check the top 56 bits of value */
1104 if (!(static_cast<uint64>(value) & 0xFFFFFFFFFFFFFF00)) {
1105 bb.RemoveInsn(insn);
1106 }
1107 }
1108 } else if (prevInsn->GetMachineOpcode() == MOP_wldrb) {
1109 auto &dstOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1110 if (dstOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) {
1111 return;
1112 }
1113 bb.RemoveInsn(insn);
1114 }
1115 } else if (thisMop == MOP_xuxth32) {
1116 if (prevInsn->GetMachineOpcode() == MOP_wmovri32 || prevInsn->GetMachineOpcode() == MOP_xmovri64) {
1117 auto &dstMovOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1118 if (!IsSameRegisterOperation(dstMovOpnd, regOpnd1, regOpnd0)) {
1119 return;
1120 }
1121 Operand &opnd = prevInsn->GetOperand(kInsnSecondOpnd);
1122 if (opnd.IsIntImmediate()) {
1123 auto &immOpnd = static_cast<ImmOperand &>(opnd);
1124 int64 value = immOpnd.GetValue();
1125 if (!(static_cast<uint64>(value) & 0xFFFFFFFFFFFF0000)) {
1126 bb.RemoveInsn(insn);
1127 }
1128 }
1129 } else if (prevInsn->GetMachineOpcode() == MOP_wldrh) {
1130 auto &dstOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1131 if (dstOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) {
1132 return;
1133 }
1134 bb.RemoveInsn(insn);
1135 }
1136 } else {
1137 /* this_mop == MOP_xuxtw64 */
1138 if (prevInsn->GetMachineOpcode() == MOP_wmovri32 || prevInsn->GetMachineOpcode() == MOP_wldrsb ||
1139 prevInsn->GetMachineOpcode() == MOP_wldrb || prevInsn->GetMachineOpcode() == MOP_wldrsh ||
1140 prevInsn->GetMachineOpcode() == MOP_wldrh || prevInsn->GetMachineOpcode() == MOP_wldr) {
1141 auto &dstOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1142 if (!IsSameRegisterOperation(dstOpnd, regOpnd1, regOpnd0)) {
1143 return;
1144 }
1145 /* 32-bit ldr does zero-extension by default, so this conversion can be skipped */
1146 bb.RemoveInsn(insn);
1147 }
1148 }
1149 }
1150
CheckCondition(Insn & insn)1151 bool FmovRegPattern::CheckCondition(Insn &insn)
1152 {
1153 nextInsn = insn.GetNextMachineInsn();
1154 if (nextInsn == nullptr) {
1155 return false;
1156 }
1157 prevInsn = insn.GetPreviousMachineInsn();
1158 if (prevInsn == nullptr) {
1159 return false;
1160 }
1161 auto &curSrcOpnd = insn.GetOperand(kInsnSecondOpnd);
1162 auto &prevSrcOpnd = prevInsn->GetOperand(kInsnSecondOpnd);
1163 if (!curSrcOpnd.IsRegister() || !prevSrcOpnd.IsRegister()) {
1164 return false;
1165 }
1166 auto &curSrcRegOpnd = static_cast<RegOperand&>(curSrcOpnd);
1167 auto &prevSrcRegOpnd = static_cast<RegOperand&>(prevSrcOpnd);
1168 /* same src freg */
1169 if (curSrcRegOpnd.GetRegisterNumber() != prevSrcRegOpnd.GetRegisterNumber()) {
1170 return false;
1171 }
1172 return true;
1173 }
1174
Run(BB & bb,Insn & insn)1175 void FmovRegPattern::Run(BB &bb, Insn &insn)
1176 {
1177 if (!CheckCondition(insn)) {
1178 return;
1179 }
1180 MOperator thisMop = insn.GetMachineOpcode();
1181 MOperator prevMop = prevInsn->GetMachineOpcode();
1182 MOperator newMop;
1183 uint32 doOpt = 0;
1184 if (prevMop == MOP_xvmovrv && thisMop == MOP_xvmovrv) {
1185 doOpt = k32BitSize;
1186 newMop = MOP_wmovrr;
1187 } else if (prevMop == MOP_xvmovrd && thisMop == MOP_xvmovrd) {
1188 doOpt = k64BitSize;
1189 newMop = MOP_xmovrr;
1190 }
1191 if (doOpt == 0) {
1192 return;
1193 }
1194 auto &curDstRegOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1195 regno_t curDstReg = curDstRegOpnd.GetRegisterNumber();
1196 /* optimize case 1 */
1197 auto &prevDstRegOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1198 regno_t prevDstReg = prevDstRegOpnd.GetRegisterNumber();
1199 auto *aarch64CGFunc = static_cast<AArch64CGFunc *>(cgFunc);
1200 RegOperand &dst =
1201 aarch64CGFunc->GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(curDstReg), doOpt, kRegTyInt);
1202 RegOperand &src =
1203 aarch64CGFunc->GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(prevDstReg), doOpt, kRegTyInt);
1204 Insn &newInsn = cgFunc->GetInsnBuilder()->BuildInsn(newMop, dst, src);
1205 bb.InsertInsnBefore(insn, newInsn);
1206 bb.RemoveInsn(insn);
1207 RegOperand &newOpnd =
1208 aarch64CGFunc->GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(prevDstReg), doOpt, kRegTyInt);
1209 uint32 opndNum = nextInsn->GetOperandSize();
1210 for (uint32 opndIdx = 0; opndIdx < opndNum; ++opndIdx) {
1211 Operand &opnd = nextInsn->GetOperand(opndIdx);
1212 if (opnd.IsMemoryAccessOperand()) {
1213 auto &memOpnd = static_cast<MemOperand &>(opnd);
1214 Operand *base = memOpnd.GetBaseRegister();
1215 if (base != nullptr) {
1216 if (base->IsRegister()) {
1217 auto *reg = static_cast<RegOperand *>(base);
1218 if (reg->GetRegisterNumber() == curDstReg) {
1219 memOpnd.SetBaseRegister(newOpnd);
1220 }
1221 }
1222 }
1223 Operand *offset = memOpnd.GetIndexRegister();
1224 if (offset != nullptr) {
1225 if (offset->IsRegister()) {
1226 auto *reg = static_cast<RegOperand *>(offset);
1227 if (reg->GetRegisterNumber() == curDstReg) {
1228 memOpnd.SetIndexRegister(newOpnd);
1229 }
1230 }
1231 }
1232 } else if (opnd.IsRegister()) {
1233 /* Check if it is a source operand. */
1234 auto *regProp = nextInsn->GetDesc()->opndMD[opndIdx];
1235 if (regProp->IsUse()) {
1236 auto ® = static_cast<RegOperand &>(opnd);
1237 if (reg.GetRegisterNumber() == curDstReg) {
1238 nextInsn->SetOperand(opndIdx, newOpnd);
1239 }
1240 }
1241 }
1242 }
1243 }
1244
CheckCondition(Insn & insn)1245 bool SbfxOptPattern::CheckCondition(Insn &insn)
1246 {
1247 nextInsn = insn.GetNextMachineInsn();
1248 if (nextInsn == nullptr) {
1249 return false;
1250 }
1251 auto &curDstRegOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1252 auto &lsb = static_cast<ImmOperand &>(insn.GetOperand(kInsnThirdOpnd));
1253 auto &width = static_cast<ImmOperand &>(insn.GetOperand(kInsnFourthOpnd));
1254 if (lsb.GetValue() != 0 || width.GetValue() < k32BitSize) {
1255 return false;
1256 }
1257 uint32 opndNum = nextInsn->GetOperandSize();
1258 const InsnDesc *md = nextInsn->GetDesc();
1259 for (uint32 opndIdx = 0; opndIdx < opndNum; ++opndIdx) {
1260 Operand &opnd = nextInsn->GetOperand(opndIdx);
1261 /* Check if it is a source operand. */
1262 if (opnd.IsMemoryAccessOperand() || opnd.IsList()) {
1263 return false;
1264 } else if (opnd.IsRegister()) {
1265 auto ® = static_cast<RegOperand &>(opnd);
1266 auto *regProp = md->opndMD[opndIdx];
1267 if (reg.GetRegisterNumber() == curDstRegOpnd.GetRegisterNumber()) {
1268 if (nextInsn->GetOperandSize(opndIdx) != k32BitSize) {
1269 return false;
1270 }
1271 if (regProp->IsDef()) {
1272 toRemove = true;
1273 } else {
1274 (void)cands.emplace_back(opndIdx);
1275 }
1276 }
1277 }
1278 }
1279 return cands.size() != 0;
1280 }
1281
Run(BB & bb,Insn & insn)1282 void SbfxOptPattern::Run(BB &bb, Insn &insn)
1283 {
1284 if (!CheckCondition(insn)) {
1285 return;
1286 }
1287 auto &srcRegOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
1288 RegOperand &newReg = static_cast<AArch64CGFunc *>(cgFunc)->GetOrCreatePhysicalRegisterOperand(
1289 static_cast<AArch64reg>(srcRegOpnd.GetRegisterNumber()), k32BitSize, srcRegOpnd.GetRegisterType());
1290 // replace use point of opnd in nextInsn
1291 for (auto i : cands) {
1292 nextInsn->SetOperand(i, newReg);
1293 }
1294 if (toRemove) {
1295 bb.RemoveInsn(insn);
1296 }
1297 }
1298
CheckCondition(Insn & insn)1299 bool CbnzToCbzPattern::CheckCondition(Insn &insn)
1300 {
1301 MOperator curMop = insn.GetMachineOpcode();
1302 if (curMop != MOP_wcbnz && curMop != MOP_xcbnz) {
1303 return false;
1304 }
1305 /* reg has to be R0, since return value is in R0 */
1306 auto ®Opnd0 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1307 if (regOpnd0.GetRegisterNumber() != R0) {
1308 return false;
1309 }
1310 nextBB = insn.GetBB()->GetNext();
1311 /* Make sure nextBB can only be reached by bb */
1312 if (nextBB->GetPreds().size() > 1) {
1313 return false;
1314 }
1315 /* Next insn should be a mov R0 = 0 */
1316 movInsn = nextBB->GetFirstMachineInsn();
1317 if (movInsn == nullptr) {
1318 return false;
1319 }
1320 MOperator movInsnMop = movInsn->GetMachineOpcode();
1321 if (movInsnMop != MOP_wmovri32 && movInsnMop != MOP_xmovri64) {
1322 return false;
1323 }
1324 auto &movDest = static_cast<RegOperand &>(movInsn->GetOperand(kInsnFirstOpnd));
1325 if (movDest.GetRegisterNumber() != R0) {
1326 return false;
1327 }
1328 auto &movImm = static_cast<ImmOperand &>(movInsn->GetOperand(kInsnSecondOpnd));
1329 if (movImm.GetValue() != 0) {
1330 return false;
1331 }
1332 Insn *nextBrInsn = movInsn->GetNextMachineInsn();
1333 if (nextBrInsn == nullptr) {
1334 return false;
1335 }
1336 if (nextBrInsn->GetMachineOpcode() != MOP_xuncond) {
1337 return false;
1338 }
1339 /* Is nextBB branch to the return-bb? */
1340 if (nextBB->GetSuccs().size() != 1) {
1341 return false;
1342 }
1343 return true;
1344 }
1345
Run(BB & bb,Insn & insn)1346 void CbnzToCbzPattern::Run(BB &bb, Insn &insn)
1347 {
1348 if (!CheckCondition(insn)) {
1349 return;
1350 }
1351 MOperator thisMop = insn.GetMachineOpcode();
1352 BB *targetBB = nullptr;
1353 auto it = bb.GetSuccsBegin();
1354 if (*it == nextBB) {
1355 ++it;
1356 }
1357 targetBB = *it;
1358 /* Make sure when nextBB is empty, targetBB is fallthru of bb. */
1359 if (targetBB != nextBB->GetNext()) {
1360 return;
1361 }
1362 BB *nextBBTarget = *(nextBB->GetSuccsBegin());
1363 if (nextBBTarget->GetKind() != BB::kBBReturn) {
1364 return;
1365 }
1366 /* Control flow looks nice, instruction looks nice */
1367 DEBUG_ASSERT(brInsn != nullptr, "brInsn should not be nullptr");
1368 Operand &brTarget = brInsn->GetOperand(kInsnFirstOpnd);
1369 insn.SetOperand(kInsnSecondOpnd, brTarget);
1370 if (thisMop == MOP_wcbnz) {
1371 insn.SetMOP(AArch64CG::kMd[MOP_wcbz]);
1372 } else {
1373 insn.SetMOP(AArch64CG::kMd[MOP_xcbz]);
1374 }
1375 nextBB->RemoveInsn(*movInsn);
1376 nextBB->RemoveInsn(*brInsn);
1377 /* nextBB is now a fallthru bb, not a goto bb */
1378 nextBB->SetKind(BB::kBBFallthru);
1379 /*
1380 * fix control flow, we have bb, nextBB, targetBB, nextBB_target
1381 * connect bb -> nextBB_target erase targetBB
1382 */
1383 it = bb.GetSuccsBegin();
1384 CHECK_FATAL(it != bb.GetSuccsEnd(), "succs is empty.");
1385 if (*it == targetBB) {
1386 bb.EraseSuccs(it);
1387 bb.PushFrontSuccs(*nextBBTarget);
1388 } else {
1389 ++it;
1390 bb.EraseSuccs(it);
1391 bb.PushBackSuccs(*nextBBTarget);
1392 }
1393 for (auto targetBBIt = targetBB->GetPredsBegin(); targetBBIt != targetBB->GetPredsEnd(); ++targetBBIt) {
1394 if (*targetBBIt == &bb) {
1395 targetBB->ErasePreds(targetBBIt);
1396 break;
1397 }
1398 }
1399 for (auto nextIt = nextBBTarget->GetPredsBegin(); nextIt != nextBBTarget->GetPredsEnd(); ++nextIt) {
1400 if (*nextIt == nextBB) {
1401 nextBBTarget->ErasePreds(nextIt);
1402 break;
1403 }
1404 }
1405 nextBBTarget->PushBackPreds(bb);
1406
1407 /* nextBB has no target, originally just branch target */
1408 nextBB->EraseSuccs(nextBB->GetSuccsBegin());
1409 DEBUG_ASSERT(nextBB->GetSuccs().empty(), "peep: branch target incorrect");
1410 /* Now make nextBB fallthru to targetBB */
1411 nextBB->PushFrontSuccs(*targetBB);
1412 targetBB->PushBackPreds(*nextBB);
1413 }
1414
HasImplicitSizeUse(const Insn & insn) const1415 bool ContiLDRorSTRToSameMEMPattern::HasImplicitSizeUse(const Insn &insn) const
1416 {
1417 if (insn.GetOperandSize(kInsnFirstOpnd) != prevInsn->GetOperandSize(kInsnFirstOpnd)) {
1418 return true;
1419 }
1420 if (loadAfterStore) {
1421 // To avoid the optimization as following:
1422 // str w10, [sp, #8]
1423 // ldr w10, [sp, #8] ---\--> can not be removed
1424 // ...
1425 // str x10, [x1, #16]
1426 auto &defOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1427 for (Insn *cursor = insn.GetNext(); cursor != nullptr; cursor = cursor->GetNext()) {
1428 if (!cursor->IsMachineInstruction()) {
1429 continue;
1430 }
1431 uint32 opndNum = cursor->GetOperandSize();
1432 for (uint32 i = 0; i < opndNum; ++i) {
1433 if (cursor->OpndIsDef(i)) {
1434 continue;
1435 }
1436 if (!cursor->GetOperand(i).IsRegister()) {
1437 continue;
1438 }
1439 auto &useOpnd = static_cast<RegOperand &>(cursor->GetOperand(i));
1440 if (useOpnd.GetRegisterNumber() == defOpnd.GetRegisterNumber() &&
1441 insn.GetOperandSize(kInsnFirstOpnd) != cursor->GetOperandSize(i)) {
1442 return true;
1443 }
1444 }
1445 }
1446 }
1447 return false;
1448 }
1449
IntrinsicOptimize(BB & bb,Insn * preInsn,Insn & insn)1450 void CsetCbzToBeqOptAArch64::IntrinsicOptimize(BB &bb, Insn *preInsn, Insn &insn)
1451 {
1452 MOperator opCode = insn.GetMachineOpcode();
1453 MOperator preOpCode = preInsn->GetMachineOpcode();
1454 bool reverse = (opCode == MOP_xcbz || opCode == MOP_wcbz);
1455 Operand &rflag = static_cast<AArch64CGFunc *>(&cgFunc)->GetOrCreateRflag();
1456 auto &label = static_cast<LabelOperand &>(insn.GetOperand(kInsnSecondOpnd));
1457 if (preOpCode == MOP_tagged_is_heapobject) {
1458 AArch64CGFunc &aarch64cgFunc = static_cast<AArch64CGFunc&>(cgFunc);
1459 RegOperand &destReg = static_cast<RegOperand&>(preInsn->GetOperand(kInsnFirstOpnd));
1460 RegOperand &srcReg = static_cast<RegOperand&>(preInsn->GetOperand(kInsnSecondOpnd));
1461 RegOperand &tmpReg = static_cast<RegOperand&>(preInsn->GetOperand(kInsnThirdOpnd));
1462 Insn &insn1 = cgFunc.GetInsnBuilder()->BuildInsn(MOP_xandrrr, destReg, srcReg, tmpReg);
1463 bb.InsertInsnBefore(insn, insn1);
1464 ImmOperand &immValueZero = aarch64cgFunc.CreateImmOperand(0, k16BitSize, false);
1465 Insn &insn2 = cgFunc.GetInsnBuilder()->BuildInsn(MOP_xcmpri, rflag, destReg, immValueZero);
1466 bb.InsertInsnBefore(insn, insn2);
1467 MOperator jmpOperator = SelectMOperator(CC_EQ, reverse);
1468 CHECK_FATAL((jmpOperator != MOP_undef), "unknown condition code");
1469 Insn &newInsn = cgFunc.GetInsnBuilder()->BuildInsn(jmpOperator, rflag, label);
1470 bb.RemoveInsn(*preInsn);
1471 bb.ReplaceInsn(insn, newInsn);
1472 } else if (preOpCode == MOP_has_pending_exception) {
1473 AArch64CGFunc &aarch64cgFunc = static_cast<AArch64CGFunc&>(cgFunc);
1474 RegOperand &destReg = static_cast<RegOperand&>(preInsn->GetOperand(kInsnFirstOpnd));
1475 RegOperand &srcReg = static_cast<RegOperand&>(preInsn->GetOperand(kInsnSecondOpnd));
1476 DEBUG_ASSERT(preInsn->GetOperand(kInsnThirdOpnd).IsImmediate(), "wrong operand type");
1477 ImmOperand &offsetOpnd = static_cast<ImmOperand&>(preInsn->GetOperand(kInsnThirdOpnd));
1478 // ExceptionOffset: 0xf70(3952)
1479 int64 offset = offsetOpnd.GetValue();
1480 Operand &exceptionMem = aarch64cgFunc.CreateMemOpnd(srcReg, offset, k64BitSize);
1481 Insn &ldrExceptionInsn = cgFunc.GetInsnBuilder()->BuildInsn(MOP_xldr, destReg, exceptionMem);
1482 bb.InsertInsnBefore(insn, ldrExceptionInsn);
1483
1484 // HOLE : 5
1485 ImmOperand &holeValue = aarch64cgFunc.CreateImmOperand(5, k16BitSize, false);
1486 Insn &cmpInsn = cgFunc.GetInsnBuilder()->BuildInsn(MOP_xcmpri, rflag, destReg, holeValue);
1487 bb.InsertInsnBefore(insn, cmpInsn);
1488
1489 MOperator jmpOperator = SelectMOperator(CC_NE, reverse);
1490 CHECK_FATAL((jmpOperator != MOP_undef), "unknown condition code");
1491 Insn &newInsn = cgFunc.GetInsnBuilder()->BuildInsn(jmpOperator, rflag, label);
1492 bb.RemoveInsn(*preInsn);
1493 bb.ReplaceInsn(insn, newInsn);
1494 }
1495 return;
1496 }
1497
Run(BB & bb,Insn & insn)1498 void CsetCbzToBeqOptAArch64::Run(BB &bb, Insn &insn)
1499 {
1500 Insn *insn1 = insn.GetPreviousMachineInsn();
1501 if (insn1 == nullptr) {
1502 return;
1503 }
1504 /* prevInsn must be "cset" insn */
1505 MOperator opCode1 = insn1->GetMachineOpcode();
1506 if (opCode1 != MOP_xcsetrc && opCode1 != MOP_wcsetrc &&
1507 opCode1 != MOP_tagged_is_heapobject && opCode1 != MOP_has_pending_exception) {
1508 return;
1509 }
1510
1511 auto &tmpRegOp1 = static_cast<RegOperand &>(insn1->GetOperand(kInsnFirstOpnd));
1512 regno_t baseRegNO1 = tmpRegOp1.GetRegisterNumber();
1513 auto &tmpRegOp2 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1514 regno_t baseRegNO2 = tmpRegOp2.GetRegisterNumber();
1515 if (baseRegNO1 != baseRegNO2) {
1516 return;
1517 }
1518 /* If the reg will be used later, we shouldn't optimize the cset insn here */
1519 if (IfOperandIsLiveAfterInsn(tmpRegOp2, insn)) {
1520 return;
1521 }
1522 if (opCode1 == MOP_tagged_is_heapobject || opCode1 == MOP_has_pending_exception) {
1523 IntrinsicOptimize(bb, insn1, insn);
1524 return;
1525 }
1526 MOperator opCode = insn.GetMachineOpcode();
1527 bool reverse = (opCode == MOP_xcbz || opCode == MOP_wcbz);
1528 Operand &rflag = static_cast<AArch64CGFunc *>(&cgFunc)->GetOrCreateRflag();
1529 auto &label = static_cast<LabelOperand &>(insn.GetOperand(kInsnSecondOpnd));
1530 auto &cond = static_cast<CondOperand &>(insn1->GetOperand(kInsnSecondOpnd));
1531 MOperator jmpOperator = SelectMOperator(cond.GetCode(), reverse);
1532 CHECK_FATAL((jmpOperator != MOP_undef), "unknown condition code");
1533 Insn &newInsn = cgFunc.GetInsnBuilder()->BuildInsn(jmpOperator, rflag, label);
1534 bb.RemoveInsn(*insn1);
1535 bb.ReplaceInsn(insn, newInsn);
1536 }
1537
SelectMOperator(ConditionCode condCode,bool inverse) const1538 MOperator CsetCbzToBeqOptAArch64::SelectMOperator(ConditionCode condCode, bool inverse) const
1539 {
1540 switch (condCode) {
1541 case CC_NE:
1542 return inverse ? MOP_beq : MOP_bne;
1543 case CC_EQ:
1544 return inverse ? MOP_bne : MOP_beq;
1545 case CC_MI:
1546 return inverse ? MOP_bpl : MOP_bmi;
1547 case CC_PL:
1548 return inverse ? MOP_bmi : MOP_bpl;
1549 case CC_VS:
1550 return inverse ? MOP_bvc : MOP_bvs;
1551 case CC_VC:
1552 return inverse ? MOP_bvs : MOP_bvc;
1553 case CC_HI:
1554 return inverse ? MOP_bls : MOP_bhi;
1555 case CC_LS:
1556 return inverse ? MOP_bhi : MOP_bls;
1557 case CC_GE:
1558 return inverse ? MOP_blt : MOP_bge;
1559 case CC_LT:
1560 return inverse ? MOP_bge : MOP_blt;
1561 case CC_HS:
1562 return inverse ? MOP_blo : MOP_bhs;
1563 case CC_LO:
1564 return inverse ? MOP_bhs : MOP_blo;
1565 case CC_LE:
1566 return inverse ? MOP_bgt : MOP_ble;
1567 case CC_GT:
1568 return inverse ? MOP_ble : MOP_bgt;
1569 case CC_CS:
1570 return inverse ? MOP_bcc : MOP_bcs;
1571 default:
1572 return MOP_undef;
1573 }
1574 }
1575
CheckCondition(Insn & insn)1576 bool ContiLDRorSTRToSameMEMPattern::CheckCondition(Insn &insn)
1577 {
1578 prevInsn = insn.GetPrev();
1579 while (prevInsn != nullptr && (prevInsn->GetMachineOpcode() == 0 || !prevInsn->IsMachineInstruction()) &&
1580 prevInsn != insn.GetBB()->GetFirstMachineInsn()) {
1581 prevInsn = prevInsn->GetPrev();
1582 }
1583 if (!insn.IsMachineInstruction() || prevInsn == nullptr) {
1584 return false;
1585 }
1586 MOperator thisMop = insn.GetMachineOpcode();
1587 MOperator prevMop = prevInsn->GetMachineOpcode();
1588 /*
1589 * store regB, RegC, offset
1590 * load regA, RegC, offset
1591 */
1592 if ((thisMop == MOP_xldr && prevMop == MOP_xstr) || (thisMop == MOP_wldr && prevMop == MOP_wstr) ||
1593 (thisMop == MOP_dldr && prevMop == MOP_dstr) || (thisMop == MOP_sldr && prevMop == MOP_sstr)) {
1594 loadAfterStore = true;
1595 }
1596 /*
1597 * load regA, RegC, offset
1598 * load regB, RegC, offset
1599 */
1600 if ((thisMop == MOP_xldr || thisMop == MOP_wldr || thisMop == MOP_dldr || thisMop == MOP_sldr) &&
1601 prevMop == thisMop) {
1602 loadAfterLoad = true;
1603 }
1604 if (!loadAfterStore && !loadAfterLoad) {
1605 return false;
1606 }
1607 if (HasImplicitSizeUse(insn)) {
1608 return false;
1609 }
1610 DEBUG_ASSERT(insn.GetOperand(kInsnSecondOpnd).IsMemoryAccessOperand(), "expects mem operands");
1611 DEBUG_ASSERT(prevInsn->GetOperand(kInsnSecondOpnd).IsMemoryAccessOperand(), "expects mem operands");
1612 return true;
1613 }
1614
Run(BB & bb,Insn & insn)1615 void ContiLDRorSTRToSameMEMPattern::Run(BB &bb, Insn &insn)
1616 {
1617 if (!CheckCondition(insn)) {
1618 return;
1619 }
1620 MOperator thisMop = insn.GetMachineOpcode();
1621 auto &memOpnd1 = static_cast<MemOperand &>(insn.GetOperand(kInsnSecondOpnd));
1622 MemOperand::AArch64AddressingMode addrMode1 = memOpnd1.GetAddrMode();
1623 if (addrMode1 != MemOperand::kAddrModeBOi || (!memOpnd1.IsIntactIndexed())) {
1624 return;
1625 }
1626
1627 auto *base1 = static_cast<RegOperand *>(memOpnd1.GetBaseRegister());
1628 DEBUG_ASSERT(base1 == nullptr || !base1->IsVirtualRegister(), "physical register has not been allocated?");
1629 OfstOperand *offset1 = memOpnd1.GetOffsetImmediate();
1630
1631 auto &memOpnd2 = static_cast<MemOperand &>(prevInsn->GetOperand(kInsnSecondOpnd));
1632 MemOperand::AArch64AddressingMode addrMode2 = memOpnd2.GetAddrMode();
1633 if (addrMode2 != MemOperand::kAddrModeBOi || (!memOpnd2.IsIntactIndexed())) {
1634 return;
1635 }
1636
1637 auto *base2 = static_cast<RegOperand *>(memOpnd2.GetBaseRegister());
1638 DEBUG_ASSERT(base2 == nullptr || !base2->IsVirtualRegister(), "physical register has not been allocated?");
1639 OfstOperand *offset2 = memOpnd2.GetOffsetImmediate();
1640
1641 if (base1 == nullptr || base2 == nullptr || offset1 == nullptr || offset2 == nullptr) {
1642 return;
1643 }
1644
1645 auto ®1 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1646 auto ®2 = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1647 int64 offsetVal1 = offset1->GetOffsetValue();
1648 int64 offsetVal2 = offset2->GetOffsetValue();
1649 if (base1->GetRegisterNumber() != base2->GetRegisterNumber() || reg1.GetRegisterType() != reg2.GetRegisterType() ||
1650 reg1.GetSize() != reg2.GetSize() || offsetVal1 != offsetVal2) {
1651 return;
1652 }
1653 if (loadAfterStore && reg1.GetRegisterNumber() != reg2.GetRegisterNumber()) {
1654 /* replace it with mov */
1655 MOperator newOp = MOP_undef;
1656 if (reg1.GetRegisterType() == kRegTyInt) {
1657 newOp = (insn.GetOperandSize(kInsnFirstOpnd) == k32BitSizeInt) ? MOP_wmovrr : MOP_xmovrr;
1658 } else if (reg1.GetRegisterType() == kRegTyFloat) {
1659 newOp = (insn.GetOperandSize(kInsnFirstOpnd) == k32BitSizeInt) ? MOP_xvmovs : MOP_xvmovd;
1660 }
1661 Insn *nextInsn = insn.GetNext();
1662 while (nextInsn != nullptr && !nextInsn->IsMachineInstruction() && nextInsn != bb.GetLastMachineInsn()) {
1663 nextInsn = nextInsn->GetNext();
1664 }
1665 bool moveSameReg = false;
1666 if (nextInsn && nextInsn->GetIsSpill() && !IfOperandIsLiveAfterInsn(reg1, *nextInsn)) {
1667 MOperator nextMop = nextInsn->GetMachineOpcode();
1668 if ((thisMop == MOP_xldr && nextMop == MOP_xstr) || (thisMop == MOP_wldr && nextMop == MOP_wstr) ||
1669 (thisMop == MOP_dldr && nextMop == MOP_dstr) || (thisMop == MOP_sldr && nextMop == MOP_sstr)) {
1670 nextInsn->Insn::SetOperand(kInsnFirstOpnd, reg2);
1671 moveSameReg = true;
1672 }
1673 }
1674 if (!moveSameReg) {
1675 bb.InsertInsnAfter(*prevInsn, cgFunc->GetInsnBuilder()->BuildInsn(newOp, reg1, reg2));
1676 }
1677 SetCurrInsn(insn.GetNextMachineInsn());
1678 optSuccess = true;
1679 bb.RemoveInsn(insn);
1680 } else if (reg1.GetRegisterNumber() == reg2.GetRegisterNumber() &&
1681 base1->GetRegisterNumber() != reg2.GetRegisterNumber()) {
1682 SetCurrInsn(insn.GetNextMachineInsn());
1683 optSuccess = true;
1684 bb.RemoveInsn(insn);
1685 }
1686 }
1687
Run(BB & bb,Insn & insn)1688 void CselZeroOneToCsetOpt::Run(BB &bb, Insn &insn)
1689 {
1690 Operand &trueValueOp = insn.GetOperand(kInsnSecondOpnd);
1691 Operand &falseValueOp = insn.GetOperand(kInsnThirdOpnd);
1692 Operand *trueTempOp = nullptr;
1693 Operand *falseTempOp = nullptr;
1694
1695 /* find fixed value in BB */
1696 if (!trueValueOp.IsIntImmediate()) {
1697 trueMovInsn = FindFixedValue(trueValueOp, bb, trueTempOp, insn);
1698 }
1699 if (!falseValueOp.IsIntImmediate()) {
1700 falseMovInsn = FindFixedValue(falseValueOp, bb, falseTempOp, insn);
1701 }
1702
1703 DEBUG_ASSERT(trueTempOp != nullptr, "trueTempOp should not be nullptr");
1704 DEBUG_ASSERT(falseTempOp != nullptr, "falseTempOp should not be nullptr");
1705 /* csel to cset */
1706 if ((trueTempOp->IsIntImmediate() || IsZeroRegister(*trueTempOp)) &&
1707 (falseTempOp->IsIntImmediate() || IsZeroRegister(*falseTempOp))) {
1708 ImmOperand *imm1 = static_cast<ImmOperand *>(trueTempOp);
1709 ImmOperand *imm2 = static_cast<ImmOperand *>(falseTempOp);
1710 bool inverse = imm1->IsOne() && (imm2->IsZero() || IsZeroRegister(*imm2));
1711 if (inverse || ((imm1->IsZero() || IsZeroRegister(*imm1)) && imm2->IsOne())) {
1712 Operand ® = insn.GetOperand(kInsnFirstOpnd);
1713 CondOperand &condOperand = static_cast<CondOperand &>(insn.GetOperand(kInsnFourthOpnd));
1714 MOperator mopCode = (reg.GetSize() == k64BitSize) ? MOP_xcsetrc : MOP_wcsetrc;
1715 /* get new cond ccCode */
1716 ConditionCode ccCode = inverse ? condOperand.GetCode() : GetReverseCC(condOperand.GetCode());
1717 if (ccCode == kCcLast) {
1718 return;
1719 }
1720 AArch64CGFunc *func = static_cast<AArch64CGFunc *>(cgFunc);
1721 CondOperand &cond = func->GetCondOperand(ccCode);
1722 Operand &rflag = func->GetOrCreateRflag();
1723 Insn &csetInsn = func->GetInsnBuilder()->BuildInsn(mopCode, reg, cond, rflag);
1724 if (CGOptions::DoCGSSA() && CGOptions::GetInstance().GetOptimizeLevel() < CGOptions::kLevel0) {
1725 CHECK_FATAL(false, "check this case in ssa opt");
1726 }
1727 insn.GetBB()->ReplaceInsn(insn, csetInsn);
1728 }
1729 }
1730 }
1731
FindFixedValue(Operand & opnd,BB & bb,Operand * & tempOp,const Insn & insn) const1732 Insn *CselZeroOneToCsetOpt::FindFixedValue(Operand &opnd, BB &bb, Operand *&tempOp, const Insn &insn) const
1733 {
1734 tempOp = &opnd;
1735 bool alreadyFindCsel = false;
1736 bool isRegDefined = false;
1737 regno_t regno = static_cast<RegOperand &>(opnd).GetRegisterNumber();
1738 FOR_BB_INSNS_REV(defInsn, &bb)
1739 {
1740 if (!defInsn->IsMachineInstruction() || defInsn->IsBranch()) {
1741 continue;
1742 }
1743 /* find csel */
1744 if (defInsn->GetId() == insn.GetId()) {
1745 alreadyFindCsel = true;
1746 }
1747 /* find def defined */
1748 if (alreadyFindCsel) {
1749 isRegDefined = defInsn->IsRegDefined(regno);
1750 }
1751 /* if def defined is movi do this opt */
1752 if (isRegDefined) {
1753 MOperator thisMop = defInsn->GetMachineOpcode();
1754 if (thisMop == MOP_wmovri32 || thisMop == MOP_xmovri64) {
1755 if (&defInsn->GetOperand(kInsnFirstOpnd) == &opnd) {
1756 tempOp = &(defInsn->GetOperand(kInsnSecondOpnd));
1757 return defInsn;
1758 }
1759 } else {
1760 return nullptr;
1761 }
1762 }
1763 }
1764 return nullptr;
1765 }
1766
Run(BB & bb,Insn & insn)1767 void AndCmpCsetEorCbzOpt::Run(BB &bb, Insn &insn)
1768 {
1769 if (insn.GetMachineOpcode() != MOP_wandrri12) {
1770 return;
1771 }
1772 RegOperand &andInsnFirstOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1773 RegOperand &andInsnSecondOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
1774 ImmOperand &andInsnThirdOpnd = static_cast<ImmOperand &>(insn.GetOperand(kInsnThirdOpnd));
1775 if (andInsnFirstOpnd.GetRegisterNumber() != andInsnSecondOpnd.GetRegisterNumber() ||
1776 andInsnThirdOpnd.GetValue() != 1) {
1777 return;
1778 }
1779 Insn *cmpInsn = insn.GetNextMachineInsn();
1780 if (cmpInsn == nullptr || cmpInsn->GetMachineOpcode() != MOP_wcmpri) {
1781 return;
1782 }
1783 RegOperand &cmpInsnSecondOpnd = static_cast<RegOperand &>(cmpInsn->GetOperand(kInsnSecondOpnd));
1784 ImmOperand &cmpInsnThirdOpnd = static_cast<ImmOperand &>(cmpInsn->GetOperand(kInsnThirdOpnd));
1785 if (cmpInsnSecondOpnd.GetRegisterNumber() != andInsnFirstOpnd.GetRegisterNumber() ||
1786 cmpInsnThirdOpnd.GetValue() != 0) {
1787 return;
1788 }
1789 Insn *csetInsn = cmpInsn->GetNextMachineInsn();
1790 if (csetInsn == nullptr || csetInsn->GetMachineOpcode() != MOP_wcsetrc) {
1791 return;
1792 }
1793 RegOperand &csetInsnFirstOpnd = static_cast<RegOperand &>(csetInsn->GetOperand(kInsnFirstOpnd));
1794 CondOperand &csetSecondOpnd = static_cast<CondOperand &>(csetInsn->GetOperand(kInsnSecondOpnd));
1795 if (csetInsnFirstOpnd.GetRegisterNumber() != andInsnFirstOpnd.GetRegisterNumber() ||
1796 csetSecondOpnd.GetCode() != CC_EQ) {
1797 return;
1798 }
1799 Insn *eorInsn = csetInsn->GetNextMachineInsn();
1800 if (eorInsn == nullptr || eorInsn->GetMachineOpcode() != MOP_weorrri12) {
1801 return;
1802 }
1803 RegOperand &eorInsnFirstOpnd = static_cast<RegOperand &>(eorInsn->GetOperand(kInsnFirstOpnd));
1804 RegOperand &eorInsnSecondOpnd = static_cast<RegOperand &>(eorInsn->GetOperand(kInsnSecondOpnd));
1805 ImmOperand &eorInsnThirdOpnd = static_cast<ImmOperand &>(eorInsn->GetOperand(kInsnThirdOpnd));
1806 if (eorInsnFirstOpnd.GetRegisterNumber() != andInsnFirstOpnd.GetRegisterNumber() ||
1807 eorInsnFirstOpnd.GetRegisterNumber() != eorInsnSecondOpnd.GetRegisterNumber() ||
1808 eorInsnThirdOpnd.GetValue() != 1) {
1809 return;
1810 }
1811 Insn *cbzInsn = eorInsn->GetNextMachineInsn();
1812 if (cbzInsn == nullptr || cbzInsn->GetMachineOpcode() != MOP_wcbz) {
1813 return;
1814 }
1815 RegOperand &cbzInsnFirstOpnd = static_cast<RegOperand &>(cbzInsn->GetOperand(kInsnFirstOpnd));
1816 if (cbzInsnFirstOpnd.GetRegisterNumber() != andInsnFirstOpnd.GetRegisterNumber()) {
1817 return;
1818 }
1819 bb.RemoveInsn(*cmpInsn);
1820 bb.RemoveInsn(*csetInsn);
1821 bb.RemoveInsn(*eorInsn);
1822 bb.RemoveInsn(*cbzInsn);
1823 /* replace insn */
1824 auto &label = static_cast<LabelOperand &>(cbzInsn->GetOperand(kInsnSecondOpnd));
1825 ImmOperand &oneHoleOpnd = static_cast<AArch64CGFunc *>(cgFunc)->CreateImmOperand(0, k8BitSize, false);
1826 bb.ReplaceInsn(insn, cgFunc->GetInsnBuilder()->BuildInsn(MOP_wtbz, cbzInsnFirstOpnd, oneHoleOpnd, label));
1827 }
1828
Run(BB & bb,Insn & insn)1829 void AddLdrOpt::Run(BB &bb, Insn &insn)
1830 {
1831 if (insn.GetMachineOpcode() != MOP_xaddrrr) {
1832 return;
1833 }
1834 Insn *nextInsn = insn.GetNextMachineInsn();
1835 if (nextInsn == nullptr) {
1836 return;
1837 }
1838 auto nextMop = nextInsn->GetMachineOpcode();
1839 if (nextMop != MOP_xldr && nextMop != MOP_wldr) {
1840 return;
1841 }
1842 RegOperand &insnFirstOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1843 RegOperand &insnSecondOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
1844 if (insnFirstOpnd.GetRegisterNumber() != insnSecondOpnd.GetRegisterNumber()) {
1845 return;
1846 }
1847 RegOperand &ldrInsnFirstOpnd = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnFirstOpnd));
1848 MemOperand &memOpnd = static_cast<MemOperand &>(nextInsn->GetOperand(kInsnSecondOpnd));
1849 if (memOpnd.GetAddrMode() != MemOperand::kAddrModeBOi ||
1850 memOpnd.GetBaseRegister()->GetRegisterNumber() != insnFirstOpnd.GetRegisterNumber() ||
1851 ldrInsnFirstOpnd.GetRegisterNumber() != insnFirstOpnd.GetRegisterNumber() ||
1852 memOpnd.GetOffsetImmediate()->GetOffsetValue() != 0) {
1853 return;
1854 }
1855 MemOperand &newMemOpnd = static_cast<AArch64CGFunc *>(cgFunc)->GetOrCreateMemOpnd(
1856 MemOperand::kAddrModeBOrX, memOpnd.GetSize(), &insnFirstOpnd,
1857 &static_cast<RegOperand &>(insn.GetOperand(kInsnThirdOpnd)), 0, false);
1858 nextInsn->SetOperand(kInsnSecondOpnd, newMemOpnd);
1859 bb.RemoveInsn(insn);
1860 }
1861
Run(BB & bb,Insn & insn)1862 void CsetEorOpt::Run(BB &bb, Insn &insn)
1863 {
1864 if (insn.GetMachineOpcode() != MOP_xcsetrc && insn.GetMachineOpcode() != MOP_wcsetrc) {
1865 return;
1866 }
1867 Insn *nextInsn = insn.GetNextMachineInsn();
1868 if (nextInsn == nullptr ||
1869 (nextInsn->GetMachineOpcode() != MOP_weorrri12 && nextInsn->GetMachineOpcode() != MOP_xeorrri13)) {
1870 return;
1871 }
1872 RegOperand &csetFirstOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1873 RegOperand &eorFirstOpnd = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnFirstOpnd));
1874 RegOperand &eorSecondOpnd = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnSecondOpnd));
1875 ImmOperand &eorThirdOpnd = static_cast<ImmOperand &>(nextInsn->GetOperand(kInsnThirdOpnd));
1876 if (eorThirdOpnd.GetValue() != 1 || eorFirstOpnd.GetRegisterNumber() != eorSecondOpnd.GetRegisterNumber() ||
1877 csetFirstOpnd.GetRegisterNumber() != eorFirstOpnd.GetRegisterNumber()) {
1878 return;
1879 }
1880 CondOperand &csetSecondOpnd = static_cast<CondOperand &>(insn.GetOperand(kInsnSecondOpnd));
1881 ConditionCode inverseCondCode = GetReverseCC(csetSecondOpnd.GetCode());
1882 if (inverseCondCode == kCcLast) {
1883 return;
1884 }
1885 auto *aarFunc = static_cast<AArch64CGFunc *>(cgFunc);
1886 CondOperand &inverseCondOpnd = aarFunc->GetCondOperand(inverseCondCode);
1887 insn.SetOperand(kInsnSecondOpnd, inverseCondOpnd);
1888 bb.RemoveInsn(*nextInsn);
1889 }
1890
Run(BB & bb,Insn & insn)1891 void MoveCmpOpt::Run(BB &bb, Insn &insn)
1892 {
1893 if (insn.GetMachineOpcode() != MOP_xmovri64 && insn.GetMachineOpcode() != MOP_wmovri32) {
1894 return;
1895 }
1896 ImmOperand &immOpnd = static_cast<ImmOperand &>(insn.GetOperand(kInsnSecondOpnd));
1897 if (!immOpnd.IsInBitSize(kMaxImmVal12Bits, 0) && !immOpnd.IsInBitSize(kMaxImmVal12Bits, kMaxImmVal12Bits)) {
1898 return;
1899 }
1900 Insn *nextInsn = insn.GetNextMachineInsn();
1901 if (nextInsn == nullptr ||
1902 (nextInsn->GetMachineOpcode() != MOP_wcmprr && nextInsn->GetMachineOpcode() != MOP_xcmprr)) {
1903 return;
1904 }
1905 RegOperand &cmpSecondOpnd = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnSecondOpnd));
1906 RegOperand &cmpThirdOpnd = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnThirdOpnd));
1907 RegOperand &movFirstOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1908 if (cmpSecondOpnd.GetRegisterNumber() == cmpThirdOpnd.GetRegisterNumber()) {
1909 return;
1910 }
1911 if (cmpThirdOpnd.GetRegisterNumber() != movFirstOpnd.GetRegisterNumber()) {
1912 return;
1913 }
1914 MOperator cmpOpCode = (cmpThirdOpnd.GetSize() == k64BitSize) ? MOP_xcmpri : MOP_wcmpri;
1915 Insn &newCmpInsn = cgFunc->GetInsnBuilder()->BuildInsn(cmpOpCode, nextInsn->GetOperand(kInsnFirstOpnd),
1916 nextInsn->GetOperand(kInsnSecondOpnd), immOpnd);
1917 bb.ReplaceInsn(*nextInsn, newCmpInsn);
1918 if (!IfOperandIsLiveAfterInsn(movFirstOpnd, newCmpInsn)) {
1919 bb.RemoveInsn(insn);
1920 }
1921 }
1922
Run(BB & bb,Insn & insn)1923 void AddImmZeroToMov::Run(BB &bb, Insn &insn)
1924 {
1925 RegOperand *insnDefReg = &static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1926 RegOperand *insnUseReg = &static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
1927 int64 immVal = static_cast<ImmOperand &>(insn.GetOperand(kInsnThirdOpnd)).GetValue();
1928 if (immVal == static_cast<int64>(k0BitSize)) {
1929 if (insnDefReg->GetRegisterNumber() == insnUseReg->GetRegisterNumber()) {
1930 bb.RemoveInsn(insn);
1931 return;
1932 } else {
1933 Insn *newInsn = &cgFunc->GetInsnBuilder()->BuildInsn(MOP_xmovrr, *insnDefReg, *insnUseReg);
1934 bb.ReplaceInsn(insn, *newInsn);
1935 return;
1936 }
1937 }
1938 return;
1939 }
1940
FindPreviousCmp(Insn & insn) const1941 Insn *AndCmpBranchesToCsetAArch64::FindPreviousCmp(Insn &insn) const
1942 {
1943 regno_t defRegNO = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd)).GetRegisterNumber();
1944 for (Insn *curInsn = insn.GetPrev(); curInsn != nullptr; curInsn = curInsn->GetPrev()) {
1945 if (!curInsn->IsMachineInstruction()) {
1946 continue;
1947 }
1948 if (curInsn->GetMachineOpcode() == MOP_wcmpri || curInsn->GetMachineOpcode() == MOP_xcmpri) {
1949 return curInsn;
1950 }
1951 /*
1952 * if any def/use of CC or insn defReg between insn and curInsn, stop searching and return nullptr.
1953 */
1954 if (curInsn->ScanReg(defRegNO) || curInsn->ScanReg(kRFLAG)) {
1955 return nullptr;
1956 }
1957 }
1958 return nullptr;
1959 }
1960
Run(BB & bb,Insn & insn)1961 void AndCmpBranchesToCsetAArch64::Run(BB &bb, Insn &insn)
1962 {
1963 /* prevInsn must be "cmp" insn */
1964 Insn *prevInsn = FindPreviousCmp(insn);
1965 if (prevInsn == nullptr) {
1966 return;
1967 }
1968 /* prevPrevInsn must be "and" insn */
1969 Insn *prevPrevInsn = prevInsn->GetPreviousMachineInsn();
1970 if (prevPrevInsn == nullptr ||
1971 (prevPrevInsn->GetMachineOpcode() != MOP_wandrri12 && prevPrevInsn->GetMachineOpcode() != MOP_xandrri13)) {
1972 return;
1973 }
1974
1975 auto &csetCond = static_cast<CondOperand &>(insn.GetOperand(kInsnSecondOpnd));
1976 auto &cmpImm = static_cast<ImmOperand &>(prevInsn->GetOperand(kInsnThirdOpnd));
1977 int64 cmpImmVal = cmpImm.GetValue();
1978 auto &andImm = static_cast<ImmOperand &>(prevPrevInsn->GetOperand(kInsnThirdOpnd));
1979 int64 andImmVal = andImm.GetValue();
1980 if ((csetCond.GetCode() == CC_EQ && cmpImmVal == andImmVal) || (csetCond.GetCode() == CC_NE && cmpImmVal == 0)) {
1981 /* if flag_reg of "cmp" is live later, we can't remove cmp insn. */
1982 auto &flagReg = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1983 if (IfOperandIsLiveAfterInsn(flagReg, insn)) {
1984 return;
1985 }
1986
1987 auto &csetReg = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1988 auto &prevInsnSecondReg = prevInsn->GetOperand(kInsnSecondOpnd);
1989 bool isRegDiff = !RegOperand::IsSameRegNO(csetReg, prevInsnSecondReg);
1990 if (isRegDiff && IfOperandIsLiveAfterInsn(static_cast<RegOperand &>(prevInsnSecondReg), insn)) {
1991 return;
1992 }
1993 if (andImmVal == 1) {
1994 if (!RegOperand::IsSameRegNO(prevInsnSecondReg, prevPrevInsn->GetOperand(kInsnFirstOpnd))) {
1995 return;
1996 }
1997 /* save the "and" insn only. */
1998 bb.RemoveInsn(insn);
1999 bb.RemoveInsn(*prevInsn);
2000 if (isRegDiff) {
2001 prevPrevInsn->Insn::SetOperand(kInsnFirstOpnd, csetReg);
2002 }
2003 } else {
2004 if (!RegOperand::IsSameReg(prevInsnSecondReg, prevPrevInsn->GetOperand(kInsnFirstOpnd)) ||
2005 !RegOperand::IsSameReg(prevInsnSecondReg, prevPrevInsn->GetOperand(kInsnSecondOpnd))) {
2006 return;
2007 }
2008
2009 /* andImmVal is n power of 2 */
2010 int n = LogValueAtBase2(andImmVal);
2011 if (n < 0) {
2012 return;
2013 }
2014
2015 /* create ubfx insn */
2016 MOperator ubfxOp = (csetReg.GetSize() <= k32BitSize) ? MOP_wubfxrri5i5 : MOP_xubfxrri6i6;
2017 if (ubfxOp == MOP_wubfxrri5i5 && static_cast<uint32>(n) >= k32BitSize) {
2018 return;
2019 }
2020 auto &dstReg = static_cast<RegOperand &>(csetReg);
2021 auto &srcReg = static_cast<RegOperand &>(prevInsnSecondReg);
2022 auto *aarch64CGFunc = static_cast<AArch64CGFunc *>(&cgFunc);
2023 ImmOperand &bitPos = aarch64CGFunc->CreateImmOperand(n, k8BitSize, false);
2024 ImmOperand &bitSize = aarch64CGFunc->CreateImmOperand(1, k8BitSize, false);
2025 Insn &ubfxInsn = cgFunc.GetInsnBuilder()->BuildInsn(ubfxOp, dstReg, srcReg, bitPos, bitSize);
2026 bb.InsertInsnBefore(*prevPrevInsn, ubfxInsn);
2027 bb.RemoveInsn(insn);
2028 bb.RemoveInsn(*prevInsn);
2029 bb.RemoveInsn(*prevPrevInsn);
2030 }
2031 }
2032 }
2033
Run(BB & bb,Insn & insn)2034 void AndCbzBranchesToTstAArch64::Run(BB &bb, Insn &insn)
2035 {
2036 /* nextInsn must be "cbz" or "cbnz" insn */
2037 Insn *nextInsn = insn.GetNextMachineInsn();
2038 if (nextInsn == nullptr || (nextInsn->GetMachineOpcode() != MOP_wcbz && nextInsn->GetMachineOpcode() != MOP_xcbz)) {
2039 return;
2040 }
2041 auto &andRegOp = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
2042 regno_t andRegNO1 = andRegOp.GetRegisterNumber();
2043 auto &cbzRegOp2 = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnFirstOpnd));
2044 regno_t cbzRegNO2 = cbzRegOp2.GetRegisterNumber();
2045 if (andRegNO1 != cbzRegNO2) {
2046 return;
2047 }
2048 /* If the reg will be used later, we shouldn't optimize the and insn here */
2049 if (IfOperandIsLiveAfterInsn(andRegOp, *nextInsn)) {
2050 return;
2051 }
2052 /* build tst insn */
2053 Operand &andOpnd3 = insn.GetOperand(kInsnThirdOpnd);
2054 auto &andRegOp2 = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
2055 MOperator newTstOp = MOP_undef;
2056 if (andOpnd3.IsRegister()) {
2057 newTstOp = (andRegOp2.GetSize() <= k32BitSize && andOpnd3.GetSize() <= k32BitSize) ? MOP_wtstrr : MOP_xtstrr;
2058 } else {
2059 newTstOp =
2060 (andRegOp2.GetSize() <= k32BitSize && andOpnd3.GetSize() <= k32BitSize) ? MOP_wtstri32 : MOP_xtstri64;
2061 }
2062 Operand &rflag = static_cast<AArch64CGFunc *>(&cgFunc)->GetOrCreateRflag();
2063 Insn &newInsnTst = cgFunc.GetInsnBuilder()->BuildInsn(newTstOp, rflag, andRegOp2, andOpnd3);
2064 if (andOpnd3.IsImmediate()) {
2065 if (!static_cast<ImmOperand &>(andOpnd3).IsBitmaskImmediate(andRegOp2.GetSize())) {
2066 return;
2067 }
2068 }
2069 /* build beq insn */
2070 MOperator opCode = nextInsn->GetMachineOpcode();
2071 bool reverse = (opCode == MOP_xcbz || opCode == MOP_wcbz);
2072 auto &label = static_cast<LabelOperand &>(nextInsn->GetOperand(kInsnSecondOpnd));
2073 MOperator jmpOperator = reverse ? MOP_beq : MOP_bne;
2074 Insn &newInsnJmp = cgFunc.GetInsnBuilder()->BuildInsn(jmpOperator, rflag, label);
2075 bb.ReplaceInsn(insn, newInsnTst);
2076 bb.ReplaceInsn(*nextInsn, newInsnJmp);
2077 }
2078
Run(BB & bb,Insn & insn)2079 void AndCmpBranchesToTstAArch64::Run(BB &bb, Insn &insn)
2080 {
2081 /* nextInsn must be "cmp" insn */
2082 Insn *nextInsn = insn.GetNextMachineInsn();
2083 if (nextInsn == nullptr ||
2084 (nextInsn->GetMachineOpcode() != MOP_wcmpri && nextInsn->GetMachineOpcode() != MOP_xcmpri)) {
2085 return;
2086 }
2087 /* nextNextInsn must be "beq" or "bne" insn */
2088 Insn *nextNextInsn = nextInsn->GetNextMachineInsn();
2089 if (nextNextInsn == nullptr ||
2090 (nextNextInsn->GetMachineOpcode() != MOP_beq && nextNextInsn->GetMachineOpcode() != MOP_bne)) {
2091 return;
2092 }
2093 auto &andRegOp = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
2094 regno_t andRegNO1 = andRegOp.GetRegisterNumber();
2095 auto &cmpRegOp2 = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnSecondOpnd));
2096 regno_t cmpRegNO2 = cmpRegOp2.GetRegisterNumber();
2097 if (andRegNO1 != cmpRegNO2) {
2098 return;
2099 }
2100 /* If the reg will be used later, we shouldn't optimize the and insn here */
2101 if (IfOperandIsLiveAfterInsn(andRegOp, *nextInsn)) {
2102 return;
2103 }
2104 Operand &immOpnd = nextInsn->GetOperand(kInsnThirdOpnd);
2105 DEBUG_ASSERT(immOpnd.IsIntImmediate(), "expects ImmOperand");
2106 auto &defConst = static_cast<ImmOperand &>(immOpnd);
2107 int64 defConstValue = defConst.GetValue();
2108 if (defConstValue != 0) {
2109 return;
2110 }
2111 /* build tst insn */
2112 Operand &andOpnd3 = insn.GetOperand(kInsnThirdOpnd);
2113 auto &andRegOp2 = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
2114 MOperator newOp = MOP_undef;
2115 if (andOpnd3.IsRegister()) {
2116 newOp = (andRegOp2.GetSize() <= k32BitSize) ? MOP_wtstrr : MOP_xtstrr;
2117 } else {
2118 newOp = (andRegOp2.GetSize() <= k32BitSize) ? MOP_wtstri32 : MOP_xtstri64;
2119 }
2120 Operand &rflag = static_cast<AArch64CGFunc *>(&cgFunc)->GetOrCreateRflag();
2121 Insn &newInsn = cgFunc.GetInsnBuilder()->BuildInsn(newOp, rflag, andRegOp2, andOpnd3);
2122 if (CGOptions::DoCGSSA() && CGOptions::GetInstance().GetOptimizeLevel() < CGOptions::kLevel0) {
2123 CHECK_FATAL(false, "check this case in ssa opt");
2124 }
2125 bb.InsertInsnAfter(*nextInsn, newInsn);
2126 bb.RemoveInsn(insn);
2127 bb.RemoveInsn(*nextInsn);
2128 }
2129
Run(BB & bb,Insn & insn)2130 void ZeroCmpBranchesAArch64::Run(BB &bb, Insn &insn)
2131 {
2132 Insn *prevInsn = insn.GetPreviousMachineInsn();
2133 if (!insn.IsBranch() || insn.GetOperandSize() <= kInsnSecondOpnd || prevInsn == nullptr) {
2134 return;
2135 }
2136 if (!insn.GetOperand(kInsnSecondOpnd).IsLabel()) {
2137 return;
2138 }
2139 LabelOperand *label = &static_cast<LabelOperand &>(insn.GetOperand(kInsnSecondOpnd));
2140 RegOperand *regOpnd = nullptr;
2141 RegOperand *reg0 = nullptr;
2142 RegOperand *reg1 = nullptr;
2143 MOperator newOp = MOP_undef;
2144 ImmOperand *imm = nullptr;
2145 switch (prevInsn->GetMachineOpcode()) {
2146 case MOP_wcmpri:
2147 case MOP_xcmpri: {
2148 regOpnd = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnSecondOpnd));
2149 imm = &static_cast<ImmOperand &>(prevInsn->GetOperand(kInsnThirdOpnd));
2150 if (imm->GetValue() != 0) {
2151 return;
2152 }
2153 if (insn.GetMachineOpcode() == MOP_bge) {
2154 newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbz : MOP_xtbz;
2155 } else if (insn.GetMachineOpcode() == MOP_blt) {
2156 newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbnz : MOP_xtbnz;
2157 } else {
2158 return;
2159 }
2160 break;
2161 }
2162 case MOP_wcmprr:
2163 case MOP_xcmprr: {
2164 reg0 = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnSecondOpnd));
2165 reg1 = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnThirdOpnd));
2166 if (!IsZeroRegister(*reg0) && !IsZeroRegister(*reg1)) {
2167 return;
2168 }
2169 switch (insn.GetMachineOpcode()) {
2170 case MOP_bge:
2171 if (IsZeroRegister(*reg1)) {
2172 regOpnd = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnSecondOpnd));
2173 newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbz : MOP_xtbz;
2174 } else {
2175 return;
2176 }
2177 break;
2178 case MOP_ble:
2179 if (IsZeroRegister(*reg0)) {
2180 regOpnd = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnThirdOpnd));
2181 newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbz : MOP_xtbz;
2182 } else {
2183 return;
2184 }
2185 break;
2186 case MOP_blt:
2187 if (IsZeroRegister(*reg1)) {
2188 regOpnd = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnSecondOpnd));
2189 newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbnz : MOP_xtbnz;
2190 } else {
2191 return;
2192 }
2193 break;
2194 case MOP_bgt:
2195 if (IsZeroRegister(*reg0)) {
2196 regOpnd = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnThirdOpnd));
2197 newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbnz : MOP_xtbnz;
2198 } else {
2199 return;
2200 }
2201 break;
2202 default:
2203 return;
2204 }
2205 break;
2206 }
2207 default:
2208 return;
2209 }
2210 auto aarch64CGFunc = static_cast<AArch64CGFunc *>(&cgFunc);
2211 ImmOperand &bitp = aarch64CGFunc->CreateImmOperand(
2212 (regOpnd->GetSize() <= k32BitSize) ? (k32BitSize - 1) : (k64BitSize - 1), k8BitSize, false);
2213 bb.InsertInsnAfter(insn,
2214 cgFunc.GetInsnBuilder()->BuildInsn(newOp, *static_cast<RegOperand *>(regOpnd), bitp, *label));
2215 bb.RemoveInsn(insn);
2216 bb.RemoveInsn(*prevInsn);
2217 }
2218
2219 /*
2220 * if there is define point of checkInsn->GetOperand(opndIdx) between startInsn and firstInsn
2221 * return define insn. else return nullptr
2222 */
DefInsnOfOperandInBB(const Insn & startInsn,const Insn & checkInsn,int opndIdx) const2223 const Insn *CmpCsetAArch64::DefInsnOfOperandInBB(const Insn &startInsn, const Insn &checkInsn, int opndIdx) const
2224 {
2225 Insn *prevInsn = nullptr;
2226 for (const Insn *insn = &startInsn; insn != nullptr; insn = prevInsn) {
2227 prevInsn = insn->GetPreviousMachineInsn();
2228 if (!insn->IsMachineInstruction()) {
2229 continue;
2230 }
2231 /* checkInsn.GetOperand(opndIdx) is thought modified conservatively */
2232 if (insn->IsCall()) {
2233 return insn;
2234 }
2235 const InsnDesc *md = insn->GetDesc();
2236 uint32 opndNum = insn->GetOperandSize();
2237 for (uint32 i = 0; i < opndNum; ++i) {
2238 Operand &opnd = insn->GetOperand(i);
2239 if (!md->opndMD[i]->IsDef()) {
2240 continue;
2241 }
2242 /* Operand is base reg of Memory, defined by str */
2243 if (opnd.IsMemoryAccessOperand()) {
2244 auto &memOpnd = static_cast<MemOperand &>(opnd);
2245 RegOperand *base = memOpnd.GetBaseRegister();
2246 DEBUG_ASSERT(base != nullptr, "nullptr check");
2247 DEBUG_ASSERT(base->IsRegister(), "expects RegOperand");
2248 if (RegOperand::IsSameRegNO(*base, checkInsn.GetOperand(static_cast<uint32>(opndIdx))) &&
2249 memOpnd.GetAddrMode() == MemOperand::kAddrModeBOi &&
2250 (memOpnd.IsPostIndexed() || memOpnd.IsPreIndexed())) {
2251 return insn;
2252 }
2253 } else {
2254 DEBUG_ASSERT(opnd.IsRegister(), "expects RegOperand");
2255 if (RegOperand::IsSameRegNO(checkInsn.GetOperand(static_cast<uint32>(opndIdx)), opnd)) {
2256 return insn;
2257 }
2258 }
2259 }
2260 }
2261 return nullptr;
2262 }
2263
OpndDefByOneValidBit(const Insn & defInsn) const2264 bool CmpCsetAArch64::OpndDefByOneValidBit(const Insn &defInsn) const
2265 {
2266 MOperator defMop = defInsn.GetMachineOpcode();
2267 switch (defMop) {
2268 case MOP_wcsetrc:
2269 case MOP_xcsetrc:
2270 return true;
2271 case MOP_wmovri32:
2272 case MOP_xmovri64: {
2273 Operand &defOpnd = defInsn.GetOperand(kInsnSecondOpnd);
2274 DEBUG_ASSERT(defOpnd.IsIntImmediate(), "expects ImmOperand");
2275 auto &defConst = static_cast<ImmOperand &>(defOpnd);
2276 int64 defConstValue = defConst.GetValue();
2277 return (defConstValue == 0 || defConstValue == 1);
2278 }
2279 case MOP_xmovrr:
2280 case MOP_wmovrr:
2281 return IsZeroRegister(defInsn.GetOperand(kInsnSecondOpnd));
2282 case MOP_wlsrrri5:
2283 case MOP_xlsrrri6: {
2284 Operand &opnd2 = defInsn.GetOperand(kInsnThirdOpnd);
2285 DEBUG_ASSERT(opnd2.IsIntImmediate(), "expects ImmOperand");
2286 auto &opndImm = static_cast<ImmOperand &>(opnd2);
2287 int64 shiftBits = opndImm.GetValue();
2288 return ((defMop == MOP_wlsrrri5 && shiftBits == (k32BitSize - 1)) ||
2289 (defMop == MOP_xlsrrri6 && shiftBits == (k64BitSize - 1)));
2290 }
2291 default:
2292 return false;
2293 }
2294 }
2295
2296 /*
2297 * help function for cmpcset optimize
2298 * if all define points of used opnd in insn has only one valid bit,return true.
2299 * for cmp reg,#0(#1), that is checking for reg
2300 */
CheckOpndDefPoints(Insn & checkInsn,int opndIdx)2301 bool CmpCsetAArch64::CheckOpndDefPoints(Insn &checkInsn, int opndIdx)
2302 {
2303 if (checkInsn.GetBB()->GetPrev() == nullptr) {
2304 /* For 1st BB, be conservative for def of parameter registers */
2305 /* Since peep is light weight, do not want to insert pseudo defs */
2306 regno_t reg = static_cast<RegOperand &>(checkInsn.GetOperand(static_cast<uint32>(opndIdx))).GetRegisterNumber();
2307 if ((reg >= R0 && reg <= R7) || (reg >= D0 && reg <= D7)) {
2308 return false;
2309 }
2310 }
2311 /* check current BB */
2312 const Insn *defInsn = DefInsnOfOperandInBB(checkInsn, checkInsn, opndIdx);
2313 if (defInsn != nullptr) {
2314 return OpndDefByOneValidBit(*defInsn);
2315 }
2316 /* check pred */
2317 for (auto predBB : checkInsn.GetBB()->GetPreds()) {
2318 const Insn *tempInsn = nullptr;
2319 if (predBB->GetLastInsn() != nullptr) {
2320 tempInsn = DefInsnOfOperandInBB(*predBB->GetLastInsn(), checkInsn, opndIdx);
2321 }
2322 if (tempInsn == nullptr || !OpndDefByOneValidBit(*tempInsn)) {
2323 return false;
2324 }
2325 }
2326 return true;
2327 }
2328
2329 /* Check there is use point of rflag start from startInsn to current bb bottom */
FlagUsedLaterInCurBB(const BB & bb,Insn & startInsn) const2330 bool CmpCsetAArch64::FlagUsedLaterInCurBB(const BB &bb, Insn &startInsn) const
2331 {
2332 if (&bb != startInsn.GetBB()) {
2333 return false;
2334 }
2335 Insn *nextInsn = nullptr;
2336 for (Insn *insn = &startInsn; insn != nullptr; insn = nextInsn) {
2337 nextInsn = insn->GetNextMachineInsn();
2338 const InsnDesc *md = insn->GetDesc();
2339 uint32 opndNum = insn->GetOperandSize();
2340 for (uint32 i = 0; i < opndNum; ++i) {
2341 Operand &opnd = insn->GetOperand(i);
2342 /*
2343 * For condition operand, such as NE, EQ and so on, the register number should be
2344 * same with RFLAG, we only need check the property of use/def.
2345 */
2346 if (!opnd.IsConditionCode()) {
2347 continue;
2348 }
2349 if (md->opndMD[i]->IsUse()) {
2350 return true;
2351 } else {
2352 DEBUG_ASSERT(md->opndMD[i]->IsDef(), "register should be redefined.");
2353 return false;
2354 }
2355 }
2356 }
2357 return false;
2358 }
2359
Run(BB & bb,Insn & insn)2360 void CmpCsetAArch64::Run(BB &bb, Insn &insn)
2361 {
2362 Insn *nextInsn = insn.GetNextMachineInsn();
2363 if (nextInsn == nullptr) {
2364 return;
2365 }
2366 MOperator firstMop = insn.GetMachineOpcode();
2367 MOperator secondMop = nextInsn->GetMachineOpcode();
2368 if ((firstMop == MOP_wcmpri || firstMop == MOP_xcmpri) && (secondMop == MOP_wcsetrc || secondMop == MOP_xcsetrc)) {
2369 Operand &cmpFirstOpnd = insn.GetOperand(kInsnSecondOpnd);
2370 /* get ImmOperand, must be 0 or 1 */
2371 Operand &cmpSecondOpnd = insn.GetOperand(kInsnThirdOpnd);
2372 auto &cmpFlagReg = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
2373 DEBUG_ASSERT(cmpSecondOpnd.IsIntImmediate(), "expects ImmOperand");
2374 auto &cmpConst = static_cast<ImmOperand &>(cmpSecondOpnd);
2375 int64 cmpConstVal = cmpConst.GetValue();
2376 Operand &csetFirstOpnd = nextInsn->GetOperand(kInsnFirstOpnd);
2377 if ((cmpConstVal != 0 && cmpConstVal != 1) || !CheckOpndDefPoints(insn, 1) ||
2378 (nextInsn->GetNextMachineInsn() != nullptr && FlagUsedLaterInCurBB(bb, *nextInsn->GetNextMachineInsn())) ||
2379 FindRegLiveOut(cmpFlagReg, *insn.GetBB())) {
2380 return;
2381 }
2382
2383 Insn *csetInsn = nextInsn;
2384 nextInsn = nextInsn->GetNextMachineInsn();
2385 auto &cond = static_cast<CondOperand &>(csetInsn->GetOperand(kInsnSecondOpnd));
2386 if ((cmpConstVal == 0 && cond.GetCode() == CC_NE) || (cmpConstVal == 1 && cond.GetCode() == CC_EQ)) {
2387 if (RegOperand::IsSameRegNO(cmpFirstOpnd, csetFirstOpnd)) {
2388 bb.RemoveInsn(insn);
2389 bb.RemoveInsn(*csetInsn);
2390 } else {
2391 if (cmpFirstOpnd.GetSize() != csetFirstOpnd.GetSize()) {
2392 return;
2393 }
2394 MOperator mopCode = (cmpFirstOpnd.GetSize() == k64BitSize) ? MOP_xmovrr : MOP_wmovrr;
2395 Insn &newInsn = cgFunc.GetInsnBuilder()->BuildInsn(mopCode, csetFirstOpnd, cmpFirstOpnd);
2396 bb.ReplaceInsn(insn, newInsn);
2397 bb.RemoveInsn(*csetInsn);
2398 }
2399 } else if ((cmpConstVal == 1 && cond.GetCode() == CC_NE) || (cmpConstVal == 0 && cond.GetCode() == CC_EQ)) {
2400 if (cmpFirstOpnd.GetSize() != csetFirstOpnd.GetSize()) {
2401 return;
2402 }
2403 MOperator mopCode = (cmpFirstOpnd.GetSize() == k64BitSize) ? MOP_xeorrri13 : MOP_weorrri12;
2404 ImmOperand &one = static_cast<AArch64CGFunc *>(&cgFunc)->CreateImmOperand(1, k8BitSize, false);
2405 Insn &newInsn = cgFunc.GetInsnBuilder()->BuildInsn(mopCode, csetFirstOpnd, cmpFirstOpnd, one);
2406 bb.ReplaceInsn(insn, newInsn);
2407 bb.RemoveInsn(*csetInsn);
2408 }
2409 }
2410 }
2411
2412 /* ldr wn, [x1, wn, SXTW]
2413 * add x2, wn, x2
2414 */
IsExpandBaseOpnd(const Insn & insn,const Insn & prevInsn) const2415 bool ComplexMemOperandAddAArch64::IsExpandBaseOpnd(const Insn &insn, const Insn &prevInsn) const
2416 {
2417 MOperator prevMop = prevInsn.GetMachineOpcode();
2418 if (prevMop >= MOP_wldrsb && prevMop <= MOP_xldr &&
2419 prevInsn.GetOperand(kInsnFirstOpnd).Equals(insn.GetOperand(kInsnSecondOpnd))) {
2420 return true;
2421 }
2422 return false;
2423 }
2424
Run(BB & bb,Insn & insn)2425 void ComplexMemOperandAddAArch64::Run(BB &bb, Insn &insn)
2426 {
2427 AArch64CGFunc *aarch64CGFunc = static_cast<AArch64CGFunc *>(&cgFunc);
2428 Insn *nextInsn = insn.GetNextMachineInsn();
2429 if (nextInsn == nullptr) {
2430 return;
2431 }
2432 Insn *prevInsn = insn.GetPreviousMachineInsn();
2433 MOperator thisMop = insn.GetMachineOpcode();
2434 if (thisMop != MOP_xaddrrr && thisMop != MOP_waddrrr) {
2435 return;
2436 }
2437 MOperator nextMop = nextInsn->GetMachineOpcode();
2438 if (nextMop && ((nextMop >= MOP_wldrsb && nextMop <= MOP_dldr) || (nextMop >= MOP_wstrb && nextMop <= MOP_dstr))) {
2439 if (!IsMemOperandOptPattern(insn, *nextInsn)) {
2440 return;
2441 }
2442 MemOperand *memOpnd = static_cast<MemOperand *>(nextInsn->GetMemOpnd());
2443 auto newBaseOpnd = static_cast<RegOperand *>(&insn.GetOperand(kInsnSecondOpnd));
2444 auto newIndexOpnd = static_cast<RegOperand *>(&insn.GetOperand(kInsnThirdOpnd));
2445 regno_t memBaseOpndRegNO = newBaseOpnd->GetRegisterNumber();
2446 if (newBaseOpnd->GetSize() <= k32BitSize && prevInsn != nullptr && IsExpandBaseOpnd(insn, *prevInsn)) {
2447 newBaseOpnd = &aarch64CGFunc->GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(memBaseOpndRegNO),
2448 k64BitSize, kRegTyInt);
2449 }
2450 if (newBaseOpnd->GetSize() != k64BitSize) {
2451 return;
2452 }
2453 DEBUG_ASSERT(memOpnd != nullptr, "memOpnd should not be nullptr");
2454 if (newIndexOpnd->GetSize() <= k32BitSize) {
2455 MemOperand &newMemOpnd = aarch64CGFunc->GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, memOpnd->GetSize(),
2456 newBaseOpnd, newIndexOpnd, 0, false);
2457 nextInsn->SetOperand(kInsnSecondOpnd, newMemOpnd);
2458 } else {
2459 MemOperand &newMemOpnd = aarch64CGFunc->GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, memOpnd->GetSize(),
2460 newBaseOpnd, newIndexOpnd, nullptr, nullptr);
2461 nextInsn->SetOperand(kInsnSecondOpnd, newMemOpnd);
2462 }
2463 bb.RemoveInsn(insn);
2464 }
2465 }
2466
Run(BB & bb,Insn & insn)2467 void RemoveSxtBeforeStrAArch64::Run(BB &bb, Insn &insn)
2468 {
2469 MOperator mop = insn.GetMachineOpcode();
2470 Insn *prevInsn = insn.GetPreviousMachineInsn();
2471 if (prevInsn == nullptr) {
2472 return;
2473 }
2474 MOperator prevMop = prevInsn->GetMachineOpcode();
2475 if (!(mop == MOP_wstrh && prevMop == MOP_xsxth32) && !(mop == MOP_wstrb && prevMop == MOP_xsxtb32)) {
2476 return;
2477 }
2478 auto &prevOpnd0 = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
2479 if (IfOperandIsLiveAfterInsn(prevOpnd0, insn)) {
2480 return;
2481 }
2482 auto &prevOpnd1 = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnSecondOpnd));
2483 regno_t prevRegNO0 = prevOpnd0.GetRegisterNumber();
2484 regno_t prevRegNO1 = prevOpnd1.GetRegisterNumber();
2485 regno_t regNO0 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd)).GetRegisterNumber();
2486 if (prevRegNO0 != prevRegNO1) {
2487 return;
2488 }
2489 if (prevRegNO0 == regNO0) {
2490 bb.RemoveInsn(*prevInsn);
2491 return;
2492 }
2493 insn.SetOperand(0, prevOpnd1);
2494 bb.RemoveInsn(*prevInsn);
2495 }
2496 } /* namespace maplebe */
2497