1 /*
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "aarch64_peep.h"
17 #include "aarch64_cg.h"
18
19 namespace maplebe {
20 #define CG_PEEP_DUMP CG_DEBUG_FUNC(*cgFunc)
21
IsZeroRegister(const Operand & opnd)22 static bool IsZeroRegister(const Operand &opnd)
23 {
24 if (!opnd.IsRegister()) {
25 return false;
26 }
27 const RegOperand *regOpnd = static_cast<const RegOperand *>(&opnd);
28 return regOpnd->GetRegisterNumber() == RZR;
29 }
30
Run()31 void AArch64CGPeepHole::Run()
32 {
33 FOR_ALL_BB(bb, cgFunc)
34 {
35 FOR_BB_INSNS_SAFE(insn, bb, nextInsn)
36 {
37 if (!insn->IsMachineInstruction()) {
38 continue;
39 }
40 DoNormalOptimize(*bb, *insn);
41 }
42 }
43 }
44
DoNormalOptimize(BB & bb,Insn & insn)45 void AArch64CGPeepHole::DoNormalOptimize(BB &bb, Insn &insn)
46 {
47 MOperator thisMop = insn.GetMachineOpcode();
48 manager = peepMemPool->New<PeepOptimizeManager>(*cgFunc, bb, insn);
49 switch (thisMop) {
50 /*
51 * e.g.
52 * execute before & after RA: manager->NormalPatternOpt<>(true)
53 * execute before RA: manager->NormalPatternOpt<>(!cgFunc->IsAfterRegAlloc())
54 * execute after RA: manager->NormalPatternOpt<>(cgFunc->IsAfterRegAlloc())
55 */
56 case MOP_wmovrr:
57 case MOP_xmovrr:
58 case MOP_xvmovs:
59 case MOP_xvmovd: {
60 manager->NormalPatternOpt<RemoveMovingtoSameRegPattern>(cgFunc->IsAfterRegAlloc());
61 break;
62 }
63 case MOP_wstrb:
64 case MOP_wldrb: {
65 // only strb ldrb can do this pattern, other patterns still need to be done, so there is no break here.
66 break;
67 }
68 case MOP_wstrh:
69 case MOP_wldrh:
70 case MOP_xldr:
71 case MOP_xstr:
72 case MOP_wldr:
73 case MOP_wstr:
74 case MOP_dldr:
75 case MOP_dstr:
76 case MOP_sldr:
77 case MOP_sstr:
78 case MOP_qldr:
79 case MOP_qstr: {
80 manager->NormalPatternOpt<CombineContiLoadAndStorePattern>(cgFunc->IsAfterRegAlloc());
81 if (!manager->OptSuccess()) {
82 manager->NormalPatternOpt<ContiLDRorSTRToSameMEMPattern>(cgFunc->IsAfterRegAlloc());
83 }
84 break;
85 }
86 case MOP_xvmovrv:
87 case MOP_xvmovrd: {
88 manager->NormalPatternOpt<FmovRegPattern>(cgFunc->IsAfterRegAlloc());
89 break;
90 }
91 case MOP_xsbfxrri6i6: {
92 manager->NormalPatternOpt<SbfxOptPattern>(cgFunc->IsAfterRegAlloc());
93 break;
94 }
95 case MOP_wcbz:
96 case MOP_xcbz:
97 case MOP_wcbnz:
98 case MOP_xcbnz: {
99 manager->NormalPatternOpt<CbnzToCbzPattern>(cgFunc->IsAfterRegAlloc());
100 break;
101 }
102 default:
103 break;
104 }
105 }
106 /* ======== CGPeepPattern End ======== */
107
InitOpts()108 void AArch64PeepHole::InitOpts()
109 {
110 optimizations.resize(kPeepholeOptsNum);
111 optimizations[kEliminateSpecifcSXTOpt] = optOwnMemPool->New<EliminateSpecifcSXTAArch64>(cgFunc);
112 optimizations[kEliminateSpecifcUXTOpt] = optOwnMemPool->New<EliminateSpecifcUXTAArch64>(cgFunc);
113 optimizations[kCsetCbzToBeqOpt] = optOwnMemPool->New<CsetCbzToBeqOptAArch64>(cgFunc);
114 optimizations[kAndCmpBranchesToCsetOpt] = optOwnMemPool->New<AndCmpBranchesToCsetAArch64>(cgFunc);
115 optimizations[kAndCmpBranchesToTstOpt] = optOwnMemPool->New<AndCmpBranchesToTstAArch64>(cgFunc);
116 optimizations[kAndCbzBranchesToTstOpt] = optOwnMemPool->New<AndCbzBranchesToTstAArch64>(cgFunc);
117 optimizations[kZeroCmpBranchesOpt] = optOwnMemPool->New<ZeroCmpBranchesAArch64>(cgFunc);
118 optimizations[kCselZeroOneToCsetOpt] = optOwnMemPool->New<CselZeroOneToCsetOpt>(cgFunc);
119 optimizations[kAndCmpCsetEorCbzOpt] = optOwnMemPool->New<AndCmpCsetEorCbzOpt>(cgFunc);
120 optimizations[kAddLdrOpt] = optOwnMemPool->New<AddLdrOpt>(cgFunc);
121 optimizations[kCsetEorOpt] = optOwnMemPool->New<CsetEorOpt>(cgFunc);
122 optimizations[kMoveCmpOpt] = optOwnMemPool->New<MoveCmpOpt>(cgFunc);
123 }
124
Run(BB & bb,Insn & insn)125 void AArch64PeepHole::Run(BB &bb, Insn &insn)
126 {
127 MOperator thisMop = insn.GetMachineOpcode();
128 switch (thisMop) {
129 case MOP_xsxtb32:
130 case MOP_xsxth32:
131 case MOP_xsxtb64:
132 case MOP_xsxth64:
133 case MOP_xsxtw64: {
134 (static_cast<EliminateSpecifcSXTAArch64 *>(optimizations[kEliminateSpecifcSXTOpt]))->Run(bb, insn);
135 break;
136 }
137 case MOP_xuxtb32:
138 case MOP_xuxth32:
139 case MOP_xuxtw64: {
140 (static_cast<EliminateSpecifcUXTAArch64 *>(optimizations[kEliminateSpecifcUXTOpt]))->Run(bb, insn);
141 break;
142 }
143 case MOP_wcbnz:
144 case MOP_xcbnz: {
145 (static_cast<CsetCbzToBeqOptAArch64 *>(optimizations[kCsetCbzToBeqOpt]))->Run(bb, insn);
146 break;
147 }
148 case MOP_wcbz:
149 case MOP_xcbz: {
150 (static_cast<CsetCbzToBeqOptAArch64 *>(optimizations[kCsetCbzToBeqOpt]))->Run(bb, insn);
151 break;
152 }
153 case MOP_xandrrr:
154 case MOP_wandrrr:
155 case MOP_wandrri12:
156 case MOP_xandrri13: {
157 (static_cast<AndCmpCsetEorCbzOpt *>(optimizations[kAndCmpCsetEorCbzOpt]))->Run(bb, insn);
158 (static_cast<AndCmpBranchesToTstAArch64 *>(optimizations[kAndCmpBranchesToTstOpt]))->Run(bb, insn);
159 (static_cast<AndCbzBranchesToTstAArch64 *>(optimizations[kAndCbzBranchesToTstOpt]))->Run(bb, insn);
160 break;
161 }
162 case MOP_wcsetrc:
163 case MOP_xcsetrc: {
164 (static_cast<CsetEorOpt *>(optimizations[kCsetEorOpt]))->Run(bb, insn);
165 (static_cast<AndCmpBranchesToCsetAArch64 *>(optimizations[kAndCmpBranchesToCsetOpt]))->Run(bb, insn);
166 break;
167 }
168 case MOP_xmovri64:
169 case MOP_wmovri32: {
170 static_cast<MoveCmpOpt *>(optimizations[kMoveCmpOpt])->Run(bb, insn);
171 break;
172 }
173 case MOP_xaddrrr: {
174 (static_cast<AddLdrOpt *>(optimizations[kAddLdrOpt]))->Run(bb, insn);
175 break;
176 }
177 case MOP_wcselrrrc:
178 case MOP_xcselrrrc: {
179 (static_cast<CselZeroOneToCsetOpt *>(optimizations[kCselZeroOneToCsetOpt]))->Run(bb, insn);
180 break;
181 }
182 default:
183 break;
184 }
185 if (&insn == bb.GetLastInsn()) {
186 (static_cast<ZeroCmpBranchesAArch64 *>(optimizations[kZeroCmpBranchesOpt]))->Run(bb, insn);
187 }
188 }
189
InitOpts()190 void AArch64PeepHole0::InitOpts()
191 {
192 optimizations.resize(kPeepholeOptsNum);
193 optimizations[kRemoveIdenticalLoadAndStoreOpt] = optOwnMemPool->New<RemoveIdenticalLoadAndStoreAArch64>(cgFunc);
194 optimizations[kCmpCsetOpt] = optOwnMemPool->New<CmpCsetAArch64>(cgFunc);
195 optimizations[kComplexMemOperandOptAdd] = optOwnMemPool->New<ComplexMemOperandAddAArch64>(cgFunc);
196 optimizations[kRemoveSxtBeforeStrOpt] = optOwnMemPool->New<RemoveSxtBeforeStrAArch64>(cgFunc);
197 optimizations[kRedundantMovAArch64Opt] = optOwnMemPool->New<RedundantMovAArch64>(cgFunc);
198 optimizations[kRemoveMovingtoSameRegOpt] = optOwnMemPool->New<RemoveMovingtoSameRegAArch64>(cgFunc);
199 optimizations[kEnhanceStrLdrAArch64Opt] = optOwnMemPool->New<EnhanceStrLdrAArch64>(cgFunc);
200 optimizations[kAddImmZeroToMov] = optOwnMemPool->New<AddImmZeroToMov>(cgFunc);
201 }
202
Run(BB & bb,Insn & insn)203 void AArch64PeepHole0::Run(BB &bb, Insn &insn)
204 {
205 MOperator thisMop = insn.GetMachineOpcode();
206 switch (thisMop) {
207 case MOP_wcmpri:
208 case MOP_xcmpri: {
209 (static_cast<CmpCsetAArch64 *>(optimizations[kCmpCsetOpt]))->Run(bb, insn);
210 break;
211 }
212 case MOP_xaddrrr: {
213 (static_cast<ComplexMemOperandAddAArch64 *>(optimizations[kComplexMemOperandOptAdd]))->Run(bb, insn);
214 break;
215 }
216 case MOP_xaddrri12: {
217 (static_cast<AddImmZeroToMov *>(optimizations[kAddImmZeroToMov]))->Run(bb, insn);
218 break;
219 }
220 case MOP_wstrh:
221 case MOP_wstrb: {
222 (static_cast<RemoveSxtBeforeStrAArch64 *>(optimizations[kRemoveSxtBeforeStrOpt]))->Run(bb, insn);
223 break;
224 }
225 case MOP_wmovrr:
226 case MOP_xmovrr:
227 case MOP_xvmovs:
228 case MOP_xvmovd: {
229 (static_cast<RedundantMovAArch64 *>(optimizations[kRedundantMovAArch64Opt]))->Run(bb, insn);
230 (static_cast<RemoveMovingtoSameRegAArch64 *>(optimizations[kRemoveMovingtoSameRegOpt]))->Run(bb, insn);
231 break;
232 }
233 case MOP_xldr:
234 case MOP_xstr:
235 case MOP_wldr:
236 case MOP_wstr:
237 case MOP_dldr:
238 case MOP_dstr:
239 case MOP_sldr:
240 case MOP_sstr: {
241 if (thisMop == MOP_wstr || thisMop == MOP_xstr) {
242 (static_cast<RemoveIdenticalLoadAndStoreAArch64 *>(optimizations[kRemoveIdenticalLoadAndStoreOpt]))
243 ->Run(bb, insn);
244 }
245 (static_cast<EnhanceStrLdrAArch64 *>(optimizations[kEnhanceStrLdrAArch64Opt]))->Run(bb, insn);
246 break;
247 }
248 default:
249 break;
250 }
251 }
252
IsMemOperandsIdentical(const Insn & insn1,const Insn & insn2) const253 bool RemoveIdenticalLoadAndStoreAArch64::IsMemOperandsIdentical(const Insn &insn1, const Insn &insn2) const
254 {
255 regno_t regNO1 = static_cast<RegOperand &>(insn1.GetOperand(kInsnFirstOpnd)).GetRegisterNumber();
256 regno_t regNO2 = static_cast<RegOperand &>(insn2.GetOperand(kInsnFirstOpnd)).GetRegisterNumber();
257 if (regNO1 != regNO2) {
258 return false;
259 }
260 /* Match only [base + offset] */
261 auto &memOpnd1 = static_cast<MemOperand &>(insn1.GetOperand(kInsnSecondOpnd));
262 if (memOpnd1.GetAddrMode() != MemOperand::kAddrModeBOi || !memOpnd1.IsIntactIndexed()) {
263 return false;
264 }
265 auto &memOpnd2 = static_cast<MemOperand &>(insn2.GetOperand(kInsnSecondOpnd));
266 if (memOpnd2.GetAddrMode() != MemOperand::kAddrModeBOi || !memOpnd1.IsIntactIndexed()) {
267 return false;
268 }
269 Operand *base1 = memOpnd1.GetBaseRegister();
270 Operand *base2 = memOpnd2.GetBaseRegister();
271 if (!((base1 != nullptr) && base1->IsRegister()) || !((base2 != nullptr) && base2->IsRegister())) {
272 return false;
273 }
274
275 regno_t baseRegNO1 = static_cast<RegOperand *>(base1)->GetRegisterNumber();
276 /* First insn re-write base addr reg1 <- [ reg1 + offset ] */
277 if (baseRegNO1 == regNO1) {
278 return false;
279 }
280
281 regno_t baseRegNO2 = static_cast<RegOperand *>(base2)->GetRegisterNumber();
282 if (baseRegNO1 != baseRegNO2) {
283 return false;
284 }
285
286 return memOpnd1.GetOffsetImmediate()->GetOffsetValue() == memOpnd2.GetOffsetImmediate()->GetOffsetValue();
287 }
288
Run(BB & bb,Insn & insn)289 void RemoveIdenticalLoadAndStoreAArch64::Run(BB &bb, Insn &insn)
290 {
291 Insn *nextInsn = insn.GetNextMachineInsn();
292 if (nextInsn == nullptr) {
293 return;
294 }
295 MOperator mop1 = insn.GetMachineOpcode();
296 MOperator mop2 = nextInsn->GetMachineOpcode();
297 if ((mop1 == MOP_wstr && mop2 == MOP_wstr) || (mop1 == MOP_xstr && mop2 == MOP_xstr)) {
298 if (IsMemOperandsIdentical(insn, *nextInsn)) {
299 bb.RemoveInsn(insn);
300 }
301 } else if ((mop1 == MOP_wstr && mop2 == MOP_wldr) || (mop1 == MOP_xstr && mop2 == MOP_xldr)) {
302 if (IsMemOperandsIdentical(insn, *nextInsn)) {
303 bb.RemoveInsn(*nextInsn);
304 }
305 }
306 }
307
CheckCondition(Insn & insn)308 bool RemoveMovingtoSameRegPattern::CheckCondition(Insn &insn)
309 {
310 DEBUG_ASSERT(insn.GetOperand(kInsnFirstOpnd).IsRegister(), "expects registers");
311 DEBUG_ASSERT(insn.GetOperand(kInsnSecondOpnd).IsRegister(), "expects registers");
312 auto ®1 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
313 auto ®2 = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
314 /* remove mov x0,x0 when it cast i32 to i64 */
315 if ((reg1.GetRegisterNumber() == reg2.GetRegisterNumber()) && (reg1.GetSize() >= reg2.GetSize())) {
316 return true;
317 }
318 return false;
319 }
320
Run(BB & bb,Insn & insn)321 void RemoveMovingtoSameRegPattern::Run(BB &bb, Insn &insn)
322 {
323 /* remove mov x0,x0 when it cast i32 to i64 */
324 if (CheckCondition(insn)) {
325 bb.RemoveInsn(insn);
326 }
327 }
328
Run(BB & bb,Insn & insn)329 void RedundantMovAArch64::Run(BB &bb, Insn &insn)
330 {
331 auto *prevInsn = insn.GetPreviousMachineInsn();
332 if (prevInsn == nullptr) {
333 return;
334 }
335 auto *prevOpndDes = prevInsn->GetDesc()->GetOpndDes(kInsnFirstOpnd);
336 auto *srcOpndDesc = insn.GetDesc()->GetOpndDes(kInsnSecondOpnd);
337 if (!prevOpndDes->IsRegDef() || prevOpndDes->IsUse() || prevOpndDes->GetSize() > srcOpndDesc->GetSize()) {
338 return;
339 }
340 auto &srcOpnd = static_cast<RegOperand&>(insn.GetOperand(kInsnSecondOpnd));
341 auto &prevDestOpnd = static_cast<RegOperand&>(prevInsn->GetOperand(kInsnFirstOpnd));
342 if (srcOpnd.GetRegisterNumber() != prevDestOpnd.GetRegisterNumber()) {
343 return;
344 }
345 if (IfOperandIsLiveAfterInsn(srcOpnd, insn)) {
346 return;
347 }
348 auto &desOpnd = static_cast<RegOperand&>(insn.GetOperand(kInsnFirstOpnd));
349 prevInsn->SetOperand(kInsnFirstOpnd, desOpnd);
350 bb.RemoveInsn(insn);
351 }
352
Run(BB & bb,Insn & insn)353 void RemoveMovingtoSameRegAArch64::Run(BB &bb, Insn &insn)
354 {
355 DEBUG_ASSERT(insn.GetOperand(kInsnFirstOpnd).IsRegister(), "expects registers");
356 DEBUG_ASSERT(insn.GetOperand(kInsnSecondOpnd).IsRegister(), "expects registers");
357 auto ®1 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
358 auto ®2 = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
359 /* remove mov x0,x0 when it cast i32 to i64 */
360 if ((reg1.GetRegisterNumber() == reg2.GetRegisterNumber()) && (reg1.GetSize() >= reg2.GetSize())) {
361 bb.RemoveInsn(insn);
362 }
363 }
364
CheckOperandIsDeadFromInsn(const RegOperand & regOpnd,Insn & insn)365 bool EnhanceStrLdrAArch64::CheckOperandIsDeadFromInsn(const RegOperand ®Opnd, Insn &insn)
366 {
367 for (uint32 i = 0; i < insn.GetOperandSize(); ++i) {
368 auto &opnd = insn.GetOperand(i);
369 if (!insn.GetDesc()->GetOpndDes(i)->IsRegDef()) {
370 continue;
371 }
372 // regOpnd is redefined at curInsn
373 if (static_cast<RegOperand &>(opnd).GetRegisterNumber() == regOpnd.GetRegisterNumber()) {
374 return true;
375 }
376 }
377 return !IfOperandIsLiveAfterInsn(regOpnd, insn);
378 }
379
GetInsnAddOrSubNewOffset(Insn & insn,ImmOperand & offset)380 ImmOperand *EnhanceStrLdrAArch64::GetInsnAddOrSubNewOffset(Insn &insn, ImmOperand &offset)
381 {
382 int64 val = 0;
383 VaryType vary = offset.GetVary();
384 auto mOp = insn.GetMachineOpcode();
385 if (mOp == MOP_xaddrri12 || mOp == MOP_xsubrri12) {
386 auto &immOpnd = static_cast<ImmOperand &>(insn.GetOperand(kInsnThirdOpnd));
387 val = immOpnd.GetValue();
388 CHECK_FATAL(!(vary == kUnAdjustVary && immOpnd.GetVary() == kUnAdjustVary), "NIY, can not deal this case!");
389 vary = immOpnd.GetVary();
390 } else {
391 auto &immOpnd = static_cast<ImmOperand &>(insn.GetOperand(kInsnThirdOpnd));
392 auto &shiftOpnd = static_cast<BitShiftOperand &>(insn.GetOperand(kInsnFourthOpnd));
393 CHECK_FATAL(shiftOpnd.GetShiftAmount() == 12, "invalid shiftAmount"); // 12: invalid shiftAmount
394 val = (immOpnd.GetValue() << shiftOpnd.GetShiftAmount());
395 }
396
397 if (mOp == MOP_xsubrri12 || mOp == MOP_xsubrri24) {
398 val = -val;
399 }
400 val += offset.GetValue();
401 auto &newImm = static_cast<AArch64CGFunc &>(cgFunc).GetOrCreateOfstOpnd(val, k64BitSize);
402 newImm.SetVary(vary);
403 return &newImm;
404 }
405
OptimizeAddrBOI(Insn & insn,MemOperand & memOpnd,Insn & prevInsn)406 void EnhanceStrLdrAArch64::OptimizeAddrBOI(Insn &insn, MemOperand &memOpnd, Insn &prevInsn)
407 {
408 auto *oriBase = memOpnd.GetBaseRegister();
409 auto *oriOffset = memOpnd.GetOffsetOperand();
410 auto &defOpnd = static_cast<RegOperand &>(prevInsn.GetOperand(kInsnFirstOpnd));
411 if (defOpnd.GetRegisterNumber() != oriBase->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) {
412 return;
413 }
414 auto *newBase = static_cast<RegOperand *>(&prevInsn.GetOperand(kInsnSecondOpnd));
415 auto *newOffset = GetInsnAddOrSubNewOffset(prevInsn, *memOpnd.GetOffsetOperand());
416 if (newOffset->GetValue() < 0) {
417 return; // obj dump cannot deal str x19, [x29,#-16]
418 }
419
420 memOpnd.SetBaseRegister(*newBase);
421 memOpnd.SetOffsetOperand(*newOffset);
422 if (!static_cast<AArch64CGFunc &>(cgFunc).IsOperandImmValid(insn.GetMachineOpcode(), &memOpnd, kInsnSecondOpnd)) {
423 // If new offset is invalid, undo it
424 memOpnd.SetBaseRegister(*oriBase);
425 memOpnd.SetOffsetOperand(*oriOffset);
426 return;
427 }
428 memOpnd.SetAddrMode(MemOperand::kAddrModeBOi);
429 prevInsn.GetBB()->RemoveInsn(prevInsn);
430 }
431
OptimizeAddrBOrXShiftExtend(Insn & insn,MemOperand & memOpnd,Insn & shiftExtendInsn)432 void EnhanceStrLdrAArch64::OptimizeAddrBOrXShiftExtend(Insn &insn, MemOperand &memOpnd, Insn &shiftExtendInsn)
433 {
434 auto mOp = shiftExtendInsn.GetMachineOpcode();
435 if (mOp != MOP_xuxtw64 && mOp != MOP_xsxtw64 && mOp != MOP_xlslrri6) {
436 return;
437 }
438 auto *oriIndex = memOpnd.GetIndexRegister();
439 auto &defOpnd = static_cast<RegOperand &>(shiftExtendInsn.GetOperand(kInsnFirstOpnd));
440 if (defOpnd.GetRegisterNumber() != oriIndex->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) {
441 return;
442 }
443 auto &newIndex = static_cast<RegOperand &>(shiftExtendInsn.GetOperand(kInsnSecondOpnd));
444 bool isSigned = (mOp == MOP_xsxtw64);
445 uint32 shift = 0;
446 if (mOp == MOP_xlslrri6) {
447 shift = static_cast<uint32>(static_cast<ImmOperand &>(shiftExtendInsn.GetOperand(kInsnThirdOpnd)).GetValue());
448 }
449 const uint32 regSize = insn.GetDesc()->GetOpndDes(kInsnFirstOpnd)->GetSize();
450 // lsl extend insn shift amount can only be 0 or 1(16-bit def opnd) or 2(32-bit def opnd) or
451 // 3(64-bit def opnd) or 4(128-bit def opnd) in ldr/str insn, and in this pattern we only have
452 // 32-bit & 64-bit situation now
453 if ((shift == k0BitSize) || (regSize == k32BitSize && shift == k2BitSize) ||
454 (regSize == k64BitSize && shift == k3BitSize)) {
455 auto *newMemOpnd = static_cast<AArch64CGFunc &>(cgFunc).CreateMemOperand(
456 MemOperand::kAddrModeBOrX, memOpnd.GetSize(), *memOpnd.GetBaseRegister(), newIndex, shift, isSigned);
457 insn.SetOperand(kInsnSecondOpnd, *newMemOpnd);
458 shiftExtendInsn.GetBB()->RemoveInsn(shiftExtendInsn);
459 }
460 }
461
OptimizeAddrBOrX(Insn & insn,MemOperand & memOpnd,Insn & prevInsn)462 void EnhanceStrLdrAArch64::OptimizeAddrBOrX(Insn &insn, MemOperand &memOpnd, Insn &prevInsn)
463 {
464 if (memOpnd.GetOffsetOperand()->GetValue() != 0 || memOpnd.GetOffsetOperand()->GetVary() == kUnAdjustVary) {
465 return;
466 }
467 auto *oriBase = memOpnd.GetBaseRegister();
468 auto &defOpnd = static_cast<RegOperand &>(prevInsn.GetOperand(kInsnFirstOpnd));
469 if (defOpnd.GetRegisterNumber() != oriBase->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) {
470 return;
471 }
472 auto *newBase = static_cast<RegOperand *>(&prevInsn.GetOperand(kInsnSecondOpnd));
473 auto *newIndex = static_cast<RegOperand *>(&prevInsn.GetOperand(kInsnThirdOpnd));
474
475 memOpnd.SetBaseRegister(*newBase);
476 memOpnd.SetIndexRegister(*newIndex);
477 memOpnd.SetAddrMode(MemOperand::kAddrModeBOrX);
478 auto *prevShiftExtendInsn = prevInsn.GetPreviousMachineInsn();
479 if (prevShiftExtendInsn != nullptr) {
480 OptimizeAddrBOrXShiftExtend(insn, memOpnd, *prevShiftExtendInsn);
481 }
482 prevInsn.GetBB()->RemoveInsn(prevInsn);
483 }
484
OptimizeWithAddrrrs(Insn & insn,MemOperand & memOpnd,Insn & addInsn)485 void EnhanceStrLdrAArch64::OptimizeWithAddrrrs(Insn &insn, MemOperand &memOpnd, Insn &addInsn)
486 {
487 if (memOpnd.GetOffsetOperand()->GetValue() != 0 || memOpnd.GetOffsetOperand()->GetVary() != kNotVary) {
488 return;
489 }
490 auto *oriBase = memOpnd.GetBaseRegister();
491 auto &defOpnd = static_cast<RegOperand &>(addInsn.GetOperand(kInsnFirstOpnd));
492 if (defOpnd.GetRegisterNumber() != oriBase->GetRegisterNumber() || !CheckOperandIsDeadFromInsn(defOpnd, insn)) {
493 return;
494 }
495 auto &newBase = static_cast<RegOperand &>(addInsn.GetOperand(kInsnSecondOpnd));
496 auto &newIndex = static_cast<RegOperand &>(addInsn.GetOperand(kInsnThirdOpnd));
497 auto &shift = static_cast<BitShiftOperand &>(addInsn.GetOperand(kInsnFourthOpnd));
498 if (shift.GetShiftOp() != BitShiftOperand::kLSL) {
499 return;
500 }
501 auto *newMemOpnd = static_cast<AArch64CGFunc &>(cgFunc).CreateMemOperand(
502 MemOperand::kAddrModeBOrX, memOpnd.GetSize(), newBase, newIndex, shift.GetShiftAmount());
503 insn.SetOperand(kInsnSecondOpnd, *newMemOpnd);
504 addInsn.GetBB()->RemoveInsn(addInsn);
505 }
506
Run(BB & bb,Insn & insn)507 void EnhanceStrLdrAArch64::Run(BB &bb, Insn &insn)
508 {
509 Operand &opnd = insn.GetOperand(kInsnSecondOpnd);
510 CHECK_FATAL(opnd.IsMemoryAccessOperand(), "Unexpected operand in EnhanceStrLdrAArch64");
511 auto &memOpnd = static_cast<MemOperand &>(opnd);
512 if (memOpnd.GetAddrMode() != MemOperand::kAddrModeBOi || !memOpnd.GetOffsetImmediate()->IsImmOffset()) {
513 return;
514 }
515
516 auto *prev = insn.GetPreviousMachineInsn();
517 while (prev != nullptr) {
518 if (prev->GetMachineOpcode() == MOP_xmovrr) {
519 auto &defOpnd = static_cast<RegOperand &>(prev->GetOperand(kInsnFirstOpnd));
520 if (defOpnd.GetRegisterNumber() != memOpnd.GetBaseRegister()->GetRegisterNumber() ||
521 !CheckOperandIsDeadFromInsn(defOpnd, insn)) {
522 return;
523 }
524 memOpnd.SetBaseRegister(static_cast<RegOperand &>(prev->GetOperand(kInsnSecondOpnd)));
525 auto *tmpInsn = prev;
526 prev = prev->GetPreviousMachineInsn();
527 tmpInsn->GetBB()->RemoveInsn(*tmpInsn);
528 continue;
529 }
530 break;
531 }
532 if (prev == nullptr) {
533 return;
534 }
535 auto prevMop = prev->GetMachineOpcode();
536 if (prevMop == MOP_xaddrri12 || prevMop == MOP_xsubrri12 || prevMop == MOP_xaddrri24 || prevMop == MOP_xsubrri24) {
537 OptimizeAddrBOI(insn, memOpnd, *prev);
538 } else if (prevMop == MOP_xaddrrr) {
539 OptimizeAddrBOrX(insn, memOpnd, *prev);
540 } else if (prevMop == MOP_xaddrrrs) {
541 OptimizeWithAddrrrs(insn, memOpnd, *prev);
542 }
543 }
544
IsSameRegisterOperation(const RegOperand & desMovOpnd,const RegOperand & uxtDestOpnd,const RegOperand & uxtFromOpnd)545 bool IsSameRegisterOperation(const RegOperand &desMovOpnd, const RegOperand &uxtDestOpnd, const RegOperand &uxtFromOpnd)
546 {
547 return ((desMovOpnd.GetRegisterNumber() == uxtDestOpnd.GetRegisterNumber()) &&
548 (uxtDestOpnd.GetRegisterNumber() == uxtFromOpnd.GetRegisterNumber()));
549 }
550
IsRegNotSameMemUseInInsn(const Insn & checkInsn,const Insn & curInsn,regno_t curBaseRegNO,bool isCurStore,int64 curBaseOfst,int64 curMemRange) const551 bool CombineContiLoadAndStorePattern::IsRegNotSameMemUseInInsn(const Insn &checkInsn, const Insn &curInsn,
552 regno_t curBaseRegNO, bool isCurStore, int64 curBaseOfst,
553 int64 curMemRange) const
554 {
555 uint32 opndNum = checkInsn.GetOperandSize();
556 for (uint32 i = 0; i < opndNum; ++i) {
557 Operand &opnd = checkInsn.GetOperand(i);
558 if (opnd.IsList()) {
559 auto &listOpnd = static_cast<const ListOperand &>(opnd);
560 for (auto &listElem : listOpnd.GetOperands()) {
561 auto *regOpnd = static_cast<RegOperand *>(listElem);
562 DEBUG_ASSERT(regOpnd != nullptr, "parameter operand must be RegOperand");
563 if (curBaseRegNO == regOpnd->GetRegisterNumber()) {
564 return true;
565 }
566 }
567 } else if (opnd.IsMemoryAccessOperand()) {
568 auto &memOperand = static_cast<MemOperand &>(opnd);
569 RegOperand *checkBaseReg = memOperand.GetBaseRegister();
570 // Check memory overlap
571 if ((isCurStore || checkInsn.IsStore()) && checkBaseReg != nullptr &&
572 memOperand.GetAddrMode() == MemOperand::kAddrModeBOi && memOperand.GetOffsetImmediate() != nullptr) {
573 // If memInsn is split with x16, we need to find the actual base register
574 int64 checkOffset = memOperand.GetOffsetImmediate()->GetOffsetValue();
575 regno_t checkRegNO = checkBaseReg->GetRegisterNumber();
576 if (checkRegNO == R16) {
577 const Insn *prevInsn = checkInsn.GetPrev();
578 // Before cgaggressiveopt, the def and use of R16 must be adjacent, and the def of R16 must be
579 // addrri, otherwise, the process is conservative and the mem insn that can be combined is not
580 // search forward.
581 if (prevInsn == nullptr || prevInsn->GetMachineOpcode() != MOP_xaddrri12 ||
582 static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd)).GetRegisterNumber() != R16) {
583 return true;
584 }
585 checkOffset += static_cast<ImmOperand &>(prevInsn->GetOperand(kInsnThirdOpnd)).GetValue();
586 }
587 auto checkMemRange = static_cast<int64>(checkInsn.GetMemoryByteSize());
588 // curOfst curOfst+curMemRange
589 // |______|_/_/_/_/_/_/_/_/_/_/_|____________|
590 if ((curBaseOfst >= checkOffset && curBaseOfst < (checkOffset + checkMemRange)) ||
591 (checkOffset >= curBaseOfst && checkOffset < (curBaseOfst + curMemRange))) {
592 return true;
593 }
594 }
595 } else if (opnd.IsConditionCode()) {
596 auto &rflagOpnd = static_cast<RegOperand &>(cgFunc->GetOrCreateRflag());
597 if (rflagOpnd.GetRegisterNumber() == curBaseRegNO) {
598 return true;
599 }
600 } else if (opnd.IsRegister()) {
601 if (!isCurStore && static_cast<RegOperand &>(opnd).GetRegisterNumber() == curBaseRegNO) {
602 return true;
603 }
604 }
605 }
606 return false;
607 }
608
FindPrevStrLdr(Insn & insn,regno_t destRegNO,regno_t memBaseRegNO,int64 baseOfst) const609 std::vector<Insn *> CombineContiLoadAndStorePattern::FindPrevStrLdr(Insn &insn, regno_t destRegNO, regno_t memBaseRegNO,
610 int64 baseOfst) const
611 {
612 std::vector<Insn *> prevContiInsns;
613 for (Insn *curInsn = insn.GetPrev(); curInsn != nullptr; curInsn = curInsn->GetPrev()) {
614 if (!curInsn->IsMachineInstruction()) {
615 continue;
616 }
617 if (curInsn->IsRegDefined(memBaseRegNO)) {
618 return prevContiInsns;
619 }
620 DEBUG_ASSERT(insn.GetOperand(kInsnSecondOpnd).IsMemoryAccessOperand(), "invalid mem insn");
621 auto baseMemRange = static_cast<int64>(insn.GetMemoryByteSize());
622 if (IsRegNotSameMemUseInInsn(*curInsn, insn, memBaseRegNO, insn.IsStore(), static_cast<int32>(baseOfst),
623 baseMemRange)) {
624 return prevContiInsns;
625 }
626 // record continuous STD/LDR insn
627 if (!curInsn->IsLoadStorePair() &&
628 ((insn.IsStore() && curInsn->IsStore()) || (insn.IsLoad() && curInsn->IsLoad()))) {
629 auto *memOperand = static_cast<MemOperand *>(curInsn->GetMemOpnd());
630 /* do not combine ldr r0, label */
631 if (memOperand != nullptr) {
632 auto *baseRegOpnd = static_cast<RegOperand *>(memOperand->GetBaseRegister());
633 DEBUG_ASSERT(baseRegOpnd == nullptr || !baseRegOpnd->IsVirtualRegister(),
634 "physical register has not been allocated?");
635 if (memOperand->GetAddrMode() == MemOperand::kAddrModeBOi &&
636 baseRegOpnd->GetRegisterNumber() == memBaseRegNO) {
637 prevContiInsns.emplace_back(curInsn);
638 }
639 }
640 }
641 /* check insn that changes the data flow */
642 /* ldr x8, [x21, #8]
643 * call foo()
644 * ldr x9, [x21, #16]
645 * although x21 is a calleeSave register, there is no guarantee data in memory [x21] is not changed
646 */
647 if (curInsn->IsCall() || curInsn->GetMachineOpcode() == MOP_asm) {
648 return prevContiInsns;
649 }
650 /* Check regOpnd for mem access:
651 * 1. if the destRegNO is RZR, we do not need to check define and use for destRegNO between PREVINSN and INSN;
652 * 2. for load insn, we forbid both use and define destRegNO between PREVINSN and INSN;
653 * 3. for store insn, we only forbit define destRegNO between PREVINSN and INSN;
654 * e.g.1
655 * ldr x2, [sp, #16]
656 * add x3, x1, #5 & add x1, x3, #5 ---\--> all [x1] use and define can not across
657 * ldr x1, [sp, #8]
658 * e.g.2
659 * str x2, [sp, #16]
660 * add x1, x3, #5 ---\---> only [x1] define can not across
661 * str x1, [sp, #8]
662 */
663 /* store opt should not cross call due to stack args */
664 if (destRegNO != RZR &&
665 ((insn.IsLoad() && curInsn->ScanReg(destRegNO)) || (insn.IsStore() && curInsn->IsRegDefined(destRegNO)))) {
666 return prevContiInsns;
667 }
668 if (curInsn->ScanReg(destRegNO)) {
669 return prevContiInsns;
670 }
671 }
672 return prevContiInsns;
673 }
674
CheckCondition(Insn & insn)675 bool CombineContiLoadAndStorePattern::CheckCondition(Insn &insn)
676 {
677 MOperator mop = insn.GetMachineOpcode();
678 if (mop == MOP_wldrb || mop == MOP_wldrh) {
679 return false;
680 }
681 auto *curMemOpnd = static_cast<MemOperand *>(insn.GetMemOpnd());
682 DEBUG_ASSERT(curMemOpnd != nullptr, "get mem operand failed");
683 if (!doAggressiveCombine || curMemOpnd->GetAddrMode() != MemOperand::kAddrModeBOi) {
684 return false;
685 }
686 return true;
687 }
688
689 /* Combining 2 STRs into 1 stp or 2 LDRs into 1 ldp */
Run(BB & bb,Insn & insn)690 void CombineContiLoadAndStorePattern::Run(BB &bb, Insn &insn)
691 {
692 if (!CheckCondition(insn)) {
693 return;
694 }
695
696 auto *curMemOpnd = static_cast<MemOperand *>(insn.GetMemOpnd());
697 CHECK_FATAL(curMemOpnd != nullptr, "nullptr check");
698 DEBUG_ASSERT(curMemOpnd->GetAddrMode() == MemOperand::kAddrModeBOi, "invalid continues mem insn");
699 OfstOperand *curOfstOpnd = curMemOpnd->GetOffsetImmediate();
700 int64 curOfstVal = curOfstOpnd ? curOfstOpnd->GetOffsetValue() : 0;
701
702 auto *baseRegOpnd = static_cast<RegOperand *>(curMemOpnd->GetBaseRegister());
703 DEBUG_ASSERT(baseRegOpnd == nullptr || !baseRegOpnd->IsVirtualRegister(),
704 "physical register has not been allocated?");
705 auto &curDestOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
706 std::vector<Insn *> prevContiInsnVec =
707 FindPrevStrLdr(insn, curDestOpnd.GetRegisterNumber(), baseRegOpnd->GetRegisterNumber(), curOfstVal);
708 for (auto prevContiInsn : prevContiInsnVec) {
709 DEBUG_ASSERT(prevContiInsn != nullptr, "get previous consecutive instructions failed");
710 auto *prevMemOpnd = static_cast<MemOperand *>(prevContiInsn->GetMemOpnd());
711 DEBUG_ASSERT(prevMemOpnd->GetAddrMode() == MemOperand::kAddrModeBOi, "invalid continues mem insn");
712 OfstOperand *prevOfstOpnd = prevMemOpnd->GetOffsetImmediate();
713 int64 prevOfstVal = prevOfstOpnd ? prevOfstOpnd->GetOffsetValue() : 0;
714 auto &prevDestOpnd = static_cast<RegOperand &>(prevContiInsn->GetOperand(kInsnFirstOpnd));
715 if (prevDestOpnd.GetRegisterType() != curDestOpnd.GetRegisterType()) {
716 continue;
717 }
718
719 MemOperand *combineMemOpnd = (curOfstVal < prevOfstVal) ? curMemOpnd : prevMemOpnd;
720 if (IsValidNormalLoadOrStorePattern(insn, *prevContiInsn, *curMemOpnd, curOfstVal, prevOfstVal)) {
721 // Process normal mem pair
722 MOperator newMop = GetMopPair(insn.GetMachineOpcode(), true);
723 Insn *combineInsn =
724 GenerateMemPairInsn(newMop, curDestOpnd, prevDestOpnd, *combineMemOpnd, curOfstVal < prevOfstVal);
725 DEBUG_ASSERT(combineInsn != nullptr, "create combineInsn failed");
726 bb.InsertInsnAfter(*prevContiInsn, *combineInsn);
727 if (!(static_cast<AArch64CGFunc &>(*cgFunc).IsOperandImmValid(
728 newMop, combineMemOpnd, isPairAfterCombine ? kInsnThirdOpnd : kInsnSecondOpnd))) {
729 if (FindUseX16AfterInsn(*prevContiInsn)) {
730 // Do not combine Insns when x16 was used after curInsn
731 bb.RemoveInsn(*combineInsn);
732 return;
733 }
734 SPLIT_INSN(combineInsn, cgFunc);
735 }
736 RemoveInsnAndKeepComment(bb, insn, *prevContiInsn);
737 SetCurrInsn(combineInsn);
738 optSuccess = true;
739 return;
740 } else if (IsValidStackArgLoadOrStorePattern(insn, *prevContiInsn, *curMemOpnd, *prevMemOpnd, curOfstVal,
741 prevOfstVal)) {
742 // Process stack-arg mem pair
743 regno_t curDestRegNo = curDestOpnd.GetRegisterNumber();
744 regno_t prevDestRegNo = prevDestOpnd.GetRegisterNumber();
745 RegOperand &newDest = static_cast<AArch64CGFunc *>(cgFunc)->GetOrCreatePhysicalRegisterOperand(
746 static_cast<AArch64reg>(curDestRegNo), k64BitSize, curDestOpnd.GetRegisterType());
747 RegOperand &newPrevDest = static_cast<AArch64CGFunc *>(cgFunc)->GetOrCreatePhysicalRegisterOperand(
748 static_cast<AArch64reg>(prevDestRegNo), k64BitSize, prevDestOpnd.GetRegisterType());
749 MOperator newMop = (curDestOpnd.GetRegisterType() == kRegTyInt) ? MOP_xstp : MOP_dstp;
750 if (!(static_cast<AArch64CGFunc &>(*cgFunc).IsOperandImmValid(newMop, combineMemOpnd, kInsnThirdOpnd))) {
751 return;
752 }
753 Insn *combineInsn =
754 GenerateMemPairInsn(newMop, newDest, newPrevDest, *combineMemOpnd, curOfstVal < prevOfstVal);
755 bb.InsertInsnAfter(*prevContiInsn, *combineInsn);
756 RemoveInsnAndKeepComment(bb, insn, *prevContiInsn);
757 SetCurrInsn(combineInsn);
758 optSuccess = true;
759 return;
760 }
761 }
762 }
763
FindUseX16AfterInsn(const Insn & curInsn) const764 bool CombineContiLoadAndStorePattern::FindUseX16AfterInsn(const Insn &curInsn) const
765 {
766 for (Insn *cursor = curInsn.GetNext(); cursor != nullptr; cursor = cursor->GetNext()) {
767 if (!cursor->IsMachineInstruction()) {
768 continue;
769 }
770 for (uint32 defRegNo : cursor->GetDefRegs()) {
771 if (defRegNo == R16) {
772 return false;
773 }
774 }
775 if ((!cursor->IsLoad() && !cursor->IsStore() && !cursor->IsLoadStorePair()) || cursor->IsAtomic()) {
776 continue;
777 }
778 const InsnDesc *md = &AArch64CG::kMd[cursor->GetMachineOpcode()];
779 if (cursor->IsLoadLabel() || md->IsLoadAddress()) {
780 continue;
781 }
782 uint32 memIdx = (cursor->IsLoadStorePair() ? kInsnThirdOpnd : kInsnSecondOpnd);
783 auto &curMemOpnd = static_cast<MemOperand &>(cursor->GetOperand(memIdx));
784 RegOperand *baseOpnd = curMemOpnd.GetBaseRegister();
785 if (baseOpnd != nullptr && baseOpnd->GetRegisterNumber() == R16) {
786 return true;
787 }
788 }
789 return false;
790 }
791
GenerateMemPairInsn(MOperator newMop,RegOperand & curDestOpnd,RegOperand & prevDestOpnd,MemOperand & combineMemOpnd,bool isCurDestFirst)792 Insn *CombineContiLoadAndStorePattern::GenerateMemPairInsn(MOperator newMop, RegOperand &curDestOpnd,
793 RegOperand &prevDestOpnd, MemOperand &combineMemOpnd,
794 bool isCurDestFirst)
795 {
796 DEBUG_ASSERT(newMop != MOP_undef, "invalid MOperator");
797 Insn *combineInsn = nullptr;
798 if (isPairAfterCombine) { // for ldr/str --> ldp/stp
799 combineInsn = (isCurDestFirst)
800 ? &cgFunc->GetInsnBuilder()->BuildInsn(newMop, curDestOpnd, prevDestOpnd, combineMemOpnd)
801 : &cgFunc->GetInsnBuilder()->BuildInsn(newMop, prevDestOpnd, curDestOpnd, combineMemOpnd);
802 } else { // for strb/strh --> strh/str, curDestOpnd == preDestOpnd
803 combineInsn = &cgFunc->GetInsnBuilder()->BuildInsn(newMop, curDestOpnd, combineMemOpnd);
804 combineMemOpnd.SetSize(newMop == MOP_wstrh ? maplebe::k16BitSize : maplebe::k32BitSize);
805 }
806 return combineInsn;
807 }
808
IsValidNormalLoadOrStorePattern(const Insn & insn,const Insn & prevInsn,const MemOperand & memOpnd,int64 curOfstVal,int64 prevOfstVal)809 bool CombineContiLoadAndStorePattern::IsValidNormalLoadOrStorePattern(const Insn &insn, const Insn &prevInsn,
810 const MemOperand &memOpnd, int64 curOfstVal,
811 int64 prevOfstVal)
812 {
813 if (memOpnd.IsStackArgMem()) {
814 return false;
815 }
816 DEBUG_ASSERT(insn.GetOperand(kInsnFirstOpnd).IsRegister(), "unexpect operand");
817 DEBUG_ASSERT(prevInsn.GetOperand(kInsnFirstOpnd).IsRegister(), "unexpect operand");
818 auto &curDestOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
819 auto &prevDestOpnd = static_cast<RegOperand &>(prevInsn.GetOperand(kInsnFirstOpnd));
820 if (prevDestOpnd.GetRegisterType() != curDestOpnd.GetRegisterType() ||
821 curDestOpnd.GetSize() != prevDestOpnd.GetSize()) {
822 return false;
823 }
824 uint32 memSize = insn.GetMemoryByteSize();
825 uint32 prevMemSize = prevInsn.GetMemoryByteSize();
826 if (memSize != prevMemSize) {
827 return false;
828 }
829
830 int64 diffVal = std::abs(curOfstVal - prevOfstVal);
831 if ((memSize == k1ByteSize && diffVal == k1BitSize) || (memSize == k2ByteSize && diffVal == k2BitSize) ||
832 (memSize == k4ByteSize && diffVal == k4BitSize) || (memSize == k8ByteSize && diffVal == k8BitSize)) {
833 MOperator curMop = insn.GetMachineOpcode();
834 DEBUG_ASSERT(curMop != MOP_wldrb && curMop != MOP_wldrh, "invalid mem insn that cannot be combined");
835 if (curMop == MOP_wstrb || curMop == MOP_wstrh) {
836 isPairAfterCombine = false;
837 }
838
839 regno_t destRegNO = curDestOpnd.GetRegisterNumber();
840 regno_t prevDestRegNO = prevDestOpnd.GetRegisterNumber();
841 if (destRegNO == RZR && prevDestRegNO == RZR) {
842 return true;
843 }
844
845 if (insn.IsLoad() && destRegNO == prevDestRegNO) {
846 return false;
847 }
848
849 if ((curMop == MOP_wstrb || curMop == MOP_wstrh) && (destRegNO != RZR || prevDestRegNO != RZR)) {
850 return false;
851 }
852
853 return true;
854 }
855
856 return false;
857 }
858
IsValidStackArgLoadOrStorePattern(const Insn & curInsn,const Insn & prevInsn,const MemOperand & curMemOpnd,const MemOperand & prevMemOpnd,int64 curOfstVal,int64 prevOfstVal) const859 bool CombineContiLoadAndStorePattern::IsValidStackArgLoadOrStorePattern(const Insn &curInsn, const Insn &prevInsn,
860 const MemOperand &curMemOpnd,
861 const MemOperand &prevMemOpnd, int64 curOfstVal,
862 int64 prevOfstVal) const
863 {
864 if (!curInsn.IsStore()) {
865 return false;
866 }
867 if (!curMemOpnd.IsStackArgMem() || !prevMemOpnd.IsStackArgMem()) {
868 return false;
869 }
870 auto &curDestOpnd = static_cast<RegOperand &>(curInsn.GetOperand(kInsnFirstOpnd));
871 auto &prevDestOpnd = static_cast<RegOperand &>(prevInsn.GetOperand(kInsnFirstOpnd));
872 uint32 memSize = curInsn.GetMemoryByteSize();
873 uint32 prevMemSize = prevInsn.GetMemoryByteSize();
874 auto diffVal = std::abs(curOfstVal - prevOfstVal);
875 if ((memSize == k4ByteSize || memSize == k8ByteSize) && (prevMemSize == k4ByteSize || prevMemSize == k8ByteSize) &&
876 (diffVal == k8BitSize) && (curDestOpnd.GetValidBitsNum() == memSize * k8BitSize) &&
877 (prevDestOpnd.GetValidBitsNum() == prevMemSize * k8BitSize)) {
878 return true;
879 }
880 return false;
881 }
882
RemoveInsnAndKeepComment(BB & bb,Insn & insn,Insn & prevInsn) const883 void CombineContiLoadAndStorePattern::RemoveInsnAndKeepComment(BB &bb, Insn &insn, Insn &prevInsn) const
884 {
885 /* keep the comment */
886 Insn *nn = prevInsn.GetNextMachineInsn();
887 std::string newComment = "";
888 MapleString comment = insn.GetComment();
889 if (comment.c_str() != nullptr && strlen(comment.c_str()) > 0) {
890 newComment += comment.c_str();
891 }
892 comment = prevInsn.GetComment();
893 if (comment.c_str() != nullptr && strlen(comment.c_str()) > 0) {
894 newComment = newComment + " " + comment.c_str();
895 }
896 if (newComment.c_str() != nullptr && strlen(newComment.c_str()) > 0) {
897 DEBUG_ASSERT(nn != nullptr, "nn should not be nullptr");
898 nn->SetComment(newComment);
899 }
900 bb.RemoveInsn(insn);
901 bb.RemoveInsn(prevInsn);
902 }
903
Run(BB & bb,Insn & insn)904 void EliminateSpecifcSXTAArch64::Run(BB &bb, Insn &insn)
905 {
906 MOperator thisMop = insn.GetMachineOpcode();
907 Insn *prevInsn = insn.GetPrev();
908 while (prevInsn != nullptr && !prevInsn->GetMachineOpcode()) {
909 prevInsn = prevInsn->GetPrev();
910 }
911 if (prevInsn == nullptr) {
912 return;
913 }
914 auto ®Opnd0 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
915 auto ®Opnd1 = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
916 if (&insn != bb.GetFirstInsn() && regOpnd0.GetRegisterNumber() == regOpnd1.GetRegisterNumber() &&
917 prevInsn->IsMachineInstruction()) {
918 if (prevInsn->GetMachineOpcode() == MOP_wmovri32 || prevInsn->GetMachineOpcode() == MOP_xmovri64) {
919 auto &dstMovOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
920 if (dstMovOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) {
921 return;
922 }
923 Operand &opnd = prevInsn->GetOperand(kInsnSecondOpnd);
924 if (opnd.IsIntImmediate()) {
925 auto &immOpnd = static_cast<ImmOperand &>(opnd);
926 int64 value = immOpnd.GetValue();
927 if (thisMop == MOP_xsxtb32) {
928 /* value should in range between -127 and 127 */
929 if (value >= static_cast<int64>(0xFFFFFFFFFFFFFF80) && value <= 0x7F &&
930 immOpnd.IsSingleInstructionMovable(regOpnd0.GetSize())) {
931 bb.RemoveInsn(insn);
932 }
933 } else if (thisMop == MOP_xsxth32) {
934 /* value should in range between -32678 and 32678 */
935 if (value >= static_cast<int64>(0xFFFFFFFFFFFF8000) && value <= 0x7FFF &&
936 immOpnd.IsSingleInstructionMovable(regOpnd0.GetSize())) {
937 bb.RemoveInsn(insn);
938 }
939 } else {
940 uint64 flag = 0xFFFFFFFFFFFFFF80; /* initialize the flag with fifty-nine 1s at top */
941 if (thisMop == MOP_xsxth64) {
942 flag = 0xFFFFFFFFFFFF8000; /* specify the flag with forty-nine 1s at top in this case */
943 } else if (thisMop == MOP_xsxtw64) {
944 flag = 0xFFFFFFFF80000000; /* specify the flag with thirty-three 1s at top in this case */
945 }
946 if (!(static_cast<uint64>(value) & flag) &&
947 immOpnd.IsSingleInstructionMovable(regOpnd0.GetSize())) {
948 auto *aarch64CGFunc = static_cast<AArch64CGFunc *>(&cgFunc);
949 RegOperand &dstOpnd = aarch64CGFunc->GetOrCreatePhysicalRegisterOperand(
950 static_cast<AArch64reg>(dstMovOpnd.GetRegisterNumber()), k64BitSize,
951 dstMovOpnd.GetRegisterType());
952 prevInsn->SetOperand(kInsnFirstOpnd, dstOpnd);
953 prevInsn->SetMOP(AArch64CG::kMd[MOP_xmovri64]);
954 bb.RemoveInsn(insn);
955 }
956 }
957 }
958 } else if (prevInsn->GetMachineOpcode() == MOP_wldrsb) {
959 auto &dstMovOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
960 if (dstMovOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) {
961 return;
962 }
963 if (thisMop == MOP_xsxtb32) {
964 bb.RemoveInsn(insn);
965 }
966 } else if (prevInsn->GetMachineOpcode() == MOP_wldrsh) {
967 auto &dstMovOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
968 if (dstMovOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) {
969 return;
970 }
971 if (thisMop == MOP_xsxth32) {
972 bb.RemoveInsn(insn);
973 }
974 }
975 }
976 }
977
Run(BB & bb,Insn & insn)978 void EliminateSpecifcUXTAArch64::Run(BB &bb, Insn &insn)
979 {
980 MOperator thisMop = insn.GetMachineOpcode();
981 Insn *prevInsn = insn.GetPreviousMachineInsn();
982 if (prevInsn == nullptr) {
983 return;
984 }
985 auto ®Opnd0 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
986 auto ®Opnd1 = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
987 if (prevInsn->IsCall() && prevInsn->GetIsCallReturnUnsigned() &&
988 regOpnd0.GetRegisterNumber() == regOpnd1.GetRegisterNumber() &&
989 (regOpnd1.GetRegisterNumber() == R0 || regOpnd1.GetRegisterNumber() == V0)) {
990 uint32 retSize = prevInsn->GetRetSize();
991 if (retSize > 0 &&
992 ((thisMop == MOP_xuxtb32 && retSize <= k1ByteSize) || (thisMop == MOP_xuxth32 && retSize <= k2ByteSize) ||
993 (thisMop == MOP_xuxtw64 && retSize <= k4ByteSize))) {
994 bb.RemoveInsn(insn);
995 }
996 return;
997 }
998 if (&insn == bb.GetFirstInsn() || regOpnd0.GetRegisterNumber() != regOpnd1.GetRegisterNumber() ||
999 !prevInsn->IsMachineInstruction()) {
1000 return;
1001 }
1002 if (cgFunc.GetMirModule().GetSrcLang() == kSrcLangC && prevInsn->IsCall() && prevInsn->GetIsCallReturnSigned()) {
1003 return;
1004 }
1005 if (thisMop == MOP_xuxtb32) {
1006 if (prevInsn->GetMachineOpcode() == MOP_wmovri32 || prevInsn->GetMachineOpcode() == MOP_xmovri64) {
1007 auto &dstMovOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1008 if (!IsSameRegisterOperation(dstMovOpnd, regOpnd1, regOpnd0)) {
1009 return;
1010 }
1011 Operand &opnd = prevInsn->GetOperand(kInsnSecondOpnd);
1012 if (opnd.IsIntImmediate()) {
1013 auto &immOpnd = static_cast<ImmOperand &>(opnd);
1014 int64 value = immOpnd.GetValue();
1015 /* check the top 56 bits of value */
1016 if (!(static_cast<uint64>(value) & 0xFFFFFFFFFFFFFF00)) {
1017 bb.RemoveInsn(insn);
1018 }
1019 }
1020 } else if (prevInsn->GetMachineOpcode() == MOP_wldrb) {
1021 auto &dstOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1022 if (dstOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) {
1023 return;
1024 }
1025 bb.RemoveInsn(insn);
1026 }
1027 } else if (thisMop == MOP_xuxth32) {
1028 if (prevInsn->GetMachineOpcode() == MOP_wmovri32 || prevInsn->GetMachineOpcode() == MOP_xmovri64) {
1029 auto &dstMovOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1030 if (!IsSameRegisterOperation(dstMovOpnd, regOpnd1, regOpnd0)) {
1031 return;
1032 }
1033 Operand &opnd = prevInsn->GetOperand(kInsnSecondOpnd);
1034 if (opnd.IsIntImmediate()) {
1035 auto &immOpnd = static_cast<ImmOperand &>(opnd);
1036 int64 value = immOpnd.GetValue();
1037 if (!(static_cast<uint64>(value) & 0xFFFFFFFFFFFF0000)) {
1038 bb.RemoveInsn(insn);
1039 }
1040 }
1041 } else if (prevInsn->GetMachineOpcode() == MOP_wldrh) {
1042 auto &dstOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1043 if (dstOpnd.GetRegisterNumber() != regOpnd1.GetRegisterNumber()) {
1044 return;
1045 }
1046 bb.RemoveInsn(insn);
1047 }
1048 } else {
1049 /* this_mop == MOP_xuxtw64 */
1050 if (prevInsn->GetMachineOpcode() == MOP_wmovri32 || prevInsn->GetMachineOpcode() == MOP_wldrsb ||
1051 prevInsn->GetMachineOpcode() == MOP_wldrb || prevInsn->GetMachineOpcode() == MOP_wldrsh ||
1052 prevInsn->GetMachineOpcode() == MOP_wldrh || prevInsn->GetMachineOpcode() == MOP_wldr) {
1053 auto &dstOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1054 if (!IsSameRegisterOperation(dstOpnd, regOpnd1, regOpnd0)) {
1055 return;
1056 }
1057 /* 32-bit ldr does zero-extension by default, so this conversion can be skipped */
1058 bb.RemoveInsn(insn);
1059 }
1060 }
1061 }
1062
CheckCondition(Insn & insn)1063 bool FmovRegPattern::CheckCondition(Insn &insn)
1064 {
1065 nextInsn = insn.GetNextMachineInsn();
1066 if (nextInsn == nullptr) {
1067 return false;
1068 }
1069 prevInsn = insn.GetPreviousMachineInsn();
1070 if (prevInsn == nullptr) {
1071 return false;
1072 }
1073 auto &curSrcOpnd = insn.GetOperand(kInsnSecondOpnd);
1074 auto &prevSrcOpnd = prevInsn->GetOperand(kInsnSecondOpnd);
1075 if (!curSrcOpnd.IsRegister() || !prevSrcOpnd.IsRegister()) {
1076 return false;
1077 }
1078 auto &curSrcRegOpnd = static_cast<RegOperand&>(curSrcOpnd);
1079 auto &prevSrcRegOpnd = static_cast<RegOperand&>(prevSrcOpnd);
1080 /* same src freg */
1081 if (curSrcRegOpnd.GetRegisterNumber() != prevSrcRegOpnd.GetRegisterNumber()) {
1082 return false;
1083 }
1084 return true;
1085 }
1086
Run(BB & bb,Insn & insn)1087 void FmovRegPattern::Run(BB &bb, Insn &insn)
1088 {
1089 if (!CheckCondition(insn)) {
1090 return;
1091 }
1092 MOperator thisMop = insn.GetMachineOpcode();
1093 MOperator prevMop = prevInsn->GetMachineOpcode();
1094 MOperator newMop;
1095 uint32 doOpt = 0;
1096 if (prevMop == MOP_xvmovrv && thisMop == MOP_xvmovrv) {
1097 doOpt = k32BitSize;
1098 newMop = MOP_wmovrr;
1099 } else if (prevMop == MOP_xvmovrd && thisMop == MOP_xvmovrd) {
1100 doOpt = k64BitSize;
1101 newMop = MOP_xmovrr;
1102 }
1103 if (doOpt == 0) {
1104 return;
1105 }
1106 auto &curDstRegOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1107 regno_t curDstReg = curDstRegOpnd.GetRegisterNumber();
1108 /* optimize case 1 */
1109 auto &prevDstRegOpnd = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1110 regno_t prevDstReg = prevDstRegOpnd.GetRegisterNumber();
1111 auto *aarch64CGFunc = static_cast<AArch64CGFunc *>(cgFunc);
1112 RegOperand &dst =
1113 aarch64CGFunc->GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(curDstReg), doOpt, kRegTyInt);
1114 RegOperand &src =
1115 aarch64CGFunc->GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(prevDstReg), doOpt, kRegTyInt);
1116 Insn &newInsn = cgFunc->GetInsnBuilder()->BuildInsn(newMop, dst, src);
1117 bb.InsertInsnBefore(insn, newInsn);
1118 bb.RemoveInsn(insn);
1119 RegOperand &newOpnd =
1120 aarch64CGFunc->GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(prevDstReg), doOpt, kRegTyInt);
1121 uint32 opndNum = nextInsn->GetOperandSize();
1122 for (uint32 opndIdx = 0; opndIdx < opndNum; ++opndIdx) {
1123 Operand &opnd = nextInsn->GetOperand(opndIdx);
1124 if (opnd.IsMemoryAccessOperand()) {
1125 auto &memOpnd = static_cast<MemOperand &>(opnd);
1126 Operand *base = memOpnd.GetBaseRegister();
1127 if (base != nullptr) {
1128 if (base->IsRegister()) {
1129 auto *reg = static_cast<RegOperand *>(base);
1130 if (reg->GetRegisterNumber() == curDstReg) {
1131 memOpnd.SetBaseRegister(newOpnd);
1132 }
1133 }
1134 }
1135 Operand *offset = memOpnd.GetIndexRegister();
1136 if (offset != nullptr) {
1137 if (offset->IsRegister()) {
1138 auto *reg = static_cast<RegOperand *>(offset);
1139 if (reg->GetRegisterNumber() == curDstReg) {
1140 memOpnd.SetIndexRegister(newOpnd);
1141 }
1142 }
1143 }
1144 } else if (opnd.IsRegister()) {
1145 /* Check if it is a source operand. */
1146 auto *regProp = nextInsn->GetDesc()->opndMD[opndIdx];
1147 if (regProp->IsUse()) {
1148 auto ® = static_cast<RegOperand &>(opnd);
1149 if (reg.GetRegisterNumber() == curDstReg) {
1150 nextInsn->SetOperand(opndIdx, newOpnd);
1151 }
1152 }
1153 }
1154 }
1155 }
1156
CheckCondition(Insn & insn)1157 bool SbfxOptPattern::CheckCondition(Insn &insn)
1158 {
1159 nextInsn = insn.GetNextMachineInsn();
1160 if (nextInsn == nullptr) {
1161 return false;
1162 }
1163 auto &curDstRegOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1164 auto &lsb = static_cast<ImmOperand &>(insn.GetOperand(kInsnThirdOpnd));
1165 auto &width = static_cast<ImmOperand &>(insn.GetOperand(kInsnFourthOpnd));
1166 if (lsb.GetValue() != 0 || width.GetValue() < k32BitSize) {
1167 return false;
1168 }
1169 uint32 opndNum = nextInsn->GetOperandSize();
1170 const InsnDesc *md = nextInsn->GetDesc();
1171 for (uint32 opndIdx = 0; opndIdx < opndNum; ++opndIdx) {
1172 Operand &opnd = nextInsn->GetOperand(opndIdx);
1173 /* Check if it is a source operand. */
1174 if (opnd.IsMemoryAccessOperand() || opnd.IsList()) {
1175 return false;
1176 } else if (opnd.IsRegister()) {
1177 auto ® = static_cast<RegOperand &>(opnd);
1178 auto *regProp = md->opndMD[opndIdx];
1179 if (reg.GetRegisterNumber() == curDstRegOpnd.GetRegisterNumber()) {
1180 if (nextInsn->GetOperandSize(opndIdx) != k32BitSize) {
1181 return false;
1182 }
1183 if (regProp->IsDef()) {
1184 toRemove = true;
1185 } else {
1186 (void)cands.emplace_back(opndIdx);
1187 }
1188 }
1189 }
1190 }
1191 return cands.size() != 0;
1192 }
1193
Run(BB & bb,Insn & insn)1194 void SbfxOptPattern::Run(BB &bb, Insn &insn)
1195 {
1196 if (!CheckCondition(insn)) {
1197 return;
1198 }
1199 auto &srcRegOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
1200 RegOperand &newReg = static_cast<AArch64CGFunc *>(cgFunc)->GetOrCreatePhysicalRegisterOperand(
1201 static_cast<AArch64reg>(srcRegOpnd.GetRegisterNumber()), k32BitSize, srcRegOpnd.GetRegisterType());
1202 // replace use point of opnd in nextInsn
1203 for (auto i : cands) {
1204 nextInsn->SetOperand(i, newReg);
1205 }
1206 if (toRemove) {
1207 bb.RemoveInsn(insn);
1208 }
1209 }
1210
CheckCondition(Insn & insn)1211 bool CbnzToCbzPattern::CheckCondition(Insn &insn)
1212 {
1213 MOperator curMop = insn.GetMachineOpcode();
1214 if (curMop != MOP_wcbnz && curMop != MOP_xcbnz) {
1215 return false;
1216 }
1217 /* reg has to be R0, since return value is in R0 */
1218 auto ®Opnd0 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1219 if (regOpnd0.GetRegisterNumber() != R0) {
1220 return false;
1221 }
1222 nextBB = insn.GetBB()->GetNext();
1223 /* Make sure nextBB can only be reached by bb */
1224 if (nextBB->GetPreds().size() > 1) {
1225 return false;
1226 }
1227 /* Next insn should be a mov R0 = 0 */
1228 movInsn = nextBB->GetFirstMachineInsn();
1229 if (movInsn == nullptr) {
1230 return false;
1231 }
1232 MOperator movInsnMop = movInsn->GetMachineOpcode();
1233 if (movInsnMop != MOP_wmovri32 && movInsnMop != MOP_xmovri64) {
1234 return false;
1235 }
1236 auto &movDest = static_cast<RegOperand &>(movInsn->GetOperand(kInsnFirstOpnd));
1237 if (movDest.GetRegisterNumber() != R0) {
1238 return false;
1239 }
1240 auto &movImm = static_cast<ImmOperand &>(movInsn->GetOperand(kInsnSecondOpnd));
1241 if (movImm.GetValue() != 0) {
1242 return false;
1243 }
1244 Insn *nextBrInsn = movInsn->GetNextMachineInsn();
1245 if (nextBrInsn == nullptr) {
1246 return false;
1247 }
1248 if (nextBrInsn->GetMachineOpcode() != MOP_xuncond) {
1249 return false;
1250 }
1251 /* Is nextBB branch to the return-bb? */
1252 if (nextBB->GetSuccs().size() != 1) {
1253 return false;
1254 }
1255 return true;
1256 }
1257
Run(BB & bb,Insn & insn)1258 void CbnzToCbzPattern::Run(BB &bb, Insn &insn)
1259 {
1260 if (!CheckCondition(insn)) {
1261 return;
1262 }
1263 MOperator thisMop = insn.GetMachineOpcode();
1264 BB *targetBB = nullptr;
1265 auto it = bb.GetSuccsBegin();
1266 if (*it == nextBB) {
1267 ++it;
1268 }
1269 targetBB = *it;
1270 /* Make sure when nextBB is empty, targetBB is fallthru of bb. */
1271 if (targetBB != nextBB->GetNext()) {
1272 return;
1273 }
1274 BB *nextBBTarget = *(nextBB->GetSuccsBegin());
1275 if (nextBBTarget->GetKind() != BB::kBBReturn) {
1276 return;
1277 }
1278 /* Control flow looks nice, instruction looks nice */
1279 DEBUG_ASSERT(brInsn != nullptr, "brInsn should not be nullptr");
1280 Operand &brTarget = brInsn->GetOperand(kInsnFirstOpnd);
1281 insn.SetOperand(kInsnSecondOpnd, brTarget);
1282 if (thisMop == MOP_wcbnz) {
1283 insn.SetMOP(AArch64CG::kMd[MOP_wcbz]);
1284 } else {
1285 insn.SetMOP(AArch64CG::kMd[MOP_xcbz]);
1286 }
1287 nextBB->RemoveInsn(*movInsn);
1288 nextBB->RemoveInsn(*brInsn);
1289 /* nextBB is now a fallthru bb, not a goto bb */
1290 nextBB->SetKind(BB::kBBFallthru);
1291 /*
1292 * fix control flow, we have bb, nextBB, targetBB, nextBB_target
1293 * connect bb -> nextBB_target erase targetBB
1294 */
1295 it = bb.GetSuccsBegin();
1296 CHECK_FATAL(it != bb.GetSuccsEnd(), "succs is empty.");
1297 if (*it == targetBB) {
1298 bb.EraseSuccs(it);
1299 bb.PushFrontSuccs(*nextBBTarget);
1300 } else {
1301 ++it;
1302 bb.EraseSuccs(it);
1303 bb.PushBackSuccs(*nextBBTarget);
1304 }
1305 for (auto targetBBIt = targetBB->GetPredsBegin(); targetBBIt != targetBB->GetPredsEnd(); ++targetBBIt) {
1306 if (*targetBBIt == &bb) {
1307 targetBB->ErasePreds(targetBBIt);
1308 break;
1309 }
1310 }
1311 for (auto nextIt = nextBBTarget->GetPredsBegin(); nextIt != nextBBTarget->GetPredsEnd(); ++nextIt) {
1312 if (*nextIt == nextBB) {
1313 nextBBTarget->ErasePreds(nextIt);
1314 break;
1315 }
1316 }
1317 nextBBTarget->PushBackPreds(bb);
1318
1319 /* nextBB has no target, originally just branch target */
1320 nextBB->EraseSuccs(nextBB->GetSuccsBegin());
1321 DEBUG_ASSERT(nextBB->GetSuccs().empty(), "peep: branch target incorrect");
1322 /* Now make nextBB fallthru to targetBB */
1323 nextBB->PushFrontSuccs(*targetBB);
1324 targetBB->PushBackPreds(*nextBB);
1325 }
1326
HasImplicitSizeUse(const Insn & insn) const1327 bool ContiLDRorSTRToSameMEMPattern::HasImplicitSizeUse(const Insn &insn) const
1328 {
1329 if (insn.GetOperandSize(kInsnFirstOpnd) != prevInsn->GetOperandSize(kInsnFirstOpnd)) {
1330 return true;
1331 }
1332 if (loadAfterStore) {
1333 // To avoid the optimization as following:
1334 // str w10, [sp, #8]
1335 // ldr w10, [sp, #8] ---\--> can not be removed
1336 // ...
1337 // str x10, [x1, #16]
1338 auto &defOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1339 for (Insn *cursor = insn.GetNext(); cursor != nullptr; cursor = cursor->GetNext()) {
1340 if (!cursor->IsMachineInstruction()) {
1341 continue;
1342 }
1343 uint32 opndNum = cursor->GetOperandSize();
1344 for (uint32 i = 0; i < opndNum; ++i) {
1345 if (cursor->OpndIsDef(i)) {
1346 continue;
1347 }
1348 if (!cursor->GetOperand(i).IsRegister()) {
1349 continue;
1350 }
1351 auto &useOpnd = static_cast<RegOperand &>(cursor->GetOperand(i));
1352 if (useOpnd.GetRegisterNumber() == defOpnd.GetRegisterNumber() &&
1353 insn.GetOperandSize(kInsnFirstOpnd) != cursor->GetOperandSize(i)) {
1354 return true;
1355 }
1356 }
1357 }
1358 }
1359 return false;
1360 }
1361
Run(BB & bb,Insn & insn)1362 void CsetCbzToBeqOptAArch64::Run(BB &bb, Insn &insn)
1363 {
1364 Insn *insn1 = insn.GetPreviousMachineInsn();
1365 if (insn1 == nullptr) {
1366 return;
1367 }
1368 /* prevInsn must be "cset" insn */
1369 MOperator opCode1 = insn1->GetMachineOpcode();
1370 if (opCode1 != MOP_xcsetrc && opCode1 != MOP_wcsetrc) {
1371 return;
1372 }
1373
1374 auto &tmpRegOp1 = static_cast<RegOperand &>(insn1->GetOperand(kInsnFirstOpnd));
1375 regno_t baseRegNO1 = tmpRegOp1.GetRegisterNumber();
1376 auto &tmpRegOp2 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1377 regno_t baseRegNO2 = tmpRegOp2.GetRegisterNumber();
1378 if (baseRegNO1 != baseRegNO2) {
1379 return;
1380 }
1381 /* If the reg will be used later, we shouldn't optimize the cset insn here */
1382 if (IfOperandIsLiveAfterInsn(tmpRegOp2, insn)) {
1383 return;
1384 }
1385 MOperator opCode = insn.GetMachineOpcode();
1386 bool reverse = (opCode == MOP_xcbz || opCode == MOP_wcbz);
1387 Operand &rflag = static_cast<AArch64CGFunc *>(&cgFunc)->GetOrCreateRflag();
1388 auto &label = static_cast<LabelOperand &>(insn.GetOperand(kInsnSecondOpnd));
1389 auto &cond = static_cast<CondOperand &>(insn1->GetOperand(kInsnSecondOpnd));
1390 MOperator jmpOperator = SelectMOperator(cond.GetCode(), reverse);
1391 CHECK_FATAL((jmpOperator != MOP_undef), "unknown condition code");
1392 Insn &newInsn = cgFunc.GetInsnBuilder()->BuildInsn(jmpOperator, rflag, label);
1393 bb.RemoveInsn(*insn1);
1394 bb.ReplaceInsn(insn, newInsn);
1395 }
1396
SelectMOperator(ConditionCode condCode,bool inverse) const1397 MOperator CsetCbzToBeqOptAArch64::SelectMOperator(ConditionCode condCode, bool inverse) const
1398 {
1399 switch (condCode) {
1400 case CC_NE:
1401 return inverse ? MOP_beq : MOP_bne;
1402 case CC_EQ:
1403 return inverse ? MOP_bne : MOP_beq;
1404 case CC_MI:
1405 return inverse ? MOP_bpl : MOP_bmi;
1406 case CC_PL:
1407 return inverse ? MOP_bmi : MOP_bpl;
1408 case CC_VS:
1409 return inverse ? MOP_bvc : MOP_bvs;
1410 case CC_VC:
1411 return inverse ? MOP_bvs : MOP_bvc;
1412 case CC_HI:
1413 return inverse ? MOP_bls : MOP_bhi;
1414 case CC_LS:
1415 return inverse ? MOP_bhi : MOP_bls;
1416 case CC_GE:
1417 return inverse ? MOP_blt : MOP_bge;
1418 case CC_LT:
1419 return inverse ? MOP_bge : MOP_blt;
1420 case CC_HS:
1421 return inverse ? MOP_blo : MOP_bhs;
1422 case CC_LO:
1423 return inverse ? MOP_bhs : MOP_blo;
1424 case CC_LE:
1425 return inverse ? MOP_bgt : MOP_ble;
1426 case CC_GT:
1427 return inverse ? MOP_ble : MOP_bgt;
1428 case CC_CS:
1429 return inverse ? MOP_bcc : MOP_bcs;
1430 default:
1431 return MOP_undef;
1432 }
1433 }
1434
CheckCondition(Insn & insn)1435 bool ContiLDRorSTRToSameMEMPattern::CheckCondition(Insn &insn)
1436 {
1437 prevInsn = insn.GetPrev();
1438 while (prevInsn != nullptr && (prevInsn->GetMachineOpcode() == 0 || !prevInsn->IsMachineInstruction()) &&
1439 prevInsn != insn.GetBB()->GetFirstMachineInsn()) {
1440 prevInsn = prevInsn->GetPrev();
1441 }
1442 if (!insn.IsMachineInstruction() || prevInsn == nullptr) {
1443 return false;
1444 }
1445 MOperator thisMop = insn.GetMachineOpcode();
1446 MOperator prevMop = prevInsn->GetMachineOpcode();
1447 /*
1448 * store regB, RegC, offset
1449 * load regA, RegC, offset
1450 */
1451 if ((thisMop == MOP_xldr && prevMop == MOP_xstr) || (thisMop == MOP_wldr && prevMop == MOP_wstr) ||
1452 (thisMop == MOP_dldr && prevMop == MOP_dstr) || (thisMop == MOP_sldr && prevMop == MOP_sstr)) {
1453 loadAfterStore = true;
1454 }
1455 /*
1456 * load regA, RegC, offset
1457 * load regB, RegC, offset
1458 */
1459 if ((thisMop == MOP_xldr || thisMop == MOP_wldr || thisMop == MOP_dldr || thisMop == MOP_sldr) &&
1460 prevMop == thisMop) {
1461 loadAfterLoad = true;
1462 }
1463 if (!loadAfterStore && !loadAfterLoad) {
1464 return false;
1465 }
1466 if (HasImplicitSizeUse(insn)) {
1467 return false;
1468 }
1469 DEBUG_ASSERT(insn.GetOperand(kInsnSecondOpnd).IsMemoryAccessOperand(), "expects mem operands");
1470 DEBUG_ASSERT(prevInsn->GetOperand(kInsnSecondOpnd).IsMemoryAccessOperand(), "expects mem operands");
1471 return true;
1472 }
1473
Run(BB & bb,Insn & insn)1474 void ContiLDRorSTRToSameMEMPattern::Run(BB &bb, Insn &insn)
1475 {
1476 if (!CheckCondition(insn)) {
1477 return;
1478 }
1479 MOperator thisMop = insn.GetMachineOpcode();
1480 auto &memOpnd1 = static_cast<MemOperand &>(insn.GetOperand(kInsnSecondOpnd));
1481 MemOperand::AArch64AddressingMode addrMode1 = memOpnd1.GetAddrMode();
1482 if (addrMode1 != MemOperand::kAddrModeBOi || (!memOpnd1.IsIntactIndexed())) {
1483 return;
1484 }
1485
1486 auto *base1 = static_cast<RegOperand *>(memOpnd1.GetBaseRegister());
1487 DEBUG_ASSERT(base1 == nullptr || !base1->IsVirtualRegister(), "physical register has not been allocated?");
1488 OfstOperand *offset1 = memOpnd1.GetOffsetImmediate();
1489
1490 auto &memOpnd2 = static_cast<MemOperand &>(prevInsn->GetOperand(kInsnSecondOpnd));
1491 MemOperand::AArch64AddressingMode addrMode2 = memOpnd2.GetAddrMode();
1492 if (addrMode2 != MemOperand::kAddrModeBOi || (!memOpnd2.IsIntactIndexed())) {
1493 return;
1494 }
1495
1496 auto *base2 = static_cast<RegOperand *>(memOpnd2.GetBaseRegister());
1497 DEBUG_ASSERT(base2 == nullptr || !base2->IsVirtualRegister(), "physical register has not been allocated?");
1498 OfstOperand *offset2 = memOpnd2.GetOffsetImmediate();
1499
1500 if (base1 == nullptr || base2 == nullptr || offset1 == nullptr || offset2 == nullptr) {
1501 return;
1502 }
1503
1504 auto ®1 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1505 auto ®2 = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1506 int64 offsetVal1 = offset1->GetOffsetValue();
1507 int64 offsetVal2 = offset2->GetOffsetValue();
1508 if (base1->GetRegisterNumber() != base2->GetRegisterNumber() || reg1.GetRegisterType() != reg2.GetRegisterType() ||
1509 reg1.GetSize() != reg2.GetSize() || offsetVal1 != offsetVal2) {
1510 return;
1511 }
1512 if (loadAfterStore && reg1.GetRegisterNumber() != reg2.GetRegisterNumber()) {
1513 /* replace it with mov */
1514 MOperator newOp = MOP_undef;
1515 if (reg1.GetRegisterType() == kRegTyInt) {
1516 newOp = (insn.GetOperandSize(kInsnFirstOpnd) == k32BitSizeInt) ? MOP_wmovrr : MOP_xmovrr;
1517 } else if (reg1.GetRegisterType() == kRegTyFloat) {
1518 newOp = (insn.GetOperandSize(kInsnFirstOpnd) == k32BitSizeInt) ? MOP_xvmovs : MOP_xvmovd;
1519 }
1520 Insn *nextInsn = insn.GetNext();
1521 while (nextInsn != nullptr && !nextInsn->IsMachineInstruction() && nextInsn != bb.GetLastMachineInsn()) {
1522 nextInsn = nextInsn->GetNext();
1523 }
1524 bool moveSameReg = false;
1525 if (nextInsn && nextInsn->GetIsSpill() && !IfOperandIsLiveAfterInsn(reg1, *nextInsn)) {
1526 MOperator nextMop = nextInsn->GetMachineOpcode();
1527 if ((thisMop == MOP_xldr && nextMop == MOP_xstr) || (thisMop == MOP_wldr && nextMop == MOP_wstr) ||
1528 (thisMop == MOP_dldr && nextMop == MOP_dstr) || (thisMop == MOP_sldr && nextMop == MOP_sstr)) {
1529 nextInsn->Insn::SetOperand(kInsnFirstOpnd, reg2);
1530 moveSameReg = true;
1531 }
1532 }
1533 if (!moveSameReg) {
1534 bb.InsertInsnAfter(*prevInsn, cgFunc->GetInsnBuilder()->BuildInsn(newOp, reg1, reg2));
1535 }
1536 SetCurrInsn(insn.GetNextMachineInsn());
1537 optSuccess = true;
1538 bb.RemoveInsn(insn);
1539 } else if (reg1.GetRegisterNumber() == reg2.GetRegisterNumber() &&
1540 base1->GetRegisterNumber() != reg2.GetRegisterNumber()) {
1541 SetCurrInsn(insn.GetNextMachineInsn());
1542 optSuccess = true;
1543 bb.RemoveInsn(insn);
1544 }
1545 }
1546
Run(BB & bb,Insn & insn)1547 void CselZeroOneToCsetOpt::Run(BB &bb, Insn &insn)
1548 {
1549 Operand &trueValueOp = insn.GetOperand(kInsnSecondOpnd);
1550 Operand &falseValueOp = insn.GetOperand(kInsnThirdOpnd);
1551 Operand *trueTempOp = nullptr;
1552 Operand *falseTempOp = nullptr;
1553
1554 /* find fixed value in BB */
1555 if (!trueValueOp.IsIntImmediate()) {
1556 trueMovInsn = FindFixedValue(trueValueOp, bb, trueTempOp, insn);
1557 }
1558 if (!falseValueOp.IsIntImmediate()) {
1559 falseMovInsn = FindFixedValue(falseValueOp, bb, falseTempOp, insn);
1560 }
1561
1562 DEBUG_ASSERT(trueTempOp != nullptr, "trueTempOp should not be nullptr");
1563 DEBUG_ASSERT(falseTempOp != nullptr, "falseTempOp should not be nullptr");
1564 /* csel to cset */
1565 if ((trueTempOp->IsIntImmediate() || IsZeroRegister(*trueTempOp)) &&
1566 (falseTempOp->IsIntImmediate() || IsZeroRegister(*falseTempOp))) {
1567 ImmOperand *imm1 = static_cast<ImmOperand *>(trueTempOp);
1568 ImmOperand *imm2 = static_cast<ImmOperand *>(falseTempOp);
1569 bool inverse = imm1->IsOne() && (imm2->IsZero() || IsZeroRegister(*imm2));
1570 if (inverse || ((imm1->IsZero() || IsZeroRegister(*imm1)) && imm2->IsOne())) {
1571 Operand ® = insn.GetOperand(kInsnFirstOpnd);
1572 CondOperand &condOperand = static_cast<CondOperand &>(insn.GetOperand(kInsnFourthOpnd));
1573 MOperator mopCode = (reg.GetSize() == k64BitSize) ? MOP_xcsetrc : MOP_wcsetrc;
1574 /* get new cond ccCode */
1575 ConditionCode ccCode = inverse ? condOperand.GetCode() : GetReverseCC(condOperand.GetCode());
1576 if (ccCode == kCcLast) {
1577 return;
1578 }
1579 AArch64CGFunc *func = static_cast<AArch64CGFunc *>(cgFunc);
1580 CondOperand &cond = func->GetCondOperand(ccCode);
1581 Operand &rflag = func->GetOrCreateRflag();
1582 Insn &csetInsn = func->GetInsnBuilder()->BuildInsn(mopCode, reg, cond, rflag);
1583 if (CGOptions::DoCGSSA() && CGOptions::GetInstance().GetOptimizeLevel() < CGOptions::kLevel0) {
1584 CHECK_FATAL(false, "check this case in ssa opt");
1585 }
1586 insn.GetBB()->ReplaceInsn(insn, csetInsn);
1587 }
1588 }
1589 }
1590
FindFixedValue(Operand & opnd,BB & bb,Operand * & tempOp,const Insn & insn) const1591 Insn *CselZeroOneToCsetOpt::FindFixedValue(Operand &opnd, BB &bb, Operand *&tempOp, const Insn &insn) const
1592 {
1593 tempOp = &opnd;
1594 bool alreadyFindCsel = false;
1595 bool isRegDefined = false;
1596 regno_t regno = static_cast<RegOperand &>(opnd).GetRegisterNumber();
1597 FOR_BB_INSNS_REV(defInsn, &bb)
1598 {
1599 if (!defInsn->IsMachineInstruction() || defInsn->IsBranch()) {
1600 continue;
1601 }
1602 /* find csel */
1603 if (defInsn->GetId() == insn.GetId()) {
1604 alreadyFindCsel = true;
1605 }
1606 /* find def defined */
1607 if (alreadyFindCsel) {
1608 isRegDefined = defInsn->IsRegDefined(regno);
1609 }
1610 /* if def defined is movi do this opt */
1611 if (isRegDefined) {
1612 MOperator thisMop = defInsn->GetMachineOpcode();
1613 if (thisMop == MOP_wmovri32 || thisMop == MOP_xmovri64) {
1614 if (&defInsn->GetOperand(kInsnFirstOpnd) == &opnd) {
1615 tempOp = &(defInsn->GetOperand(kInsnSecondOpnd));
1616 return defInsn;
1617 }
1618 } else {
1619 return nullptr;
1620 }
1621 }
1622 }
1623 return nullptr;
1624 }
1625
Run(BB & bb,Insn & insn)1626 void AndCmpCsetEorCbzOpt::Run(BB &bb, Insn &insn)
1627 {
1628 if (insn.GetMachineOpcode() != MOP_wandrri12) {
1629 return;
1630 }
1631 RegOperand &andInsnFirstOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1632 RegOperand &andInsnSecondOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
1633 ImmOperand &andInsnThirdOpnd = static_cast<ImmOperand &>(insn.GetOperand(kInsnThirdOpnd));
1634 if (andInsnFirstOpnd.GetRegisterNumber() != andInsnSecondOpnd.GetRegisterNumber() ||
1635 andInsnThirdOpnd.GetValue() != 1) {
1636 return;
1637 }
1638 Insn *cmpInsn = insn.GetNextMachineInsn();
1639 if (cmpInsn == nullptr || cmpInsn->GetMachineOpcode() != MOP_wcmpri) {
1640 return;
1641 }
1642 RegOperand &cmpInsnSecondOpnd = static_cast<RegOperand &>(cmpInsn->GetOperand(kInsnSecondOpnd));
1643 ImmOperand &cmpInsnThirdOpnd = static_cast<ImmOperand &>(cmpInsn->GetOperand(kInsnThirdOpnd));
1644 if (cmpInsnSecondOpnd.GetRegisterNumber() != andInsnFirstOpnd.GetRegisterNumber() ||
1645 cmpInsnThirdOpnd.GetValue() != 0) {
1646 return;
1647 }
1648 Insn *csetInsn = cmpInsn->GetNextMachineInsn();
1649 if (csetInsn == nullptr || csetInsn->GetMachineOpcode() != MOP_wcsetrc) {
1650 return;
1651 }
1652 RegOperand &csetInsnFirstOpnd = static_cast<RegOperand &>(csetInsn->GetOperand(kInsnFirstOpnd));
1653 CondOperand &csetSecondOpnd = static_cast<CondOperand &>(csetInsn->GetOperand(kInsnSecondOpnd));
1654 if (csetInsnFirstOpnd.GetRegisterNumber() != andInsnFirstOpnd.GetRegisterNumber() ||
1655 csetSecondOpnd.GetCode() != CC_EQ) {
1656 return;
1657 }
1658 Insn *eorInsn = csetInsn->GetNextMachineInsn();
1659 if (eorInsn == nullptr || eorInsn->GetMachineOpcode() != MOP_weorrri12) {
1660 return;
1661 }
1662 RegOperand &eorInsnFirstOpnd = static_cast<RegOperand &>(eorInsn->GetOperand(kInsnFirstOpnd));
1663 RegOperand &eorInsnSecondOpnd = static_cast<RegOperand &>(eorInsn->GetOperand(kInsnSecondOpnd));
1664 ImmOperand &eorInsnThirdOpnd = static_cast<ImmOperand &>(eorInsn->GetOperand(kInsnThirdOpnd));
1665 if (eorInsnFirstOpnd.GetRegisterNumber() != andInsnFirstOpnd.GetRegisterNumber() ||
1666 eorInsnFirstOpnd.GetRegisterNumber() != eorInsnSecondOpnd.GetRegisterNumber() ||
1667 eorInsnThirdOpnd.GetValue() != 1) {
1668 return;
1669 }
1670 Insn *cbzInsn = eorInsn->GetNextMachineInsn();
1671 if (cbzInsn == nullptr || cbzInsn->GetMachineOpcode() != MOP_wcbz) {
1672 return;
1673 }
1674 RegOperand &cbzInsnFirstOpnd = static_cast<RegOperand &>(cbzInsn->GetOperand(kInsnFirstOpnd));
1675 if (cbzInsnFirstOpnd.GetRegisterNumber() != andInsnFirstOpnd.GetRegisterNumber()) {
1676 return;
1677 }
1678 bb.RemoveInsn(*cmpInsn);
1679 bb.RemoveInsn(*csetInsn);
1680 bb.RemoveInsn(*eorInsn);
1681 bb.RemoveInsn(*cbzInsn);
1682 /* replace insn */
1683 auto &label = static_cast<LabelOperand &>(cbzInsn->GetOperand(kInsnSecondOpnd));
1684 ImmOperand &oneHoleOpnd = static_cast<AArch64CGFunc *>(cgFunc)->CreateImmOperand(0, k8BitSize, false);
1685 bb.ReplaceInsn(insn, cgFunc->GetInsnBuilder()->BuildInsn(MOP_wtbz, cbzInsnFirstOpnd, oneHoleOpnd, label));
1686 }
1687
Run(BB & bb,Insn & insn)1688 void AddLdrOpt::Run(BB &bb, Insn &insn)
1689 {
1690 if (insn.GetMachineOpcode() != MOP_xaddrrr) {
1691 return;
1692 }
1693 Insn *nextInsn = insn.GetNextMachineInsn();
1694 if (nextInsn == nullptr) {
1695 return;
1696 }
1697 auto nextMop = nextInsn->GetMachineOpcode();
1698 if (nextMop != MOP_xldr && nextMop != MOP_wldr) {
1699 return;
1700 }
1701 RegOperand &insnFirstOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1702 RegOperand &insnSecondOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
1703 if (insnFirstOpnd.GetRegisterNumber() != insnSecondOpnd.GetRegisterNumber()) {
1704 return;
1705 }
1706 RegOperand &ldrInsnFirstOpnd = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnFirstOpnd));
1707 MemOperand &memOpnd = static_cast<MemOperand &>(nextInsn->GetOperand(kInsnSecondOpnd));
1708 if (memOpnd.GetAddrMode() != MemOperand::kAddrModeBOi ||
1709 memOpnd.GetBaseRegister()->GetRegisterNumber() != insnFirstOpnd.GetRegisterNumber() ||
1710 ldrInsnFirstOpnd.GetRegisterNumber() != insnFirstOpnd.GetRegisterNumber() ||
1711 memOpnd.GetOffsetImmediate()->GetOffsetValue() != 0) {
1712 return;
1713 }
1714 MemOperand &newMemOpnd = static_cast<AArch64CGFunc *>(cgFunc)->GetOrCreateMemOpnd(
1715 MemOperand::kAddrModeBOrX, memOpnd.GetSize(), &insnFirstOpnd,
1716 &static_cast<RegOperand &>(insn.GetOperand(kInsnThirdOpnd)), 0, false);
1717 nextInsn->SetOperand(kInsnSecondOpnd, newMemOpnd);
1718 bb.RemoveInsn(insn);
1719 }
1720
Run(BB & bb,Insn & insn)1721 void CsetEorOpt::Run(BB &bb, Insn &insn)
1722 {
1723 if (insn.GetMachineOpcode() != MOP_xcsetrc && insn.GetMachineOpcode() != MOP_wcsetrc) {
1724 return;
1725 }
1726 Insn *nextInsn = insn.GetNextMachineInsn();
1727 if (nextInsn == nullptr ||
1728 (nextInsn->GetMachineOpcode() != MOP_weorrri12 && nextInsn->GetMachineOpcode() != MOP_xeorrri13)) {
1729 return;
1730 }
1731 RegOperand &csetFirstOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1732 RegOperand &eorFirstOpnd = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnFirstOpnd));
1733 RegOperand &eorSecondOpnd = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnSecondOpnd));
1734 ImmOperand &eorThirdOpnd = static_cast<ImmOperand &>(nextInsn->GetOperand(kInsnThirdOpnd));
1735 if (eorThirdOpnd.GetValue() != 1 || eorFirstOpnd.GetRegisterNumber() != eorSecondOpnd.GetRegisterNumber() ||
1736 csetFirstOpnd.GetRegisterNumber() != eorFirstOpnd.GetRegisterNumber()) {
1737 return;
1738 }
1739 CondOperand &csetSecondOpnd = static_cast<CondOperand &>(insn.GetOperand(kInsnSecondOpnd));
1740 ConditionCode inverseCondCode = GetReverseCC(csetSecondOpnd.GetCode());
1741 if (inverseCondCode == kCcLast) {
1742 return;
1743 }
1744 auto *aarFunc = static_cast<AArch64CGFunc *>(cgFunc);
1745 CondOperand &inverseCondOpnd = aarFunc->GetCondOperand(inverseCondCode);
1746 insn.SetOperand(kInsnSecondOpnd, inverseCondOpnd);
1747 bb.RemoveInsn(*nextInsn);
1748 }
1749
Run(BB & bb,Insn & insn)1750 void MoveCmpOpt::Run(BB &bb, Insn &insn)
1751 {
1752 if (insn.GetMachineOpcode() != MOP_xmovri64 && insn.GetMachineOpcode() != MOP_wmovri32) {
1753 return;
1754 }
1755 ImmOperand &immOpnd = static_cast<ImmOperand &>(insn.GetOperand(kInsnSecondOpnd));
1756 if (!immOpnd.IsInBitSize(kMaxImmVal12Bits, 0) && !immOpnd.IsInBitSize(kMaxImmVal12Bits, kMaxImmVal12Bits)) {
1757 return;
1758 }
1759 Insn *nextInsn = insn.GetNextMachineInsn();
1760 if (nextInsn == nullptr ||
1761 (nextInsn->GetMachineOpcode() != MOP_wcmprr && nextInsn->GetMachineOpcode() != MOP_xcmprr)) {
1762 return;
1763 }
1764 RegOperand &cmpSecondOpnd = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnSecondOpnd));
1765 RegOperand &cmpThirdOpnd = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnThirdOpnd));
1766 RegOperand &movFirstOpnd = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1767 if (cmpSecondOpnd.GetRegisterNumber() == cmpThirdOpnd.GetRegisterNumber()) {
1768 return;
1769 }
1770 if (cmpThirdOpnd.GetRegisterNumber() != movFirstOpnd.GetRegisterNumber()) {
1771 return;
1772 }
1773 MOperator cmpOpCode = (cmpThirdOpnd.GetSize() == k64BitSize) ? MOP_xcmpri : MOP_wcmpri;
1774 Insn &newCmpInsn = cgFunc->GetInsnBuilder()->BuildInsn(cmpOpCode, nextInsn->GetOperand(kInsnFirstOpnd),
1775 nextInsn->GetOperand(kInsnSecondOpnd), immOpnd);
1776 bb.ReplaceInsn(*nextInsn, newCmpInsn);
1777 if (!IfOperandIsLiveAfterInsn(movFirstOpnd, newCmpInsn)) {
1778 bb.RemoveInsn(insn);
1779 }
1780 }
1781
Run(BB & bb,Insn & insn)1782 void AddImmZeroToMov::Run(BB &bb, Insn &insn)
1783 {
1784 RegOperand *insnDefReg = &static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1785 RegOperand *insnUseReg = &static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
1786 int64 immVal = static_cast<ImmOperand &>(insn.GetOperand(kInsnThirdOpnd)).GetValue();
1787 if (immVal == static_cast<int64>(k0BitSize)) {
1788 if (insnDefReg->GetRegisterNumber() == insnUseReg->GetRegisterNumber()) {
1789 bb.RemoveInsn(insn);
1790 return;
1791 } else {
1792 Insn *newInsn = &cgFunc->GetInsnBuilder()->BuildInsn(MOP_xmovrr, *insnDefReg, *insnUseReg);
1793 bb.ReplaceInsn(insn, *newInsn);
1794 return;
1795 }
1796 }
1797 return;
1798 }
1799
FindPreviousCmp(Insn & insn) const1800 Insn *AndCmpBranchesToCsetAArch64::FindPreviousCmp(Insn &insn) const
1801 {
1802 regno_t defRegNO = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd)).GetRegisterNumber();
1803 for (Insn *curInsn = insn.GetPrev(); curInsn != nullptr; curInsn = curInsn->GetPrev()) {
1804 if (!curInsn->IsMachineInstruction()) {
1805 continue;
1806 }
1807 if (curInsn->GetMachineOpcode() == MOP_wcmpri || curInsn->GetMachineOpcode() == MOP_xcmpri) {
1808 return curInsn;
1809 }
1810 /*
1811 * if any def/use of CC or insn defReg between insn and curInsn, stop searching and return nullptr.
1812 */
1813 if (curInsn->ScanReg(defRegNO) || curInsn->ScanReg(kRFLAG)) {
1814 return nullptr;
1815 }
1816 }
1817 return nullptr;
1818 }
1819
Run(BB & bb,Insn & insn)1820 void AndCmpBranchesToCsetAArch64::Run(BB &bb, Insn &insn)
1821 {
1822 /* prevInsn must be "cmp" insn */
1823 Insn *prevInsn = FindPreviousCmp(insn);
1824 if (prevInsn == nullptr) {
1825 return;
1826 }
1827 /* prevPrevInsn must be "and" insn */
1828 Insn *prevPrevInsn = prevInsn->GetPreviousMachineInsn();
1829 if (prevPrevInsn == nullptr ||
1830 (prevPrevInsn->GetMachineOpcode() != MOP_wandrri12 && prevPrevInsn->GetMachineOpcode() != MOP_xandrri13)) {
1831 return;
1832 }
1833
1834 auto &csetCond = static_cast<CondOperand &>(insn.GetOperand(kInsnSecondOpnd));
1835 auto &cmpImm = static_cast<ImmOperand &>(prevInsn->GetOperand(kInsnThirdOpnd));
1836 int64 cmpImmVal = cmpImm.GetValue();
1837 auto &andImm = static_cast<ImmOperand &>(prevPrevInsn->GetOperand(kInsnThirdOpnd));
1838 int64 andImmVal = andImm.GetValue();
1839 if ((csetCond.GetCode() == CC_EQ && cmpImmVal == andImmVal) || (csetCond.GetCode() == CC_NE && cmpImmVal == 0)) {
1840 /* if flag_reg of "cmp" is live later, we can't remove cmp insn. */
1841 auto &flagReg = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
1842 if (IfOperandIsLiveAfterInsn(flagReg, insn)) {
1843 return;
1844 }
1845
1846 auto &csetReg = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1847 auto &prevInsnSecondReg = prevInsn->GetOperand(kInsnSecondOpnd);
1848 bool isRegDiff = !RegOperand::IsSameRegNO(csetReg, prevInsnSecondReg);
1849 if (isRegDiff && IfOperandIsLiveAfterInsn(static_cast<RegOperand &>(prevInsnSecondReg), insn)) {
1850 return;
1851 }
1852 if (andImmVal == 1) {
1853 if (!RegOperand::IsSameRegNO(prevInsnSecondReg, prevPrevInsn->GetOperand(kInsnFirstOpnd))) {
1854 return;
1855 }
1856 /* save the "and" insn only. */
1857 bb.RemoveInsn(insn);
1858 bb.RemoveInsn(*prevInsn);
1859 if (isRegDiff) {
1860 prevPrevInsn->Insn::SetOperand(kInsnFirstOpnd, csetReg);
1861 }
1862 } else {
1863 if (!RegOperand::IsSameReg(prevInsnSecondReg, prevPrevInsn->GetOperand(kInsnFirstOpnd)) ||
1864 !RegOperand::IsSameReg(prevInsnSecondReg, prevPrevInsn->GetOperand(kInsnSecondOpnd))) {
1865 return;
1866 }
1867
1868 /* andImmVal is n power of 2 */
1869 int n = LogValueAtBase2(andImmVal);
1870 if (n < 0) {
1871 return;
1872 }
1873
1874 /* create ubfx insn */
1875 MOperator ubfxOp = (csetReg.GetSize() <= k32BitSize) ? MOP_wubfxrri5i5 : MOP_xubfxrri6i6;
1876 if (ubfxOp == MOP_wubfxrri5i5 && static_cast<uint32>(n) >= k32BitSize) {
1877 return;
1878 }
1879 auto &dstReg = static_cast<RegOperand &>(csetReg);
1880 auto &srcReg = static_cast<RegOperand &>(prevInsnSecondReg);
1881 auto *aarch64CGFunc = static_cast<AArch64CGFunc *>(&cgFunc);
1882 ImmOperand &bitPos = aarch64CGFunc->CreateImmOperand(n, k8BitSize, false);
1883 ImmOperand &bitSize = aarch64CGFunc->CreateImmOperand(1, k8BitSize, false);
1884 Insn &ubfxInsn = cgFunc.GetInsnBuilder()->BuildInsn(ubfxOp, dstReg, srcReg, bitPos, bitSize);
1885 bb.InsertInsnBefore(*prevPrevInsn, ubfxInsn);
1886 bb.RemoveInsn(insn);
1887 bb.RemoveInsn(*prevInsn);
1888 bb.RemoveInsn(*prevPrevInsn);
1889 }
1890 }
1891 }
1892
Run(BB & bb,Insn & insn)1893 void AndCbzBranchesToTstAArch64::Run(BB &bb, Insn &insn)
1894 {
1895 /* nextInsn must be "cbz" or "cbnz" insn */
1896 Insn *nextInsn = insn.GetNextMachineInsn();
1897 if (nextInsn == nullptr || (nextInsn->GetMachineOpcode() != MOP_wcbz && nextInsn->GetMachineOpcode() != MOP_xcbz)) {
1898 return;
1899 }
1900 auto &andRegOp = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1901 regno_t andRegNO1 = andRegOp.GetRegisterNumber();
1902 auto &cbzRegOp2 = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnFirstOpnd));
1903 regno_t cbzRegNO2 = cbzRegOp2.GetRegisterNumber();
1904 if (andRegNO1 != cbzRegNO2) {
1905 return;
1906 }
1907 /* If the reg will be used later, we shouldn't optimize the and insn here */
1908 if (IfOperandIsLiveAfterInsn(andRegOp, *nextInsn)) {
1909 return;
1910 }
1911 /* build tst insn */
1912 Operand &andOpnd3 = insn.GetOperand(kInsnThirdOpnd);
1913 auto &andRegOp2 = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
1914 MOperator newTstOp = MOP_undef;
1915 if (andOpnd3.IsRegister()) {
1916 newTstOp = (andRegOp2.GetSize() <= k32BitSize && andOpnd3.GetSize() <= k32BitSize) ? MOP_wtstrr : MOP_xtstrr;
1917 } else {
1918 newTstOp =
1919 (andRegOp2.GetSize() <= k32BitSize && andOpnd3.GetSize() <= k32BitSize) ? MOP_wtstri32 : MOP_xtstri64;
1920 }
1921 Operand &rflag = static_cast<AArch64CGFunc *>(&cgFunc)->GetOrCreateRflag();
1922 Insn &newInsnTst = cgFunc.GetInsnBuilder()->BuildInsn(newTstOp, rflag, andRegOp2, andOpnd3);
1923 if (andOpnd3.IsImmediate()) {
1924 if (!static_cast<ImmOperand &>(andOpnd3).IsBitmaskImmediate(andRegOp2.GetSize())) {
1925 return;
1926 }
1927 }
1928 /* build beq insn */
1929 MOperator opCode = nextInsn->GetMachineOpcode();
1930 bool reverse = (opCode == MOP_xcbz || opCode == MOP_wcbz);
1931 auto &label = static_cast<LabelOperand &>(nextInsn->GetOperand(kInsnSecondOpnd));
1932 MOperator jmpOperator = reverse ? MOP_beq : MOP_bne;
1933 Insn &newInsnJmp = cgFunc.GetInsnBuilder()->BuildInsn(jmpOperator, rflag, label);
1934 bb.ReplaceInsn(insn, newInsnTst);
1935 bb.ReplaceInsn(*nextInsn, newInsnJmp);
1936 }
1937
Run(BB & bb,Insn & insn)1938 void AndCmpBranchesToTstAArch64::Run(BB &bb, Insn &insn)
1939 {
1940 /* nextInsn must be "cmp" insn */
1941 Insn *nextInsn = insn.GetNextMachineInsn();
1942 if (nextInsn == nullptr ||
1943 (nextInsn->GetMachineOpcode() != MOP_wcmpri && nextInsn->GetMachineOpcode() != MOP_xcmpri)) {
1944 return;
1945 }
1946 /* nextNextInsn must be "beq" or "bne" insn */
1947 Insn *nextNextInsn = nextInsn->GetNextMachineInsn();
1948 if (nextNextInsn == nullptr ||
1949 (nextNextInsn->GetMachineOpcode() != MOP_beq && nextNextInsn->GetMachineOpcode() != MOP_bne)) {
1950 return;
1951 }
1952 auto &andRegOp = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
1953 regno_t andRegNO1 = andRegOp.GetRegisterNumber();
1954 auto &cmpRegOp2 = static_cast<RegOperand &>(nextInsn->GetOperand(kInsnSecondOpnd));
1955 regno_t cmpRegNO2 = cmpRegOp2.GetRegisterNumber();
1956 if (andRegNO1 != cmpRegNO2) {
1957 return;
1958 }
1959 /* If the reg will be used later, we shouldn't optimize the and insn here */
1960 if (IfOperandIsLiveAfterInsn(andRegOp, *nextInsn)) {
1961 return;
1962 }
1963 Operand &immOpnd = nextInsn->GetOperand(kInsnThirdOpnd);
1964 DEBUG_ASSERT(immOpnd.IsIntImmediate(), "expects ImmOperand");
1965 auto &defConst = static_cast<ImmOperand &>(immOpnd);
1966 int64 defConstValue = defConst.GetValue();
1967 if (defConstValue != 0) {
1968 return;
1969 }
1970 /* build tst insn */
1971 Operand &andOpnd3 = insn.GetOperand(kInsnThirdOpnd);
1972 auto &andRegOp2 = static_cast<RegOperand &>(insn.GetOperand(kInsnSecondOpnd));
1973 MOperator newOp = MOP_undef;
1974 if (andOpnd3.IsRegister()) {
1975 newOp = (andRegOp2.GetSize() <= k32BitSize) ? MOP_wtstrr : MOP_xtstrr;
1976 } else {
1977 newOp = (andRegOp2.GetSize() <= k32BitSize) ? MOP_wtstri32 : MOP_xtstri64;
1978 }
1979 Operand &rflag = static_cast<AArch64CGFunc *>(&cgFunc)->GetOrCreateRflag();
1980 Insn &newInsn = cgFunc.GetInsnBuilder()->BuildInsn(newOp, rflag, andRegOp2, andOpnd3);
1981 if (CGOptions::DoCGSSA() && CGOptions::GetInstance().GetOptimizeLevel() < CGOptions::kLevel0) {
1982 CHECK_FATAL(false, "check this case in ssa opt");
1983 }
1984 bb.InsertInsnAfter(*nextInsn, newInsn);
1985 bb.RemoveInsn(insn);
1986 bb.RemoveInsn(*nextInsn);
1987 }
1988
Run(BB & bb,Insn & insn)1989 void ZeroCmpBranchesAArch64::Run(BB &bb, Insn &insn)
1990 {
1991 Insn *prevInsn = insn.GetPreviousMachineInsn();
1992 if (!insn.IsBranch() || insn.GetOperandSize() <= kInsnSecondOpnd || prevInsn == nullptr) {
1993 return;
1994 }
1995 if (!insn.GetOperand(kInsnSecondOpnd).IsLabel()) {
1996 return;
1997 }
1998 LabelOperand *label = &static_cast<LabelOperand &>(insn.GetOperand(kInsnSecondOpnd));
1999 RegOperand *regOpnd = nullptr;
2000 RegOperand *reg0 = nullptr;
2001 RegOperand *reg1 = nullptr;
2002 MOperator newOp = MOP_undef;
2003 ImmOperand *imm = nullptr;
2004 switch (prevInsn->GetMachineOpcode()) {
2005 case MOP_wcmpri:
2006 case MOP_xcmpri: {
2007 regOpnd = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnSecondOpnd));
2008 imm = &static_cast<ImmOperand &>(prevInsn->GetOperand(kInsnThirdOpnd));
2009 if (imm->GetValue() != 0) {
2010 return;
2011 }
2012 if (insn.GetMachineOpcode() == MOP_bge) {
2013 newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbz : MOP_xtbz;
2014 } else if (insn.GetMachineOpcode() == MOP_blt) {
2015 newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbnz : MOP_xtbnz;
2016 } else {
2017 return;
2018 }
2019 break;
2020 }
2021 case MOP_wcmprr:
2022 case MOP_xcmprr: {
2023 reg0 = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnSecondOpnd));
2024 reg1 = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnThirdOpnd));
2025 if (!IsZeroRegister(*reg0) && !IsZeroRegister(*reg1)) {
2026 return;
2027 }
2028 switch (insn.GetMachineOpcode()) {
2029 case MOP_bge:
2030 if (IsZeroRegister(*reg1)) {
2031 regOpnd = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnSecondOpnd));
2032 newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbz : MOP_xtbz;
2033 } else {
2034 return;
2035 }
2036 break;
2037 case MOP_ble:
2038 if (IsZeroRegister(*reg0)) {
2039 regOpnd = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnThirdOpnd));
2040 newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbz : MOP_xtbz;
2041 } else {
2042 return;
2043 }
2044 break;
2045 case MOP_blt:
2046 if (IsZeroRegister(*reg1)) {
2047 regOpnd = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnSecondOpnd));
2048 newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbnz : MOP_xtbnz;
2049 } else {
2050 return;
2051 }
2052 break;
2053 case MOP_bgt:
2054 if (IsZeroRegister(*reg0)) {
2055 regOpnd = &static_cast<RegOperand &>(prevInsn->GetOperand(kInsnThirdOpnd));
2056 newOp = (regOpnd->GetSize() <= k32BitSize) ? MOP_wtbnz : MOP_xtbnz;
2057 } else {
2058 return;
2059 }
2060 break;
2061 default:
2062 return;
2063 }
2064 break;
2065 }
2066 default:
2067 return;
2068 }
2069 auto aarch64CGFunc = static_cast<AArch64CGFunc *>(&cgFunc);
2070 ImmOperand &bitp = aarch64CGFunc->CreateImmOperand(
2071 (regOpnd->GetSize() <= k32BitSize) ? (k32BitSize - 1) : (k64BitSize - 1), k8BitSize, false);
2072 bb.InsertInsnAfter(insn,
2073 cgFunc.GetInsnBuilder()->BuildInsn(newOp, *static_cast<RegOperand *>(regOpnd), bitp, *label));
2074 bb.RemoveInsn(insn);
2075 bb.RemoveInsn(*prevInsn);
2076 }
2077
2078 /*
2079 * if there is define point of checkInsn->GetOperand(opndIdx) between startInsn and firstInsn
2080 * return define insn. else return nullptr
2081 */
DefInsnOfOperandInBB(const Insn & startInsn,const Insn & checkInsn,int opndIdx) const2082 const Insn *CmpCsetAArch64::DefInsnOfOperandInBB(const Insn &startInsn, const Insn &checkInsn, int opndIdx) const
2083 {
2084 Insn *prevInsn = nullptr;
2085 for (const Insn *insn = &startInsn; insn != nullptr; insn = prevInsn) {
2086 prevInsn = insn->GetPreviousMachineInsn();
2087 if (!insn->IsMachineInstruction()) {
2088 continue;
2089 }
2090 /* checkInsn.GetOperand(opndIdx) is thought modified conservatively */
2091 if (insn->IsCall()) {
2092 return insn;
2093 }
2094 const InsnDesc *md = insn->GetDesc();
2095 uint32 opndNum = insn->GetOperandSize();
2096 for (uint32 i = 0; i < opndNum; ++i) {
2097 Operand &opnd = insn->GetOperand(i);
2098 if (!md->opndMD[i]->IsDef()) {
2099 continue;
2100 }
2101 /* Operand is base reg of Memory, defined by str */
2102 if (opnd.IsMemoryAccessOperand()) {
2103 auto &memOpnd = static_cast<MemOperand &>(opnd);
2104 RegOperand *base = memOpnd.GetBaseRegister();
2105 DEBUG_ASSERT(base != nullptr, "nullptr check");
2106 DEBUG_ASSERT(base->IsRegister(), "expects RegOperand");
2107 if (RegOperand::IsSameRegNO(*base, checkInsn.GetOperand(static_cast<uint32>(opndIdx))) &&
2108 memOpnd.GetAddrMode() == MemOperand::kAddrModeBOi &&
2109 (memOpnd.IsPostIndexed() || memOpnd.IsPreIndexed())) {
2110 return insn;
2111 }
2112 } else {
2113 DEBUG_ASSERT(opnd.IsRegister(), "expects RegOperand");
2114 if (RegOperand::IsSameRegNO(checkInsn.GetOperand(static_cast<uint32>(opndIdx)), opnd)) {
2115 return insn;
2116 }
2117 }
2118 }
2119 }
2120 return nullptr;
2121 }
2122
OpndDefByOneValidBit(const Insn & defInsn) const2123 bool CmpCsetAArch64::OpndDefByOneValidBit(const Insn &defInsn) const
2124 {
2125 MOperator defMop = defInsn.GetMachineOpcode();
2126 switch (defMop) {
2127 case MOP_wcsetrc:
2128 case MOP_xcsetrc:
2129 return true;
2130 case MOP_wmovri32:
2131 case MOP_xmovri64: {
2132 Operand &defOpnd = defInsn.GetOperand(kInsnSecondOpnd);
2133 DEBUG_ASSERT(defOpnd.IsIntImmediate(), "expects ImmOperand");
2134 auto &defConst = static_cast<ImmOperand &>(defOpnd);
2135 int64 defConstValue = defConst.GetValue();
2136 return (defConstValue == 0 || defConstValue == 1);
2137 }
2138 case MOP_xmovrr:
2139 case MOP_wmovrr:
2140 return IsZeroRegister(defInsn.GetOperand(kInsnSecondOpnd));
2141 case MOP_wlsrrri5:
2142 case MOP_xlsrrri6: {
2143 Operand &opnd2 = defInsn.GetOperand(kInsnThirdOpnd);
2144 DEBUG_ASSERT(opnd2.IsIntImmediate(), "expects ImmOperand");
2145 auto &opndImm = static_cast<ImmOperand &>(opnd2);
2146 int64 shiftBits = opndImm.GetValue();
2147 return ((defMop == MOP_wlsrrri5 && shiftBits == (k32BitSize - 1)) ||
2148 (defMop == MOP_xlsrrri6 && shiftBits == (k64BitSize - 1)));
2149 }
2150 default:
2151 return false;
2152 }
2153 }
2154
2155 /*
2156 * help function for cmpcset optimize
2157 * if all define points of used opnd in insn has only one valid bit,return true.
2158 * for cmp reg,#0(#1), that is checking for reg
2159 */
CheckOpndDefPoints(Insn & checkInsn,int opndIdx)2160 bool CmpCsetAArch64::CheckOpndDefPoints(Insn &checkInsn, int opndIdx)
2161 {
2162 if (checkInsn.GetBB()->GetPrev() == nullptr) {
2163 /* For 1st BB, be conservative for def of parameter registers */
2164 /* Since peep is light weight, do not want to insert pseudo defs */
2165 regno_t reg = static_cast<RegOperand &>(checkInsn.GetOperand(static_cast<uint32>(opndIdx))).GetRegisterNumber();
2166 if ((reg >= R0 && reg <= R7) || (reg >= D0 && reg <= D7)) {
2167 return false;
2168 }
2169 }
2170 /* check current BB */
2171 const Insn *defInsn = DefInsnOfOperandInBB(checkInsn, checkInsn, opndIdx);
2172 if (defInsn != nullptr) {
2173 return OpndDefByOneValidBit(*defInsn);
2174 }
2175 /* check pred */
2176 for (auto predBB : checkInsn.GetBB()->GetPreds()) {
2177 const Insn *tempInsn = nullptr;
2178 if (predBB->GetLastInsn() != nullptr) {
2179 tempInsn = DefInsnOfOperandInBB(*predBB->GetLastInsn(), checkInsn, opndIdx);
2180 }
2181 if (tempInsn == nullptr || !OpndDefByOneValidBit(*tempInsn)) {
2182 return false;
2183 }
2184 }
2185 return true;
2186 }
2187
2188 /* Check there is use point of rflag start from startInsn to current bb bottom */
FlagUsedLaterInCurBB(const BB & bb,Insn & startInsn) const2189 bool CmpCsetAArch64::FlagUsedLaterInCurBB(const BB &bb, Insn &startInsn) const
2190 {
2191 if (&bb != startInsn.GetBB()) {
2192 return false;
2193 }
2194 Insn *nextInsn = nullptr;
2195 for (Insn *insn = &startInsn; insn != nullptr; insn = nextInsn) {
2196 nextInsn = insn->GetNextMachineInsn();
2197 const InsnDesc *md = insn->GetDesc();
2198 uint32 opndNum = insn->GetOperandSize();
2199 for (uint32 i = 0; i < opndNum; ++i) {
2200 Operand &opnd = insn->GetOperand(i);
2201 /*
2202 * For condition operand, such as NE, EQ and so on, the register number should be
2203 * same with RFLAG, we only need check the property of use/def.
2204 */
2205 if (!opnd.IsConditionCode()) {
2206 continue;
2207 }
2208 if (md->opndMD[i]->IsUse()) {
2209 return true;
2210 } else {
2211 DEBUG_ASSERT(md->opndMD[i]->IsDef(), "register should be redefined.");
2212 return false;
2213 }
2214 }
2215 }
2216 return false;
2217 }
2218
Run(BB & bb,Insn & insn)2219 void CmpCsetAArch64::Run(BB &bb, Insn &insn)
2220 {
2221 Insn *nextInsn = insn.GetNextMachineInsn();
2222 if (nextInsn == nullptr) {
2223 return;
2224 }
2225 MOperator firstMop = insn.GetMachineOpcode();
2226 MOperator secondMop = nextInsn->GetMachineOpcode();
2227 if ((firstMop == MOP_wcmpri || firstMop == MOP_xcmpri) && (secondMop == MOP_wcsetrc || secondMop == MOP_xcsetrc)) {
2228 Operand &cmpFirstOpnd = insn.GetOperand(kInsnSecondOpnd);
2229 /* get ImmOperand, must be 0 or 1 */
2230 Operand &cmpSecondOpnd = insn.GetOperand(kInsnThirdOpnd);
2231 auto &cmpFlagReg = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd));
2232 DEBUG_ASSERT(cmpSecondOpnd.IsIntImmediate(), "expects ImmOperand");
2233 auto &cmpConst = static_cast<ImmOperand &>(cmpSecondOpnd);
2234 int64 cmpConstVal = cmpConst.GetValue();
2235 Operand &csetFirstOpnd = nextInsn->GetOperand(kInsnFirstOpnd);
2236 if ((cmpConstVal != 0 && cmpConstVal != 1) || !CheckOpndDefPoints(insn, 1) ||
2237 (nextInsn->GetNextMachineInsn() != nullptr && FlagUsedLaterInCurBB(bb, *nextInsn->GetNextMachineInsn())) ||
2238 FindRegLiveOut(cmpFlagReg, *insn.GetBB())) {
2239 return;
2240 }
2241
2242 Insn *csetInsn = nextInsn;
2243 nextInsn = nextInsn->GetNextMachineInsn();
2244 auto &cond = static_cast<CondOperand &>(csetInsn->GetOperand(kInsnSecondOpnd));
2245 if ((cmpConstVal == 0 && cond.GetCode() == CC_NE) || (cmpConstVal == 1 && cond.GetCode() == CC_EQ)) {
2246 if (RegOperand::IsSameRegNO(cmpFirstOpnd, csetFirstOpnd)) {
2247 bb.RemoveInsn(insn);
2248 bb.RemoveInsn(*csetInsn);
2249 } else {
2250 if (cmpFirstOpnd.GetSize() != csetFirstOpnd.GetSize()) {
2251 return;
2252 }
2253 MOperator mopCode = (cmpFirstOpnd.GetSize() == k64BitSize) ? MOP_xmovrr : MOP_wmovrr;
2254 Insn &newInsn = cgFunc.GetInsnBuilder()->BuildInsn(mopCode, csetFirstOpnd, cmpFirstOpnd);
2255 bb.ReplaceInsn(insn, newInsn);
2256 bb.RemoveInsn(*csetInsn);
2257 }
2258 } else if ((cmpConstVal == 1 && cond.GetCode() == CC_NE) || (cmpConstVal == 0 && cond.GetCode() == CC_EQ)) {
2259 if (cmpFirstOpnd.GetSize() != csetFirstOpnd.GetSize()) {
2260 return;
2261 }
2262 MOperator mopCode = (cmpFirstOpnd.GetSize() == k64BitSize) ? MOP_xeorrri13 : MOP_weorrri12;
2263 ImmOperand &one = static_cast<AArch64CGFunc *>(&cgFunc)->CreateImmOperand(1, k8BitSize, false);
2264 Insn &newInsn = cgFunc.GetInsnBuilder()->BuildInsn(mopCode, csetFirstOpnd, cmpFirstOpnd, one);
2265 bb.ReplaceInsn(insn, newInsn);
2266 bb.RemoveInsn(*csetInsn);
2267 }
2268 }
2269 }
2270
2271 /* ldr wn, [x1, wn, SXTW]
2272 * add x2, wn, x2
2273 */
IsExpandBaseOpnd(const Insn & insn,const Insn & prevInsn) const2274 bool ComplexMemOperandAddAArch64::IsExpandBaseOpnd(const Insn &insn, const Insn &prevInsn) const
2275 {
2276 MOperator prevMop = prevInsn.GetMachineOpcode();
2277 if (prevMop >= MOP_wldrsb && prevMop <= MOP_xldr &&
2278 prevInsn.GetOperand(kInsnFirstOpnd).Equals(insn.GetOperand(kInsnSecondOpnd))) {
2279 return true;
2280 }
2281 return false;
2282 }
2283
Run(BB & bb,Insn & insn)2284 void ComplexMemOperandAddAArch64::Run(BB &bb, Insn &insn)
2285 {
2286 AArch64CGFunc *aarch64CGFunc = static_cast<AArch64CGFunc *>(&cgFunc);
2287 Insn *nextInsn = insn.GetNextMachineInsn();
2288 if (nextInsn == nullptr) {
2289 return;
2290 }
2291 Insn *prevInsn = insn.GetPreviousMachineInsn();
2292 MOperator thisMop = insn.GetMachineOpcode();
2293 if (thisMop != MOP_xaddrrr && thisMop != MOP_waddrrr) {
2294 return;
2295 }
2296 MOperator nextMop = nextInsn->GetMachineOpcode();
2297 if (nextMop && ((nextMop >= MOP_wldrsb && nextMop <= MOP_dldr) || (nextMop >= MOP_wstrb && nextMop <= MOP_dstr))) {
2298 if (!IsMemOperandOptPattern(insn, *nextInsn)) {
2299 return;
2300 }
2301 MemOperand *memOpnd = static_cast<MemOperand *>(nextInsn->GetMemOpnd());
2302 auto newBaseOpnd = static_cast<RegOperand *>(&insn.GetOperand(kInsnSecondOpnd));
2303 auto newIndexOpnd = static_cast<RegOperand *>(&insn.GetOperand(kInsnThirdOpnd));
2304 regno_t memBaseOpndRegNO = newBaseOpnd->GetRegisterNumber();
2305 if (newBaseOpnd->GetSize() <= k32BitSize && prevInsn != nullptr && IsExpandBaseOpnd(insn, *prevInsn)) {
2306 newBaseOpnd = &aarch64CGFunc->GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(memBaseOpndRegNO),
2307 k64BitSize, kRegTyInt);
2308 }
2309 if (newBaseOpnd->GetSize() != k64BitSize) {
2310 return;
2311 }
2312 DEBUG_ASSERT(memOpnd != nullptr, "memOpnd should not be nullptr");
2313 if (newIndexOpnd->GetSize() <= k32BitSize) {
2314 MemOperand &newMemOpnd = aarch64CGFunc->GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, memOpnd->GetSize(),
2315 newBaseOpnd, newIndexOpnd, 0, false);
2316 nextInsn->SetOperand(kInsnSecondOpnd, newMemOpnd);
2317 } else {
2318 MemOperand &newMemOpnd = aarch64CGFunc->GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, memOpnd->GetSize(),
2319 newBaseOpnd, newIndexOpnd, nullptr, nullptr);
2320 nextInsn->SetOperand(kInsnSecondOpnd, newMemOpnd);
2321 }
2322 bb.RemoveInsn(insn);
2323 }
2324 }
2325
Run(BB & bb,Insn & insn)2326 void RemoveSxtBeforeStrAArch64::Run(BB &bb, Insn &insn)
2327 {
2328 MOperator mop = insn.GetMachineOpcode();
2329 Insn *prevInsn = insn.GetPreviousMachineInsn();
2330 if (prevInsn == nullptr) {
2331 return;
2332 }
2333 MOperator prevMop = prevInsn->GetMachineOpcode();
2334 if (!(mop == MOP_wstrh && prevMop == MOP_xsxth32) && !(mop == MOP_wstrb && prevMop == MOP_xsxtb32)) {
2335 return;
2336 }
2337 auto &prevOpnd0 = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnFirstOpnd));
2338 if (IfOperandIsLiveAfterInsn(prevOpnd0, insn)) {
2339 return;
2340 }
2341 auto &prevOpnd1 = static_cast<RegOperand &>(prevInsn->GetOperand(kInsnSecondOpnd));
2342 regno_t prevRegNO0 = prevOpnd0.GetRegisterNumber();
2343 regno_t prevRegNO1 = prevOpnd1.GetRegisterNumber();
2344 regno_t regNO0 = static_cast<RegOperand &>(insn.GetOperand(kInsnFirstOpnd)).GetRegisterNumber();
2345 if (prevRegNO0 != prevRegNO1) {
2346 return;
2347 }
2348 if (prevRegNO0 == regNO0) {
2349 bb.RemoveInsn(*prevInsn);
2350 return;
2351 }
2352 insn.SetOperand(0, prevOpnd1);
2353 bb.RemoveInsn(*prevInsn);
2354 }
2355 } /* namespace maplebe */
2356