• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) 2023 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  *     http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "aarch64_cg.h"
17 #include "aarch64_cgfunc.h"
18 #include <vector>
19 #include <cstdint>
20 #include <sys/stat.h>
21 #include <atomic>
22 #include "cfi.h"
23 #include "mpl_logging.h"
24 #include "rt.h"
25 #include "opcode_info.h"
26 #include "mir_builder.h"
27 #include "mir_symbol_builder.h"
28 #include "mpl_atomic.h"
29 #include "metadata_layout.h"
30 #include "emit.h"
31 #include "simplify.h"
32 #include <algorithm>
33 
34 namespace maplebe {
35 using namespace maple;
36 CondOperand AArch64CGFunc::ccOperands[kCcLast] = {
37     CondOperand(CC_EQ), CondOperand(CC_NE), CondOperand(CC_CS), CondOperand(CC_HS), CondOperand(CC_CC),
38     CondOperand(CC_LO), CondOperand(CC_MI), CondOperand(CC_PL), CondOperand(CC_VS), CondOperand(CC_VC),
39     CondOperand(CC_HI), CondOperand(CC_LS), CondOperand(CC_GE), CondOperand(CC_LT), CondOperand(CC_GT),
40     CondOperand(CC_LE), CondOperand(CC_AL),
41 };
42 
43 namespace {
44 constexpr int32 kSignedDimension = 2;        /* signed and unsigned */
45 constexpr int32 kIntByteSizeDimension = 4;   /* 1 byte, 2 byte, 4 bytes, 8 bytes */
46 constexpr int32 kFloatByteSizeDimension = 3; /* 4 bytes, 8 bytes, 16 bytes(vector) */
47 constexpr int32 kShiftAmount12 = 12;         /* for instruction that can use shift, shift amount must be 0 or 12 */
48 
49 MOperator ldIs[kSignedDimension][kIntByteSizeDimension] = {
50     /* unsigned == 0 */
51     {MOP_wldrb, MOP_wldrh, MOP_wldr, MOP_xldr},
52     /* signed == 1 */
53     {MOP_wldrsb, MOP_wldrsh, MOP_wldr, MOP_xldr}};
54 
55 MOperator stIs[kSignedDimension][kIntByteSizeDimension] = {
56     /* unsigned == 0 */
57     {MOP_wstrb, MOP_wstrh, MOP_wstr, MOP_xstr},
58     /* signed == 1 */
59     {MOP_wstrb, MOP_wstrh, MOP_wstr, MOP_xstr}};
60 
61 MOperator ldIsAcq[kSignedDimension][kIntByteSizeDimension] = {
62     /* unsigned == 0 */
63     {MOP_wldarb, MOP_wldarh, MOP_wldar, MOP_xldar},
64     /* signed == 1 */
65     {MOP_undef, MOP_undef, MOP_wldar, MOP_xldar}};
66 
67 MOperator stIsRel[kSignedDimension][kIntByteSizeDimension] = {
68     /* unsigned == 0 */
69     {MOP_wstlrb, MOP_wstlrh, MOP_wstlr, MOP_xstlr},
70     /* signed == 1 */
71     {MOP_wstlrb, MOP_wstlrh, MOP_wstlr, MOP_xstlr}};
72 
73 MOperator ldFs[kFloatByteSizeDimension] = {MOP_sldr, MOP_dldr, MOP_qldr};
74 MOperator stFs[kFloatByteSizeDimension] = {MOP_sstr, MOP_dstr, MOP_qstr};
75 
76 MOperator ldFsAcq[kFloatByteSizeDimension] = {MOP_undef, MOP_undef, MOP_undef};
77 MOperator stFsRel[kFloatByteSizeDimension] = {MOP_undef, MOP_undef, MOP_undef};
78 
79 /* extended to unsigned ints */
80 MOperator uextIs[kIntByteSizeDimension][kIntByteSizeDimension] = {
81     /*  u8         u16          u32          u64      */
82     {MOP_undef, MOP_xuxtb32, MOP_xuxtb32, MOP_xuxtb32}, /* u8/i8   */
83     {MOP_undef, MOP_undef, MOP_xuxth32, MOP_xuxth32},   /* u16/i16 */
84     {MOP_undef, MOP_undef, MOP_xuxtw64, MOP_xuxtw64},   /* u32/i32 */
85     {MOP_undef, MOP_undef, MOP_undef, MOP_undef}        /* u64/u64 */
86 };
87 
88 /* extended to signed ints */
89 MOperator extIs[kIntByteSizeDimension][kIntByteSizeDimension] = {
90     /*  i8         i16          i32          i64      */
91     {MOP_undef, MOP_xsxtb32, MOP_xsxtb32, MOP_xsxtb64}, /* u8/i8   */
92     {MOP_undef, MOP_undef, MOP_xsxth32, MOP_xsxth64},   /* u16/i16 */
93     {MOP_undef, MOP_undef, MOP_undef, MOP_xsxtw64},     /* u32/i32 */
94     {MOP_undef, MOP_undef, MOP_undef, MOP_undef}        /* u64/u64 */
95 };
96 
PickLdStInsn(bool isLoad,uint32 bitSize,PrimType primType,AArch64isa::MemoryOrdering memOrd)97 MOperator PickLdStInsn(bool isLoad, uint32 bitSize, PrimType primType, AArch64isa::MemoryOrdering memOrd)
98 {
99     DEBUG_ASSERT(__builtin_popcount(static_cast<uint32>(memOrd)) <= 1, "must be kMoNone or kMoAcquire");
100     DEBUG_ASSERT(bitSize >= k8BitSize, "PTY_u1 should have been lowered?");
101     DEBUG_ASSERT(__builtin_popcount(bitSize) == 1, "PTY_u1 should have been lowered?");
102     if (isLoad) {
103         DEBUG_ASSERT((memOrd == AArch64isa::kMoNone) || (memOrd == AArch64isa::kMoAcquire) ||
104                          (memOrd == AArch64isa::kMoAcquireRcpc) || (memOrd == AArch64isa::kMoLoacquire),
105                      "unknown Memory Order");
106     } else {
107         DEBUG_ASSERT((memOrd == AArch64isa::kMoNone) || (memOrd == AArch64isa::kMoRelease) ||
108                          (memOrd == AArch64isa::kMoLorelease),
109                      "unknown Memory Order");
110     }
111 
112     /* __builtin_ffs(x) returns: 0 -> 0, 1 -> 1, 2 -> 2, 4 -> 3, 8 -> 4 */
113     if ((IsPrimitiveInteger(primType) || primType == PTY_agg) && !IsPrimitiveVector(primType)) {
114         MOperator(*table)[kIntByteSizeDimension];
115         if (isLoad) {
116             table = (memOrd == AArch64isa::kMoAcquire) ? ldIsAcq : ldIs;
117         } else {
118             table = (memOrd == AArch64isa::kMoRelease) ? stIsRel : stIs;
119         }
120 
121         int32 signedUnsigned = IsUnsignedInteger(primType) ? 0 : 1;
122         if (primType == PTY_agg) {
123             CHECK_FATAL(bitSize >= k8BitSize, " unexpect agg size");
124             bitSize = static_cast<uint32>(RoundUp(bitSize, k8BitSize));
125             DEBUG_ASSERT((bitSize & (bitSize - 1)) == 0, "bitlen error");
126         }
127 
128         /* __builtin_ffs(x) returns: 8 -> 4, 16 -> 5, 32 -> 6, 64 -> 7 */
129         if (primType == PTY_i128 || primType == PTY_u128) {
130             bitSize = k64BitSize;
131         }
132         uint32 size = static_cast<uint32>(__builtin_ffs(static_cast<int32>(bitSize))) - k4BitSize;
133         DEBUG_ASSERT(size <= 3, "wrong bitSize"); // size must <= 3
134         return table[signedUnsigned][size];
135     } else {
136         MOperator *table = nullptr;
137         if (isLoad) {
138             table = (memOrd == AArch64isa::kMoAcquire) ? ldFsAcq : ldFs;
139         } else {
140             table = (memOrd == AArch64isa::kMoRelease) ? stFsRel : stFs;
141         }
142 
143         /* __builtin_ffs(x) returns: 32 -> 6, 64 -> 7, 128 -> 8 */
144         uint32 size = static_cast<uint32>(__builtin_ffs(static_cast<int32>(bitSize))) - k6BitSize;
145         DEBUG_ASSERT(size <= k2BitSize, "size must be 0 to 2");
146         return table[size];
147     }
148 }
149 }  // namespace
150 
IsBlkassignForPush(const BlkassignoffNode & bNode)151 bool IsBlkassignForPush(const BlkassignoffNode &bNode)
152 {
153     BaseNode *dest = bNode.Opnd(0);
154     bool spBased = false;
155     if (dest->GetOpCode() == OP_regread) {
156         RegreadNode &node = static_cast<RegreadNode &>(*dest);
157         if (-node.GetRegIdx() == kSregSp) {
158             spBased = true;
159         }
160     }
161     return spBased;
162 }
163 
GetOrCreateResOperand(const BaseNode & parent,PrimType primType)164 RegOperand &AArch64CGFunc::GetOrCreateResOperand(const BaseNode &parent, PrimType primType)
165 {
166     RegOperand *resOpnd = nullptr;
167     if (parent.GetOpCode() == OP_regassign) {
168         auto &regAssignNode = static_cast<const RegassignNode &>(parent);
169         PregIdx pregIdx = regAssignNode.GetRegIdx();
170         if (IsSpecialPseudoRegister(pregIdx)) {
171             /* if it is one of special registers */
172             resOpnd = &GetOrCreateSpecialRegisterOperand(-pregIdx, primType);
173         } else {
174             resOpnd = &GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx));
175         }
176     } else {
177         resOpnd = &CreateRegisterOperandOfType(primType);
178     }
179     return *resOpnd;
180 }
181 
PickLdInsn(uint32 bitSize,PrimType primType,AArch64isa::MemoryOrdering memOrd) const182 MOperator AArch64CGFunc::PickLdInsn(uint32 bitSize, PrimType primType, AArch64isa::MemoryOrdering memOrd) const
183 {
184     return PickLdStInsn(true, bitSize, primType, memOrd);
185 }
186 
PickStInsn(uint32 bitSize,PrimType primType,AArch64isa::MemoryOrdering memOrd) const187 MOperator AArch64CGFunc::PickStInsn(uint32 bitSize, PrimType primType, AArch64isa::MemoryOrdering memOrd) const
188 {
189     return PickLdStInsn(false, bitSize, primType, memOrd);
190 }
191 
PickExtInsn(PrimType dtype,PrimType stype) const192 MOperator AArch64CGFunc::PickExtInsn(PrimType dtype, PrimType stype) const
193 {
194     int32 sBitSize = static_cast<int32>(GetPrimTypeBitSize(stype));
195     int32 dBitSize = static_cast<int32>(GetPrimTypeBitSize(dtype));
196     /* __builtin_ffs(x) returns: 0 -> 0, 1 -> 1, 2 -> 2, 4 -> 3, 8 -> 4 */
197     if (IsPrimitiveInteger(stype) && IsPrimitiveInteger(dtype)) {
198         MOperator(*table)[kIntByteSizeDimension];
199         table = IsUnsignedInteger(stype) ? uextIs : extIs;
200         if (stype == PTY_i128 || stype == PTY_u128) {
201             sBitSize = static_cast<int32>(k64BitSize);
202         }
203         /* __builtin_ffs(x) returns: 8 -> 4, 16 -> 5, 32 -> 6, 64 -> 7 */
204         uint32 row = static_cast<uint32>(__builtin_ffs(sBitSize)) - k4BitSize;
205         DEBUG_ASSERT(row <= k3BitSize, "wrong bitSize");
206         if (dtype == PTY_i128 || dtype == PTY_u128) {
207             dBitSize = static_cast<int32>(k64BitSize);
208         }
209         uint32 col = static_cast<uint32>(__builtin_ffs(dBitSize)) - k4BitSize;
210         DEBUG_ASSERT(col <= k3BitSize, "wrong bitSize");
211         return table[row][col];
212     }
213     CHECK_FATAL(0, "extend not primitive integer");
214     return MOP_undef;
215 }
216 
PickMovBetweenRegs(PrimType destType,PrimType srcType) const217 MOperator AArch64CGFunc::PickMovBetweenRegs(PrimType destType, PrimType srcType) const
218 {
219     if (IsPrimitiveVector(destType) && IsPrimitiveVector(srcType)) {
220         return GetPrimTypeSize(srcType) == k8ByteSize ? MOP_vmovuu : MOP_vmovvv;
221     }
222     if (IsPrimitiveInteger(destType) && IsPrimitiveInteger(srcType)) {
223         return GetPrimTypeSize(srcType) <= k4ByteSize ? MOP_wmovrr : MOP_xmovrr;
224     }
225     if (IsPrimitiveFloat(destType) && IsPrimitiveFloat(srcType)) {
226         return GetPrimTypeSize(srcType) <= k4ByteSize ? MOP_xvmovs : MOP_xvmovd;
227     }
228     if (IsPrimitiveInteger(destType) && IsPrimitiveFloat(srcType)) {
229         return GetPrimTypeSize(srcType) <= k4ByteSize ? MOP_xvmovrs : MOP_xvmovrd;
230     }
231     if (IsPrimitiveFloat(destType) && IsPrimitiveInteger(srcType)) {
232         return GetPrimTypeSize(srcType) <= k4ByteSize ? MOP_xvmovsr : MOP_xvmovdr;
233     }
234     if (IsPrimitiveInteger(destType) && IsPrimitiveVector(srcType)) {
235         return GetPrimTypeSize(srcType) == k8ByteSize
236                    ? MOP_vwmovru
237                    : GetPrimTypeSize(destType) <= k4ByteSize ? MOP_vwmovrv : MOP_vxmovrv;
238     }
239     CHECK_FATAL(false, "unexpected operand primtype for mov");
240     return MOP_undef;
241 }
242 
PickMovInsn(const RegOperand & lhs,const RegOperand & rhs) const243 MOperator AArch64CGFunc::PickMovInsn(const RegOperand &lhs, const RegOperand &rhs) const
244 {
245     CHECK_FATAL(lhs.GetRegisterType() == rhs.GetRegisterType(), "PickMovInsn: unequal kind NYI");
246     CHECK_FATAL(lhs.GetSize() == rhs.GetSize(), "PickMovInsn: unequal size NYI");
247     DEBUG_ASSERT(((lhs.GetSize() < k64BitSize) || (lhs.GetRegisterType() == kRegTyFloat)),
248                  "should split the 64 bits or more mov");
249     if (lhs.GetRegisterType() == kRegTyInt) {
250         return MOP_wmovrr;
251     }
252     if (lhs.GetRegisterType() == kRegTyFloat) {
253         return (lhs.GetSize() <= k32BitSize) ? MOP_xvmovs : MOP_xvmovd;
254     }
255     DEBUG_ASSERT(false, "PickMovInsn: kind NYI");
256     return MOP_undef;
257 }
258 
SelectLoadAcquire(Operand & dest,PrimType dtype,Operand & src,PrimType stype,AArch64isa::MemoryOrdering memOrd,bool isDirect)259 void AArch64CGFunc::SelectLoadAcquire(Operand &dest, PrimType dtype, Operand &src, PrimType stype,
260                                       AArch64isa::MemoryOrdering memOrd, bool isDirect)
261 {
262     DEBUG_ASSERT(src.GetKind() == Operand::kOpdMem, "Just checking");
263     DEBUG_ASSERT(memOrd != AArch64isa::kMoNone, "Just checking");
264 
265     uint32 ssize = isDirect ? src.GetSize() : GetPrimTypeBitSize(dtype);
266     uint32 dsize = GetPrimTypeBitSize(dtype);
267     MOperator mOp = PickLdInsn(ssize, stype, memOrd);
268 
269     Operand *newSrc = &src;
270     auto &memOpnd = static_cast<MemOperand &>(src);
271     OfstOperand *immOpnd = memOpnd.GetOffsetImmediate();
272     int32 offset = static_cast<int32>(immOpnd->GetOffsetValue());
273     RegOperand *origBaseReg = memOpnd.GetBaseRegister();
274     if (offset != 0) {
275         RegOperand &resOpnd = CreateRegisterOperandOfType(PTY_i64);
276         DEBUG_ASSERT(origBaseReg != nullptr, "nullptr check");
277         SelectAdd(resOpnd, *origBaseReg, *immOpnd, PTY_i64);
278         newSrc = &CreateReplacementMemOperand(ssize, resOpnd, 0);
279     }
280 
281     std::string key;
282     if (isDirect && GetCG()->GenerateVerboseCG()) {
283         key = GenerateMemOpndVerbose(src);
284     }
285 
286     /* Check if the right load-acquire instruction is available. */
287     if (mOp != MOP_undef) {
288         Insn &insn = GetInsnBuilder()->BuildInsn(mOp, dest, *newSrc);
289         if (isDirect && GetCG()->GenerateVerboseCG()) {
290             insn.SetComment(key);
291         }
292         GetCurBB()->AppendInsn(insn);
293     } else {
294         if (IsPrimitiveFloat(stype)) {
295             /* Uses signed integer version ldar followed by a floating-point move(fmov).  */
296             DEBUG_ASSERT(stype == dtype, "Just checking");
297             PrimType itype = (stype == PTY_f32) ? PTY_i32 : PTY_i64;
298             RegOperand &regOpnd = CreateRegisterOperandOfType(itype);
299             Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(ssize, itype, memOrd), regOpnd, *newSrc);
300             if (isDirect && GetCG()->GenerateVerboseCG()) {
301                 insn.SetComment(key);
302             }
303             GetCurBB()->AppendInsn(insn);
304             mOp = (stype == PTY_f32) ? MOP_xvmovsr : MOP_xvmovdr;
305             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, dest, regOpnd));
306         } else {
307             /* Use unsigned version ldarb/ldarh followed by a sign-extension instruction(sxtb/sxth).  */
308             DEBUG_ASSERT((ssize == k8BitSize) || (ssize == k16BitSize), "Just checking");
309             PrimType utype = (ssize == k8BitSize) ? PTY_u8 : PTY_u16;
310             Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(ssize, utype, memOrd), dest, *newSrc);
311             if (isDirect && GetCG()->GenerateVerboseCG()) {
312                 insn.SetComment(key);
313             }
314             GetCurBB()->AppendInsn(insn);
315             mOp = ((dsize == k32BitSize) ? ((ssize == k8BitSize) ? MOP_xsxtb32 : MOP_xsxth32)
316                                          : ((ssize == k8BitSize) ? MOP_xsxtb64 : MOP_xsxth64));
317             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, dest, dest));
318         }
319     }
320 }
321 
SelectStoreRelease(Operand & dest,PrimType dtype,Operand & src,PrimType stype,AArch64isa::MemoryOrdering memOrd,bool isDirect)322 void AArch64CGFunc::SelectStoreRelease(Operand &dest, PrimType dtype, Operand &src, PrimType stype,
323                                        AArch64isa::MemoryOrdering memOrd, bool isDirect)
324 {
325     DEBUG_ASSERT(dest.GetKind() == Operand::kOpdMem, "Just checking");
326 
327     uint32 dsize = isDirect ? dest.GetSize() : GetPrimTypeBitSize(stype);
328     MOperator mOp = PickStInsn(dsize, stype, memOrd);
329 
330     Operand *newDest = &dest;
331     MemOperand *memOpnd = static_cast<MemOperand *>(&dest);
332     OfstOperand *immOpnd = memOpnd->GetOffsetImmediate();
333     int32 offset = static_cast<int32>(immOpnd->GetOffsetValue());
334     RegOperand *origBaseReg = memOpnd->GetBaseRegister();
335     if (offset != 0) {
336         RegOperand &resOpnd = CreateRegisterOperandOfType(PTY_i64);
337         DEBUG_ASSERT(origBaseReg != nullptr, "nullptr check");
338         SelectAdd(resOpnd, *origBaseReg, *immOpnd, PTY_i64);
339         newDest = &CreateReplacementMemOperand(dsize, resOpnd, 0);
340     }
341 
342     std::string key;
343     if (isDirect && GetCG()->GenerateVerboseCG()) {
344         key = GenerateMemOpndVerbose(dest);
345     }
346 
347     /* Check if the right store-release instruction is available. */
348     if (mOp != MOP_undef) {
349         Insn &insn = GetInsnBuilder()->BuildInsn(mOp, src, *newDest);
350         if (isDirect && GetCG()->GenerateVerboseCG()) {
351             insn.SetComment(key);
352         }
353         GetCurBB()->AppendInsn(insn);
354     } else {
355         /* Use a floating-point move(fmov) followed by a stlr.  */
356         DEBUG_ASSERT(IsPrimitiveFloat(stype), "must be float type");
357         CHECK_FATAL(stype == dtype, "Just checking");
358         PrimType itype = (stype == PTY_f32) ? PTY_i32 : PTY_i64;
359         RegOperand &regOpnd = CreateRegisterOperandOfType(itype);
360         mOp = (stype == PTY_f32) ? MOP_xvmovrs : MOP_xvmovrd;
361         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, regOpnd, src));
362         Insn &insn = GetInsnBuilder()->BuildInsn(PickStInsn(dsize, itype, memOrd), regOpnd, *newDest);
363         if (isDirect && GetCG()->GenerateVerboseCG()) {
364             insn.SetComment(key);
365         }
366         GetCurBB()->AppendInsn(insn);
367     }
368 }
369 
SelectCopyImm(Operand & dest,PrimType dType,ImmOperand & src,PrimType sType)370 void AArch64CGFunc::SelectCopyImm(Operand &dest, PrimType dType, ImmOperand &src, PrimType sType)
371 {
372     if (IsPrimitiveInteger(dType) != IsPrimitiveInteger(sType)) {
373         RegOperand &tempReg = CreateRegisterOperandOfType(sType);
374         SelectCopyImm(tempReg, src, sType);
375         SelectCopy(dest, dType, tempReg, sType);
376     } else {
377         SelectCopyImm(dest, src, sType);
378     }
379 }
380 
SelectCopyImm(Operand & dest,ImmOperand & src,PrimType dtype)381 void AArch64CGFunc::SelectCopyImm(Operand &dest, ImmOperand &src, PrimType dtype)
382 {
383     uint32 dsize = GetPrimTypeBitSize(dtype);
384     DEBUG_ASSERT(IsPrimitiveInteger(dtype), "The type of destination operand must be Integer");
385     DEBUG_ASSERT(((dsize == k8BitSize) || (dsize == k16BitSize) || (dsize == k32BitSize) || (dsize == k64BitSize)),
386                  "The destination operand must be >= 8-bit");
387     if (src.IsSingleInstructionMovable()) {
388         MOperator mOp = (dsize == k32BitSize) ? MOP_wmovri32 : MOP_xmovri64;
389         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, dest, src));
390         return;
391     }
392     uint64 srcVal = static_cast<uint64>(src.GetValue());
393     /* using mov/movk to load the immediate value */
394     if (dsize == k8BitSize) {
395         /* compute lower 8 bits value */
396         if (dtype == PTY_u8) {
397             /* zero extend */
398             srcVal = (srcVal << k56BitSize) >> k56BitSize;
399             dtype = PTY_u16;
400         } else {
401             /* sign extend */
402             srcVal = (static_cast<int64>(srcVal) << k56BitSize) >> k56BitSize;
403             dtype = PTY_i16;
404         }
405         dsize = k16BitSize;
406     }
407     if (dsize == k16BitSize) {
408         if (dtype == PTY_u16) {
409             /* check lower 16 bits and higher 16 bits respectively */
410             DEBUG_ASSERT((srcVal & 0x0000FFFFULL) != 0, "unexpected value");
411             DEBUG_ASSERT(((srcVal >> k16BitSize) & 0x0000FFFFULL) == 0, "unexpected value");
412             DEBUG_ASSERT((srcVal & 0x0000FFFFULL) != 0xFFFFULL, "unexpected value");
413             /* create an imm opereand which represents lower 16 bits of the immediate */
414             ImmOperand &srcLower = CreateImmOperand(static_cast<int64>(srcVal & 0x0000FFFFULL), k16BitSize, false);
415             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wmovri32, dest, srcLower));
416             return;
417         } else {
418             /* sign extend and let `dsize == 32` case take care of it */
419             srcVal = (static_cast<int64>(srcVal) << k48BitSize) >> k48BitSize;
420             dsize = k32BitSize;
421         }
422     }
423     if (dsize == k32BitSize) {
424         /* check lower 16 bits and higher 16 bits respectively */
425         DEBUG_ASSERT((srcVal & 0x0000FFFFULL) != 0, "unexpected val");
426         DEBUG_ASSERT(((srcVal >> k16BitSize) & 0x0000FFFFULL) != 0, "unexpected val");
427         DEBUG_ASSERT((srcVal & 0x0000FFFFULL) != 0xFFFFULL, "unexpected val");
428         DEBUG_ASSERT(((srcVal >> k16BitSize) & 0x0000FFFFULL) != 0xFFFFULL, "unexpected val");
429         /* create an imm opereand which represents lower 16 bits of the immediate */
430         ImmOperand &srcLower = CreateImmOperand(static_cast<int64>(srcVal & 0x0000FFFFULL), k16BitSize, false);
431         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wmovri32, dest, srcLower));
432         /* create an imm opereand which represents upper 16 bits of the immediate */
433         ImmOperand &srcUpper =
434             CreateImmOperand(static_cast<int64>((srcVal >> k16BitSize) & 0x0000FFFFULL), k16BitSize, false);
435         BitShiftOperand *lslOpnd = GetLogicalShiftLeftOperand(k16BitSize, false);
436         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wmovkri16, dest, srcUpper, *lslOpnd));
437     } else {
438         /*
439          * partition it into 4 16-bit chunks
440          * if more 0's than 0xFFFF's, use movz as the initial instruction.
441          * otherwise, movn.
442          */
443         bool useMovz = BetterUseMOVZ(srcVal);
444         bool useMovk = false;
445         /* get lower 32 bits of the immediate */
446         uint64 chunkLval = srcVal & 0xFFFFFFFFULL;
447         /* get upper 32 bits of the immediate */
448         uint64 chunkHval = (srcVal >> k32BitSize) & 0xFFFFFFFFULL;
449         int32 maxLoopTime = 4;
450 
451         if (chunkLval == chunkHval) {
452             /* compute lower 32 bits, and then copy to higher 32 bits, so only 2 chunks need be processed */
453             maxLoopTime = 2;
454         }
455 
456         uint64 sa = 0;
457 
458         for (int64 i = 0; i < maxLoopTime; ++i, sa += k16BitSize) {
459             /* create an imm opereand which represents the i-th 16-bit chunk of the immediate */
460             uint64 chunkVal = (srcVal >> (static_cast<uint64>(sa))) & 0x0000FFFFULL;
461             if (useMovz ? (chunkVal == 0) : (chunkVal == 0x0000FFFFULL)) {
462                 continue;
463             }
464             ImmOperand &src16 = CreateImmOperand(static_cast<int64>(chunkVal), k16BitSize, false);
465             BitShiftOperand *lslOpnd = GetLogicalShiftLeftOperand(sa, true);
466             if (!useMovk) {
467                 /* use movz or movn */
468                 if (!useMovz) {
469                     src16.BitwiseNegate();
470                 }
471                 GetCurBB()->AppendInsn(
472                     GetInsnBuilder()->BuildInsn(useMovz ? MOP_xmovzri16 : MOP_xmovnri16, dest, src16, *lslOpnd));
473                 useMovk = true;
474             } else {
475                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xmovkri16, dest, src16, *lslOpnd));
476             }
477         }
478 
479         if (maxLoopTime == 2) { /* as described above, only 2 chunks need be processed */
480             /* copy lower 32 bits to higher 32 bits */
481             ImmOperand &immOpnd = CreateImmOperand(k32BitSize, k8BitSize, false);
482             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xbfirri6i6, dest, dest, immOpnd, immOpnd));
483         }
484     }
485 }
486 
GenerateMemOpndVerbose(const Operand & src) const487 std::string AArch64CGFunc::GenerateMemOpndVerbose(const Operand &src) const
488 {
489     DEBUG_ASSERT(src.GetKind() == Operand::kOpdMem, "Just checking");
490     const MIRSymbol *symSecond = static_cast<const MemOperand *>(&src)->GetSymbol();
491     if (symSecond != nullptr) {
492         std::string key;
493         MIRStorageClass sc = symSecond->GetStorageClass();
494         if (sc == kScFormal) {
495             key = "param: ";
496         } else if (sc == kScAuto) {
497             key = "local var: ";
498         } else {
499             key = "global: ";
500         }
501         key += symSecond->GetName();
502         return key;
503     }
504     return "";
505 }
506 
SelectCopyMemOpnd(Operand & dest,PrimType dtype,uint32 dsize,Operand & src,PrimType stype)507 void AArch64CGFunc::SelectCopyMemOpnd(Operand &dest, PrimType dtype, uint32 dsize, Operand &src, PrimType stype)
508 {
509     AArch64isa::MemoryOrdering memOrd = AArch64isa::kMoNone;
510     const MIRSymbol *sym = static_cast<MemOperand *>(&src)->GetSymbol();
511     if ((sym != nullptr) && (sym->GetStorageClass() == kScGlobal) && sym->GetAttr(ATTR_memory_order_acquire)) {
512         memOrd = AArch64isa::kMoAcquire;
513     }
514 
515     if (memOrd != AArch64isa::kMoNone) {
516         AArch64CGFunc::SelectLoadAcquire(dest, dtype, src, stype, memOrd, true);
517         return;
518     }
519     Insn *insn = nullptr;
520     uint32 ssize = src.GetSize();
521     PrimType regTy = PTY_void;
522     RegOperand *loadReg = nullptr;
523     MOperator mop = MOP_undef;
524     if (IsPrimitiveFloat(stype) || IsPrimitiveVector(stype)) {
525         CHECK_FATAL(dsize == ssize, "dsize %u expect equals ssize %u", dtype, ssize);
526         insn = &GetInsnBuilder()->BuildInsn(PickLdInsn(ssize, stype), dest, src);
527     } else {
528         if (stype == PTY_agg && dtype == PTY_agg) {
529             mop = MOP_undef;
530         } else {
531             mop = PickExtInsn(dtype, stype);
532         }
533         if (ssize == (GetPrimTypeSize(dtype) * kBitsPerByte) || mop == MOP_undef) {
534             insn = &GetInsnBuilder()->BuildInsn(PickLdInsn(ssize, stype), dest, src);
535         } else {
536             regTy = dsize == k64BitSize ? dtype : PTY_i32;
537             loadReg = &CreateRegisterOperandOfType(regTy);
538             insn = &GetInsnBuilder()->BuildInsn(PickLdInsn(ssize, stype), *loadReg, src);
539         }
540     }
541 
542     if (GetCG()->GenerateVerboseCG()) {
543         insn->SetComment(GenerateMemOpndVerbose(src));
544     }
545 
546     GetCurBB()->AppendInsn(*insn);
547     if (regTy != PTY_void && mop != MOP_undef) {
548         DEBUG_ASSERT(loadReg != nullptr, "loadReg should not be nullptr");
549         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mop, dest, *loadReg));
550     }
551 }
552 
IsImmediateValueInRange(MOperator mOp,int64 immVal,bool is64Bits,bool isIntactIndexed,bool isPostIndexed,bool isPreIndexed) const553 bool AArch64CGFunc::IsImmediateValueInRange(MOperator mOp, int64 immVal, bool is64Bits, bool isIntactIndexed,
554                                             bool isPostIndexed, bool isPreIndexed) const
555 {
556     bool isInRange = false;
557     switch (mOp) {
558         case MOP_xstr:
559         case MOP_wstr:
560             isInRange =
561                 (isIntactIndexed &&
562                  ((!is64Bits && (immVal >= kStrAllLdrAllImmLowerBound) && (immVal <= kStrLdrImm32UpperBound)) ||
563                   (is64Bits && (immVal >= kStrAllLdrAllImmLowerBound) && (immVal <= kStrLdrImm64UpperBound)))) ||
564                 ((isPostIndexed || isPreIndexed) && (immVal >= kStrLdrPerPostLowerBound) &&
565                  (immVal <= kStrLdrPerPostUpperBound));
566             break;
567         case MOP_wstrb:
568             isInRange =
569                 (isIntactIndexed && (immVal >= kStrAllLdrAllImmLowerBound) && (immVal <= kStrbLdrbImmUpperBound)) ||
570                 ((isPostIndexed || isPreIndexed) && (immVal >= kStrLdrPerPostLowerBound) &&
571                  (immVal <= kStrLdrPerPostUpperBound));
572             break;
573         case MOP_wstrh:
574             isInRange =
575                 (isIntactIndexed && (immVal >= kStrAllLdrAllImmLowerBound) && (immVal <= kStrhLdrhImmUpperBound)) ||
576                 ((isPostIndexed || isPreIndexed) && (immVal >= kStrLdrPerPostLowerBound) &&
577                  (immVal <= kStrLdrPerPostUpperBound));
578             break;
579         default:
580             break;
581     }
582     return isInRange;
583 }
584 
IsStoreMop(MOperator mOp) const585 bool AArch64CGFunc::IsStoreMop(MOperator mOp) const
586 {
587     switch (mOp) {
588         case MOP_sstr:
589         case MOP_dstr:
590         case MOP_qstr:
591         case MOP_xstr:
592         case MOP_wstr:
593         case MOP_wstrb:
594         case MOP_wstrh:
595             return true;
596         default:
597             return false;
598     }
599 }
600 
SplitMovImmOpndInstruction(int64 immVal,RegOperand & destReg,Insn * curInsn)601 void AArch64CGFunc::SplitMovImmOpndInstruction(int64 immVal, RegOperand &destReg, Insn *curInsn)
602 {
603     bool useMovz = BetterUseMOVZ(immVal);
604     bool useMovk = false;
605     /* get lower 32 bits of the immediate */
606     uint64 chunkLval = static_cast<uint64>(immVal) & 0xFFFFFFFFULL;
607     /* get upper 32 bits of the immediate */
608     uint64 chunkHval = (static_cast<uint64>(immVal) >> k32BitSize) & 0xFFFFFFFFULL;
609     int32 maxLoopTime = 4;
610 
611     if (chunkLval == chunkHval) {
612         /* compute lower 32 bits, and then copy to higher 32 bits, so only 2 chunks need be processed */
613         maxLoopTime = 2;
614     }
615 
616     uint64 sa = 0;
617     auto *bb = (curInsn != nullptr) ? curInsn->GetBB() : GetCurBB();
618     for (int64 i = 0; i < maxLoopTime; ++i, sa += k16BitSize) {
619         /* create an imm opereand which represents the i-th 16-bit chunk of the immediate */
620         uint64 chunkVal = (static_cast<uint64>(immVal) >> sa) & 0x0000FFFFULL;
621         if (useMovz ? (chunkVal == 0) : (chunkVal == 0x0000FFFFULL)) {
622             continue;
623         }
624         ImmOperand &src16 = CreateImmOperand(static_cast<int64>(chunkVal), k16BitSize, false);
625         BitShiftOperand *lslOpnd = GetLogicalShiftLeftOperand(sa, true);
626         Insn *newInsn = nullptr;
627         if (!useMovk) {
628             /* use movz or movn */
629             if (!useMovz) {
630                 src16.BitwiseNegate();
631             }
632             MOperator mOpCode = useMovz ? MOP_xmovzri16 : MOP_xmovnri16;
633             newInsn = &GetInsnBuilder()->BuildInsn(mOpCode, destReg, src16, *lslOpnd);
634             useMovk = true;
635         } else {
636             newInsn = &GetInsnBuilder()->BuildInsn(MOP_xmovkri16, destReg, src16, *lslOpnd);
637         }
638         if (curInsn != nullptr) {
639             bb->InsertInsnBefore(*curInsn, *newInsn);
640         } else {
641             bb->AppendInsn(*newInsn);
642         }
643     }
644 
645     if (maxLoopTime == 2) { // compute lower 32 bits, and copy to higher 32 bits, so only 2 chunks need be processed
646         /* copy lower 32 bits to higher 32 bits */
647         ImmOperand &immOpnd = CreateImmOperand(k32BitSize, k8BitSize, false);
648         Insn &insn = GetInsnBuilder()->BuildInsn(MOP_xbfirri6i6, destReg, destReg, immOpnd, immOpnd);
649         if (curInsn != nullptr) {
650             bb->InsertInsnBefore(*curInsn, insn);
651         } else {
652             bb->AppendInsn(insn);
653         }
654     }
655 }
656 
SelectCopyRegOpnd(Operand & dest,PrimType dtype,Operand::OperandType opndType,uint32 dsize,Operand & src,PrimType stype)657 void AArch64CGFunc::SelectCopyRegOpnd(Operand &dest, PrimType dtype, Operand::OperandType opndType, uint32 dsize,
658                                       Operand &src, PrimType stype)
659 {
660     if (opndType != Operand::kOpdMem) {
661         if (!CGOptions::IsArm64ilp32()) {
662             DEBUG_ASSERT(stype != PTY_a32, "");
663         }
664         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickMovBetweenRegs(dtype, stype), dest, src));
665         return;
666     }
667     AArch64isa::MemoryOrdering memOrd = AArch64isa::kMoNone;
668     const MIRSymbol *sym = static_cast<MemOperand *>(&dest)->GetSymbol();
669     if ((sym != nullptr) && (sym->GetStorageClass() == kScGlobal) && sym->GetAttr(ATTR_memory_order_release)) {
670         memOrd = AArch64isa::kMoRelease;
671     }
672 
673     if (memOrd != AArch64isa::kMoNone) {
674         AArch64CGFunc::SelectStoreRelease(dest, dtype, src, stype, memOrd, true);
675         return;
676     }
677 
678     bool is64Bits = (dest.GetSize() == k64BitSize) ? true : false;
679     MOperator strMop = PickStInsn(dsize, stype);
680     if (!dest.IsMemoryAccessOperand()) {
681         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(strMop, src, dest));
682         return;
683     }
684 
685     MemOperand *memOpnd = static_cast<MemOperand *>(&dest);
686     DEBUG_ASSERT(memOpnd != nullptr, "memOpnd should not be nullptr");
687     if (memOpnd->GetAddrMode() == MemOperand::kAddrModeLo12Li) {
688         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(strMop, src, dest));
689         return;
690     }
691     if (memOpnd->GetOffsetOperand() == nullptr) {
692         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(strMop, src, dest));
693         return;
694     }
695     ImmOperand *immOpnd = static_cast<ImmOperand *>(memOpnd->GetOffsetOperand());
696     DEBUG_ASSERT(immOpnd != nullptr, "immOpnd should not be nullptr");
697     int64 immVal = immOpnd->GetValue();
698     bool isIntactIndexed = memOpnd->IsIntactIndexed();
699     bool isPostIndexed = memOpnd->IsPostIndexed();
700     bool isPreIndexed = memOpnd->IsPreIndexed();
701     DEBUG_ASSERT(!isPostIndexed, "memOpnd should not be post-index type");
702     DEBUG_ASSERT(!isPreIndexed, "memOpnd should not be pre-index type");
703     bool isInRange = false;
704     if (!GetMirModule().IsCModule()) {
705         isInRange = IsImmediateValueInRange(strMop, immVal, is64Bits, isIntactIndexed, isPostIndexed, isPreIndexed);
706     } else {
707         isInRange = IsOperandImmValid(strMop, memOpnd, kInsnSecondOpnd);
708     }
709     bool isMopStr = IsStoreMop(strMop);
710     if (isInRange || !isMopStr) {
711         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(strMop, src, dest));
712         return;
713     }
714     DEBUG_ASSERT(memOpnd->GetBaseRegister() != nullptr, "nullptr check");
715     if (isIntactIndexed) {
716         memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, dsize);
717         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(strMop, src, *memOpnd));
718     } else if (isPostIndexed || isPreIndexed) {
719         RegOperand &reg = CreateRegisterOperandOfType(PTY_i64);
720         MOperator mopMov = MOP_xmovri64;
721         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopMov, reg, *immOpnd));
722         MOperator mopAdd = MOP_xaddrrr;
723         MemOperand &newDest =
724             GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, GetPrimTypeBitSize(dtype), memOpnd->GetBaseRegister(), nullptr,
725                                &GetOrCreateOfstOpnd(0, k32BitSize), nullptr);
726         Insn &insn1 = GetInsnBuilder()->BuildInsn(strMop, src, newDest);
727         Insn &insn2 = GetInsnBuilder()->BuildInsn(mopAdd, *newDest.GetBaseRegister(), *newDest.GetBaseRegister(), reg);
728         if (isPostIndexed) {
729             GetCurBB()->AppendInsn(insn1);
730             GetCurBB()->AppendInsn(insn2);
731         } else {
732             /* isPreIndexed */
733             GetCurBB()->AppendInsn(insn2);
734             GetCurBB()->AppendInsn(insn1);
735         }
736     }
737 }
738 
SelectCopy(Operand & dest,PrimType dtype,Operand & src,PrimType stype)739 void AArch64CGFunc::SelectCopy(Operand &dest, PrimType dtype, Operand &src, PrimType stype)
740 {
741     DEBUG_ASSERT(dest.IsRegister() || dest.IsMemoryAccessOperand(), "");
742     uint32 dsize = GetPrimTypeBitSize(dtype);
743     if (dest.IsRegister()) {
744         dsize = dest.GetSize();
745     }
746     Operand::OperandType opnd0Type = dest.GetKind();
747     Operand::OperandType opnd1Type = src.GetKind();
748     DEBUG_ASSERT(((dsize >= src.GetSize()) || (opnd0Type == Operand::kOpdRegister) || (opnd0Type == Operand::kOpdMem)),
749                  "NYI");
750     DEBUG_ASSERT(((opnd0Type == Operand::kOpdRegister) || (src.GetKind() == Operand::kOpdRegister)),
751                  "either src or dest should be register");
752 
753     switch (opnd1Type) {
754         case Operand::kOpdMem:
755             SelectCopyMemOpnd(dest, dtype, dsize, src, stype);
756             break;
757         case Operand::kOpdOffset:
758         case Operand::kOpdImmediate:
759             SelectCopyImm(dest, dtype, static_cast<ImmOperand &>(src), stype);
760             break;
761         case Operand::kOpdFPImmediate:
762             CHECK_FATAL(static_cast<ImmOperand &>(src).GetValue() == 0, "NIY");
763             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn((dsize == k32BitSize) ? MOP_xvmovsr : MOP_xvmovdr, dest,
764                                                                GetZeroOpnd(dsize)));
765             break;
766         case Operand::kOpdRegister: {
767             if (opnd0Type == Operand::kOpdRegister && IsPrimitiveVector(stype)) {
768                 /* check vector reg to vector reg move */
769                 CHECK_FATAL(IsPrimitiveVector(dtype), "invalid vectreg to vectreg move");
770                 MOperator mop = (dsize <= k64BitSize) ? MOP_vmovuu : MOP_vmovvv;
771                 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
772                 vInsn.AddOpndChain(dest).AddOpndChain(src);
773                 auto *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(dsize >> k3ByteSize, k8BitSize);
774                 auto *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(dsize >> k3ByteSize, k8BitSize);
775                 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpecSrc);
776                 GetCurBB()->AppendInsn(vInsn);
777                 break;
778             }
779             if (dest.IsRegister()) {
780                 RegOperand &desReg = static_cast<RegOperand &>(dest);
781                 RegOperand &srcReg = static_cast<RegOperand &>(src);
782                 if (desReg.GetRegisterNumber() == srcReg.GetRegisterNumber()) {
783                     break;
784                 }
785             }
786             SelectCopyRegOpnd(dest, dtype, opnd0Type, dsize, src, stype);
787             break;
788         }
789         default:
790             CHECK_FATAL(false, "NYI");
791     }
792 }
793 
794 /* This function copies src to a register, the src can be an imm, mem or a label */
SelectCopy(Operand & src,PrimType stype,PrimType dtype)795 RegOperand &AArch64CGFunc::SelectCopy(Operand &src, PrimType stype, PrimType dtype)
796 {
797     RegOperand &dest = CreateRegisterOperandOfType(dtype);
798     SelectCopy(dest, dtype, src, stype);
799     return dest;
800 }
801 
802 /*
803  * We need to adjust the offset of a stack allocated local variable
804  * if we store FP/SP before any other local variables to save an instruction.
805  * See AArch64CGFunc::OffsetAdjustmentForFPLR() in aarch64_cgfunc.cpp
806  *
807  * That is when we !UsedStpSubPairForCallFrameAllocation().
808  *
809  * Because we need to use the STP/SUB instruction pair to store FP/SP 'after'
810  * local variables when the call frame size is greater that the max offset
811  * value allowed for the STP instruction (we cannot use STP w/ prefix, LDP w/
812  * postfix), if UsedStpSubPairForCallFrameAllocation(), we don't need to
813  * adjust the offsets.
814  */
IsImmediateOffsetOutOfRange(const MemOperand & memOpnd,uint32 bitLen)815 bool AArch64CGFunc::IsImmediateOffsetOutOfRange(const MemOperand &memOpnd, uint32 bitLen)
816 {
817     DEBUG_ASSERT(bitLen >= k8BitSize, "bitlen error");
818     DEBUG_ASSERT(bitLen <= k128BitSize, "bitlen error");
819 
820     if (bitLen >= k8BitSize) {
821         bitLen = static_cast<uint32>(RoundUp(bitLen, k8BitSize));
822     }
823     DEBUG_ASSERT((bitLen & (bitLen - 1)) == 0, "bitlen error");
824 
825     MemOperand::AArch64AddressingMode mode = memOpnd.GetAddrMode();
826     if ((mode == MemOperand::kAddrModeBOi) && memOpnd.IsIntactIndexed()) {
827         int32 offsetValue = static_cast<int32>(memOpnd.GetOffsetImmediate()->GetOffsetValue());
828         if (memOpnd.GetOffsetImmediate()->GetVary() == kUnAdjustVary) {
829             offsetValue +=
830                 static_cast<int32>(static_cast<AArch64MemLayout *>(GetMemlayout())->RealStackFrameSize() + 0xff);
831         }
832         offsetValue += kIntregBytelen << 1; /* Refer to the above comment */
833         return MemOperand::IsPIMMOffsetOutOfRange(offsetValue, bitLen);
834     } else {
835         return false;
836     }
837 }
838 
IsOperandImmValid(MOperator mOp,Operand * o,uint32 opndIdx)839 bool AArch64CGFunc::IsOperandImmValid(MOperator mOp, Operand *o, uint32 opndIdx)
840 {
841     const InsnDesc *md = &AArch64CG::kMd[mOp];
842     auto *opndProp = md->opndMD[opndIdx];
843 
844     Operand::OperandType opndTy = opndProp->GetOperandType();
845     if (opndTy == Operand::kOpdMem) {
846         auto *memOpnd = static_cast<MemOperand *>(o);
847         if (memOpnd->GetAddrMode() == MemOperand::kAddrModeBOrX) {
848             return true;
849         }
850         if (md->IsLoadStorePair() ||
851             (memOpnd->GetAddrMode() == MemOperand::kAddrModeBOi && memOpnd->IsIntactIndexed())) {
852             int64 offsetValue = memOpnd->GetOffsetImmediate()->GetOffsetValue();
853             if (memOpnd->GetOffsetImmediate()->GetVary() == kUnAdjustVary) {
854                 offsetValue += static_cast<AArch64MemLayout *>(GetMemlayout())->RealStackFrameSize() + 0xffL;
855             }
856             return md->IsValidImmOpnd(offsetValue);
857         } else if (memOpnd->GetAddrMode() == MemOperand::kAddrModeLo12Li) {
858             int32 offsetValue = static_cast<int32>(memOpnd->GetOffsetImmediate()->GetOffsetValue());
859             return offsetValue == 0;
860         } else {
861             CHECK_FATAL(!memOpnd->IsIntactIndexed(), "CHECK WHAT?");
862             int32 offsetValue = static_cast<int32>(memOpnd->GetOffsetImmediate()->GetOffsetValue());
863             return (offsetValue <= static_cast<int32>(k256BitSize) && offsetValue >= kNegative256BitSize);
864         }
865     } else if (opndTy == Operand::kOpdImmediate) {
866         return md->IsValidImmOpnd(static_cast<ImmOperand *>(o)->GetValue());
867     }
868     return true;
869 }
870 
CreateReplacementMemOperand(uint32 bitLen,RegOperand & baseReg,int64 offset)871 MemOperand &AArch64CGFunc::CreateReplacementMemOperand(uint32 bitLen, RegOperand &baseReg, int64 offset)
872 {
873     return CreateMemOpnd(baseReg, offset, bitLen);
874 }
875 
CheckIfSplitOffsetWithAdd(const MemOperand & memOpnd,uint32 bitLen) const876 bool AArch64CGFunc::CheckIfSplitOffsetWithAdd(const MemOperand &memOpnd, uint32 bitLen) const
877 {
878     if (memOpnd.GetAddrMode() != MemOperand::kAddrModeBOi || !memOpnd.IsIntactIndexed()) {
879         return false;
880     }
881     OfstOperand *ofstOpnd = memOpnd.GetOffsetImmediate();
882     int32 opndVal = static_cast<int32>(ofstOpnd->GetOffsetValue());
883     int32 maxPimm = memOpnd.GetMaxPIMM(bitLen);
884     int32 q0 = opndVal / maxPimm;
885     int32 addend = q0 * maxPimm;
886     int32 r0 = opndVal - addend;
887     int32 alignment = memOpnd.GetImmediateOffsetAlignment(bitLen);
888     int32 r1 = static_cast<uint32>(r0) & ((1u << static_cast<uint32>(alignment)) - 1);
889     addend = addend + r1;
890     return (addend > 0);
891 }
892 
GetBaseRegForSplit(uint32 baseRegNum)893 RegOperand *AArch64CGFunc::GetBaseRegForSplit(uint32 baseRegNum)
894 {
895     RegOperand *resOpnd = nullptr;
896     if (baseRegNum == AArch64reg::kRinvalid) {
897         resOpnd = &CreateRegisterOperandOfType(PTY_i64);
898     } else if (AArch64isa::IsPhysicalRegister(baseRegNum)) {
899         resOpnd = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(baseRegNum),
900                                                       GetPointerSize() * kBitsPerByte, kRegTyInt);
901     } else {
902         resOpnd = &GetOrCreateVirtualRegisterOperand(baseRegNum);
903     }
904     return resOpnd;
905 }
906 
907 /*
908  * When immediate of str/ldr is over 256bits, it should be aligned according to the reg byte size.
909  * Here we split the offset into (512 * n) and +/-(new Offset) when misaligned, to make sure that
910  * the new offet is always under 256 bits.
911  */
ConstraintOffsetToSafeRegion(uint32 bitLen,const MemOperand & memOpnd)912 MemOperand &AArch64CGFunc::ConstraintOffsetToSafeRegion(uint32 bitLen, const MemOperand &memOpnd)
913 {
914     auto it = hashMemOpndTable.find(memOpnd);
915     if (it != hashMemOpndTable.end()) {
916         hashMemOpndTable.erase(memOpnd);
917     }
918     int32 offsetValue = static_cast<int32>(memOpnd.GetOffsetImmediate()->GetOffsetValue());
919     int32 multiplier = (offsetValue / k512BitSize) + static_cast<int32>(offsetValue % k512BitSize > k256BitSize);
920     int32 addMount = multiplier * k512BitSizeInt;
921     int32 newOffset = offsetValue - addMount;
922     RegOperand *baseReg = memOpnd.GetBaseRegister();
923     ImmOperand &immAddMount = CreateImmOperand(addMount, k64BitSize, true);
924     if (memOpnd.GetOffsetImmediate()->GetVary() == kUnAdjustVary) {
925         immAddMount.SetVary(kUnAdjustVary);
926     }
927 
928     RegOperand *resOpnd = GetBaseRegForSplit(kRinvalid);
929     SelectAdd(*resOpnd, *baseReg, immAddMount, PTY_i64);
930     MemOperand &newMemOpnd = CreateReplacementMemOperand(bitLen, *resOpnd, newOffset);
931     newMemOpnd.SetStackMem(memOpnd.IsStackMem());
932     return newMemOpnd;
933 }
934 
SplitAndGetRemained(const MemOperand & memOpnd,uint32 bitLen,RegOperand * resOpnd,int64 ofstVal,bool isDest,Insn * insn,bool forPair)935 ImmOperand &AArch64CGFunc::SplitAndGetRemained(const MemOperand &memOpnd, uint32 bitLen, RegOperand *resOpnd,
936                                                int64 ofstVal, bool isDest, Insn *insn, bool forPair)
937 {
938     auto it = hashMemOpndTable.find(memOpnd);
939     if (it != hashMemOpndTable.end()) {
940         hashMemOpndTable.erase(memOpnd);
941     }
942     /*
943      * opndVal == Q0 * 32760(16380) + R0
944      * R0 == Q1 * 8(4) + R1
945      * ADDEND == Q0 * 32760(16380) + R1
946      * NEW_OFFSET = Q1 * 8(4)
947      * we want to generate two instructions:
948      * ADD TEMP_REG, X29, ADDEND
949      * LDR/STR TEMP_REG, [ TEMP_REG, #NEW_OFFSET ]
950      */
951     int32 maxPimm = 0;
952     if (!forPair) {
953         maxPimm = MemOperand::GetMaxPIMM(bitLen);
954     } else {
955         maxPimm = MemOperand::GetMaxPairPIMM(bitLen);
956     }
957     DEBUG_ASSERT(maxPimm != 0, "get max pimm failed");
958 
959     int64 q0 = ofstVal / maxPimm + (ofstVal < 0 ? -1 : 0);
960     int64 addend = q0 * maxPimm;
961     int64 r0 = ofstVal - addend;
962     int64 alignment = MemOperand::GetImmediateOffsetAlignment(bitLen);
963     auto q1 = static_cast<int64>(static_cast<uint64>(r0) >> static_cast<uint64>(alignment));
964     auto r1 = static_cast<int64>(static_cast<uint64>(r0) & ((1u << static_cast<uint64>(alignment)) - 1));
965     auto remained = static_cast<int64>(static_cast<uint64>(q1) << static_cast<uint64>(alignment));
966     addend = addend + r1;
967     if (addend > 0) {
968         int64 suffixClear = 0xfff;
969         if (forPair) {
970             suffixClear = 0xff;
971         }
972         int64 remainedTmp = remained + (addend & suffixClear);
973         if (!MemOperand::IsPIMMOffsetOutOfRange(static_cast<int32>(remainedTmp), bitLen) &&
974             ((static_cast<uint64>(remainedTmp) & ((1u << static_cast<uint64>(alignment)) - 1)) == 0)) {
975             remained = remainedTmp;
976             addend = (addend & ~suffixClear);
977         }
978     }
979     ImmOperand &immAddend = CreateImmOperand(addend, k64BitSize, true);
980     if (memOpnd.GetOffsetImmediate()->GetVary() == kUnAdjustVary) {
981         immAddend.SetVary(kUnAdjustVary);
982     }
983     return immAddend;
984 }
985 
SplitOffsetWithAddInstruction(const MemOperand & memOpnd,uint32 bitLen,uint32 baseRegNum,bool isDest,Insn * insn,bool forPair)986 MemOperand &AArch64CGFunc::SplitOffsetWithAddInstruction(const MemOperand &memOpnd, uint32 bitLen, uint32 baseRegNum,
987                                                          bool isDest, Insn *insn, bool forPair)
988 {
989     DEBUG_ASSERT((memOpnd.GetAddrMode() == MemOperand::kAddrModeBOi), "expect kAddrModeBOi memOpnd");
990     DEBUG_ASSERT(memOpnd.IsIntactIndexed(), "expect intactIndexed memOpnd");
991     OfstOperand *ofstOpnd = memOpnd.GetOffsetImmediate();
992     int64 ofstVal = ofstOpnd->GetOffsetValue();
993     RegOperand *resOpnd = GetBaseRegForSplit(baseRegNum);
994     ImmOperand &immAddend = SplitAndGetRemained(memOpnd, bitLen, resOpnd, ofstVal, isDest, insn, forPair);
995     int64 remained = (ofstVal - immAddend.GetValue());
996     RegOperand *origBaseReg = memOpnd.GetBaseRegister();
997     DEBUG_ASSERT(origBaseReg != nullptr, "nullptr check");
998     if (insn == nullptr) {
999         SelectAdd(*resOpnd, *origBaseReg, immAddend, PTY_i64);
1000     } else {
1001         SelectAddAfterInsn(*resOpnd, *origBaseReg, immAddend, PTY_i64, isDest, *insn);
1002     }
1003     MemOperand &newMemOpnd = CreateReplacementMemOperand(bitLen, *resOpnd, remained);
1004     newMemOpnd.SetStackMem(memOpnd.IsStackMem());
1005     return newMemOpnd;
1006 }
1007 
SelectDassign(DassignNode & stmt,Operand & opnd0)1008 void AArch64CGFunc::SelectDassign(DassignNode &stmt, Operand &opnd0)
1009 {
1010     SelectDassign(stmt.GetStIdx(), stmt.GetFieldID(), stmt.GetRHS()->GetPrimType(), opnd0);
1011 }
1012 
1013 /*
1014  * Used for SelectDassign when do optimization for volatile store, because the stlr instruction only allow
1015  * store to the memory addrress with the register base offset 0.
1016  * STLR <Wt>, [<Xn|SP>{,#0}], 32-bit variant (size = 10)
1017  * STLR <Xt>, [<Xn|SP>{,#0}], 64-bit variant (size = 11)
1018  * So the function do the prehandle of the memory operand to satisify the Store-Release..
1019  */
ExtractNewMemBase(const MemOperand & memOpnd)1020 RegOperand *AArch64CGFunc::ExtractNewMemBase(const MemOperand &memOpnd)
1021 {
1022     const MIRSymbol *sym = memOpnd.GetSymbol();
1023     MemOperand::AArch64AddressingMode mode = memOpnd.GetAddrMode();
1024     if (mode == MemOperand::kAddrModeLiteral) {
1025         return nullptr;
1026     }
1027     RegOperand *baseOpnd = memOpnd.GetBaseRegister();
1028     DEBUG_ASSERT(baseOpnd != nullptr, "nullptr check");
1029     RegOperand &resultOpnd =
1030         CreateRegisterOperandOfType(baseOpnd->GetRegisterType(), baseOpnd->GetSize() / kBitsPerByte);
1031     bool is64Bits = (baseOpnd->GetSize() == k64BitSize);
1032     if (mode == MemOperand::kAddrModeLo12Li) {
1033         StImmOperand &stImm = CreateStImmOperand(*sym, 0, 0);
1034         Insn &addInsn = GetInsnBuilder()->BuildInsn(MOP_xadrpl12, resultOpnd, *baseOpnd, stImm);
1035         addInsn.SetComment("new add insn");
1036         GetCurBB()->AppendInsn(addInsn);
1037     } else if (mode == MemOperand::kAddrModeBOi) {
1038         OfstOperand *offsetOpnd = memOpnd.GetOffsetImmediate();
1039         if (offsetOpnd->GetOffsetValue() != 0) {
1040             MOperator mOp = is64Bits ? MOP_xaddrri12 : MOP_waddrri12;
1041             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resultOpnd, *baseOpnd, *offsetOpnd));
1042         } else {
1043             return baseOpnd;
1044         }
1045     } else {
1046         CHECK_FATAL(mode == MemOperand::kAddrModeBOrX, "unexpect addressing mode.");
1047         RegOperand *regOpnd = static_cast<const MemOperand *>(&memOpnd)->GetIndexRegister();
1048         MOperator mOp = is64Bits ? MOP_xaddrrr : MOP_waddrrr;
1049         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resultOpnd, *baseOpnd, *regOpnd));
1050     }
1051     return &resultOpnd;
1052 }
1053 
1054 /*
1055  * NOTE: I divided SelectDassign so that we can create "virtual" assignments
1056  * when selecting other complex Maple IR instructions. For example, the atomic
1057  * exchange and other intrinsics will need to assign its results to local
1058  * variables. Such Maple IR instructions are pltform-specific (e.g.
1059  * atomic_exchange can be implemented as one single machine intruction on x86_64
1060  * and ARMv8.1, but ARMv8.0 needs an LL/SC loop), therefore they cannot (in
1061  * principle) be lowered at BELowerer or CGLowerer.
1062  */
SelectDassign(StIdx stIdx,FieldID fieldId,PrimType rhsPType,Operand & opnd0)1063 void AArch64CGFunc::SelectDassign(StIdx stIdx, FieldID fieldId, PrimType rhsPType, Operand &opnd0)
1064 {
1065     MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(stIdx);
1066     int32 offset = 0;
1067     bool parmCopy = false;
1068     if (fieldId != 0) {
1069         MIRStructType *structType = static_cast<MIRStructType *>(symbol->GetType());
1070         DEBUG_ASSERT(structType != nullptr, "SelectDassign: non-zero fieldID for non-structure");
1071         offset = GetBecommon().GetFieldOffset(*structType, fieldId).first;
1072         parmCopy = IsParamStructCopy(*symbol);
1073     }
1074     uint32 regSize = GetPrimTypeBitSize(rhsPType);
1075     MIRType *type = symbol->GetType();
1076     Operand &stOpnd = LoadIntoRegister(opnd0, IsPrimitiveInteger(rhsPType) || IsPrimitiveVectorInteger(rhsPType),
1077                                        regSize, IsSignedInteger(type->GetPrimType()));
1078     MOperator mOp = MOP_undef;
1079     if ((type->GetKind() == kTypeStruct) || (type->GetKind() == kTypeUnion)) {
1080         MIRStructType *structType = static_cast<MIRStructType *>(type);
1081         type = structType->GetFieldType(fieldId);
1082     } else if (type->GetKind() == kTypeClass) {
1083         MIRClassType *classType = static_cast<MIRClassType *>(type);
1084         type = classType->GetFieldType(fieldId);
1085     }
1086 
1087     uint32 dataSize = GetPrimTypeBitSize(type->GetPrimType());
1088     if (type->GetPrimType() == PTY_agg) {
1089         dataSize = GetPrimTypeBitSize(PTY_a64);
1090     }
1091     MemOperand *memOpnd = nullptr;
1092     if (parmCopy) {
1093         memOpnd = &LoadStructCopyBase(*symbol, offset, static_cast<int>(dataSize));
1094     } else {
1095         memOpnd = &GetOrCreateMemOpnd(*symbol, offset, dataSize);
1096     }
1097     if ((memOpnd->GetMemVaryType() == kNotVary) && IsImmediateOffsetOutOfRange(*memOpnd, dataSize)) {
1098         memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, dataSize);
1099     }
1100 
1101     /* In bpl mode, a func symbol's type is represented as a MIRFuncType instead of a MIRPtrType (pointing to
1102      * MIRFuncType), so we allow `kTypeFunction` to appear here */
1103     DEBUG_ASSERT(((type->GetKind() == kTypeScalar) || (type->GetKind() == kTypePointer) ||
1104                   (type->GetKind() == kTypeFunction) || (type->GetKind() == kTypeStruct) ||
1105                   (type->GetKind() == kTypeUnion) || (type->GetKind() == kTypeArray)),
1106                  "NYI dassign type");
1107     PrimType ptyp = type->GetPrimType();
1108     if (ptyp == PTY_agg) {
1109         ptyp = PTY_a64;
1110     }
1111 
1112     AArch64isa::MemoryOrdering memOrd = AArch64isa::kMoNone;
1113     if (isVolStore) {
1114         RegOperand *baseOpnd = ExtractNewMemBase(*memOpnd);
1115         if (baseOpnd != nullptr) {
1116             memOpnd = &CreateMemOpnd(*baseOpnd, 0, dataSize);
1117             memOrd = AArch64isa::kMoRelease;
1118             isVolStore = false;
1119         }
1120     }
1121 
1122     memOpnd = memOpnd->IsOffsetMisaligned(dataSize) ? &ConstraintOffsetToSafeRegion(dataSize, *memOpnd) : memOpnd;
1123     if (symbol->GetAsmAttr() != UStrIdx(0) && symbol->GetStorageClass() != kScPstatic &&
1124         symbol->GetStorageClass() != kScFstatic) {
1125         std::string regDesp = GlobalTables::GetUStrTable().GetStringFromStrIdx(symbol->GetAsmAttr());
1126         RegOperand &specifiedOpnd = GetOrCreatePhysicalRegisterOperand(regDesp);
1127         SelectCopy(specifiedOpnd, type->GetPrimType(), opnd0, rhsPType);
1128     } else if (memOrd == AArch64isa::kMoNone) {
1129         mOp = PickStInsn(GetPrimTypeBitSize(ptyp), ptyp);
1130         Insn &insn = GetInsnBuilder()->BuildInsn(mOp, stOpnd, *memOpnd);
1131         if (GetCG()->GenerateVerboseCG()) {
1132             insn.SetComment(GenerateMemOpndVerbose(*memOpnd));
1133         }
1134         GetCurBB()->AppendInsn(insn);
1135     } else {
1136         AArch64CGFunc::SelectStoreRelease(*memOpnd, ptyp, stOpnd, ptyp, memOrd, true);
1137     }
1138 }
1139 
SelectDassignoff(DassignoffNode & stmt,Operand & opnd0)1140 void AArch64CGFunc::SelectDassignoff(DassignoffNode &stmt, Operand &opnd0)
1141 {
1142     MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(stmt.stIdx);
1143     int64 offset = stmt.offset;
1144     uint32 size = GetPrimTypeSize(stmt.GetPrimType()) * k8ByteSize;
1145     MOperator mOp = (size == k16BitSize)
1146                         ? MOP_wstrh
1147                         : ((size == k32BitSize) ? MOP_wstr : ((size == k64BitSize) ? MOP_xstr : MOP_undef));
1148     CHECK_FATAL(mOp != MOP_undef, "illegal size for dassignoff");
1149     MemOperand *memOpnd = &GetOrCreateMemOpnd(*symbol, offset, size);
1150     if ((memOpnd->GetMemVaryType() == kNotVary) &&
1151         (IsImmediateOffsetOutOfRange(*memOpnd, size) || (offset % k8BitSize != 0))) {
1152         memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, size);
1153     }
1154     Operand &stOpnd = LoadIntoRegister(opnd0, true, size, false);
1155     memOpnd = memOpnd->IsOffsetMisaligned(size) ? &ConstraintOffsetToSafeRegion(size, *memOpnd) : memOpnd;
1156     Insn &insn = GetInsnBuilder()->BuildInsn(mOp, stOpnd, *memOpnd);
1157     GetCurBB()->AppendInsn(insn);
1158 }
1159 
SelectAssertNull(UnaryStmtNode & stmt)1160 void AArch64CGFunc::SelectAssertNull(UnaryStmtNode &stmt)
1161 {
1162     Operand *opnd0 = HandleExpr(stmt, *stmt.Opnd(0));
1163     RegOperand &baseReg = LoadIntoRegister(*opnd0, PTY_a64);
1164     auto &zwr = GetZeroOpnd(k32BitSize);
1165     auto &mem = CreateMemOpnd(baseReg, 0, k32BitSize);
1166     Insn &loadRef = GetInsnBuilder()->BuildInsn(MOP_wldr, zwr, mem);
1167     loadRef.SetDoNotRemove(true);
1168     if (GetCG()->GenerateVerboseCG()) {
1169         loadRef.SetComment("null pointer check");
1170     }
1171     GetCurBB()->AppendInsn(loadRef);
1172 }
1173 
SelectAbort()1174 void AArch64CGFunc::SelectAbort()
1175 {
1176     RegOperand &inOpnd = GetOrCreatePhysicalRegisterOperand(R16, k64BitSize, kRegTyInt);
1177     auto &mem = CreateMemOpnd(inOpnd, 0, k64BitSize);
1178     Insn &movXzr = GetInsnBuilder()->BuildInsn(MOP_xmovri64, inOpnd, CreateImmOperand(0, k64BitSize, false));
1179     Insn &loadRef = GetInsnBuilder()->BuildInsn(MOP_wldr, GetZeroOpnd(k64BitSize), mem);
1180     loadRef.SetDoNotRemove(true);
1181     movXzr.SetDoNotRemove(true);
1182     GetCurBB()->AppendInsn(movXzr);
1183     GetCurBB()->AppendInsn(loadRef);
1184 }
1185 
GetRegPrefixFromPrimType(PrimType pType,uint32 size,const std::string & constraint)1186 static std::string GetRegPrefixFromPrimType(PrimType pType, uint32 size, const std::string &constraint)
1187 {
1188     std::string regPrefix = "";
1189     /* memory access check */
1190     if (constraint.find("m") != std::string::npos || constraint.find("Q") != std::string::npos) {
1191         regPrefix += "[";
1192     }
1193     if (IsPrimitiveVector(pType)) {
1194         regPrefix += "v";
1195     } else if (IsPrimitiveInteger(pType)) {
1196         if (size == k32BitSize) {
1197             regPrefix += "w";
1198         } else {
1199             regPrefix += "x";
1200         }
1201     } else {
1202         if (size == k32BitSize) {
1203             regPrefix += "s";
1204         } else {
1205             regPrefix += "d";
1206         }
1207     }
1208     return regPrefix;
1209 }
1210 
SelectAsm(AsmNode & node)1211 void AArch64CGFunc::SelectAsm(AsmNode &node)
1212 {
1213     SetHasAsm();
1214     if (Globals::GetInstance()->GetOptimLevel() > CGOptions::kLevel0) {
1215         if (GetCG()->GetCGOptions().DoLinearScanRegisterAllocation()) {
1216             LogInfo::MapleLogger() << "Using coloring RA\n";
1217             const_cast<CGOptions &>(GetCG()->GetCGOptions()).SetOption(CGOptions::kDoColorRegAlloc);
1218             const_cast<CGOptions &>(GetCG()->GetCGOptions()).ClearOption(CGOptions::kDoLinearScanRegAlloc);
1219         }
1220     }
1221     Operand *asmString = &CreateStringOperand(node.asmString);
1222     ListOperand *listInputOpnd = CreateListOpnd(*GetFuncScopeAllocator());
1223     ListOperand *listOutputOpnd = CreateListOpnd(*GetFuncScopeAllocator());
1224     ListOperand *listClobber = CreateListOpnd(*GetFuncScopeAllocator());
1225     ListConstraintOperand *listInConstraint = memPool->New<ListConstraintOperand>(*GetFuncScopeAllocator());
1226     ListConstraintOperand *listOutConstraint = memPool->New<ListConstraintOperand>(*GetFuncScopeAllocator());
1227     ListConstraintOperand *listInRegPrefix = memPool->New<ListConstraintOperand>(*GetFuncScopeAllocator());
1228     ListConstraintOperand *listOutRegPrefix = memPool->New<ListConstraintOperand>(*GetFuncScopeAllocator());
1229     std::list<std::pair<Operand *, PrimType>> rPlusOpnd;
1230     bool noReplacement = false;
1231     if (node.asmString.find('$') == std::string::npos) {
1232         /* no replacements */
1233         noReplacement = true;
1234     }
1235     /* input constraints should be processed before OP_asm instruction */
1236     for (size_t i = 0; i < node.numOpnds; ++i) {
1237         /* process input constraint */
1238         std::string str = GlobalTables::GetUStrTable().GetStringFromStrIdx(node.inputConstraints[i]);
1239         bool isOutputTempNode = false;
1240         if (str[0] == '+') {
1241             isOutputTempNode = true;
1242         }
1243         listInConstraint->stringList.push_back(static_cast<StringOperand *>(&CreateStringOperand(str)));
1244         /* process input operands */
1245         switch (node.Opnd(i)->op) {
1246             case OP_dread: {
1247                 DreadNode &dread = static_cast<DreadNode &>(*node.Opnd(i));
1248                 Operand *inOpnd = SelectDread(node, dread);
1249                 PrimType pType = dread.GetPrimType();
1250                 listInputOpnd->PushOpnd(static_cast<RegOperand &>(*inOpnd));
1251                 listInRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1252                     &CreateStringOperand(GetRegPrefixFromPrimType(pType, inOpnd->GetSize(), str))));
1253                 if (isOutputTempNode) {
1254                     rPlusOpnd.emplace_back(std::make_pair(inOpnd, pType));
1255                 }
1256                 break;
1257             }
1258             case OP_addrof: {
1259                 auto &addrofNode = static_cast<AddrofNode &>(*node.Opnd(i));
1260                 Operand *inOpnd = SelectAddrof(addrofNode, node);
1261                 listInputOpnd->PushOpnd(static_cast<RegOperand &>(*inOpnd));
1262                 PrimType pType = addrofNode.GetPrimType();
1263                 listInRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1264                     &CreateStringOperand(GetRegPrefixFromPrimType(pType, inOpnd->GetSize(), str))));
1265                 if (isOutputTempNode) {
1266                     rPlusOpnd.emplace_back(std::make_pair(inOpnd, pType));
1267                 }
1268                 break;
1269             }
1270             case OP_constval: {
1271                 CHECK_FATAL(!isOutputTempNode, "Unexpect");
1272                 auto &constNode = static_cast<ConstvalNode &>(*node.Opnd(i));
1273                 CHECK_FATAL(constNode.GetConstVal()->GetKind() == kConstInt,
1274                             "expect MIRIntConst does not support float yet");
1275                 MIRIntConst *mirIntConst = safe_cast<MIRIntConst>(constNode.GetConstVal());
1276                 CHECK_FATAL(mirIntConst != nullptr, "just checking");
1277                 int64 scale = mirIntConst->GetExtValue();
1278                 if (str.find("r") != std::string::npos) {
1279                     bool isSigned = scale < 0;
1280                     ImmOperand &immOpnd = CreateImmOperand(scale, k64BitSize, isSigned);
1281                     /* set default type as a 64 bit reg */
1282                     PrimType pty = isSigned ? PTY_i64 : PTY_u64;
1283                     auto &tempReg = static_cast<Operand &>(CreateRegisterOperandOfType(pty));
1284                     SelectCopy(tempReg, pty, immOpnd, isSigned ? PTY_i64 : PTY_u64);
1285                     listInputOpnd->PushOpnd(static_cast<RegOperand &>(tempReg));
1286                     listInRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1287                         &CreateStringOperand(GetRegPrefixFromPrimType(pty, tempReg.GetSize(), str))));
1288                 } else {
1289                     RegOperand &inOpnd = GetOrCreatePhysicalRegisterOperand(RZR, k64BitSize, kRegTyInt);
1290                     listInputOpnd->PushOpnd(static_cast<RegOperand &>(inOpnd));
1291 
1292                     listInRegPrefix->stringList.push_back(
1293                         static_cast<StringOperand *>(&CreateStringOperand("i" + std::to_string(scale))));
1294                 }
1295                 break;
1296             }
1297             case OP_regread: {
1298                 auto &regreadNode = static_cast<RegreadNode &>(*node.Opnd(i));
1299                 PregIdx pregIdx = regreadNode.GetRegIdx();
1300                 RegOperand &inOpnd = GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx));
1301                 listInputOpnd->PushOpnd(static_cast<RegOperand &>(inOpnd));
1302                 MIRPreg *preg = GetFunction().GetPregTab()->PregFromPregIdx(pregIdx);
1303                 PrimType pType = preg->GetPrimType();
1304                 listInRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1305                     &CreateStringOperand(GetRegPrefixFromPrimType(pType, inOpnd.GetSize(), str))));
1306                 if (isOutputTempNode) {
1307                     rPlusOpnd.emplace_back(std::make_pair(&static_cast<Operand &>(inOpnd), pType));
1308                 }
1309                 break;
1310             }
1311             default:
1312                 CHECK_FATAL(0, "Inline asm input expression not handled");
1313         }
1314     }
1315     std::vector<Operand *> intrnOpnds;
1316     intrnOpnds.emplace_back(asmString);
1317     intrnOpnds.emplace_back(listOutputOpnd);
1318     intrnOpnds.emplace_back(listClobber);
1319     intrnOpnds.emplace_back(listInputOpnd);
1320     intrnOpnds.emplace_back(listOutConstraint);
1321     intrnOpnds.emplace_back(listInConstraint);
1322     intrnOpnds.emplace_back(listOutRegPrefix);
1323     intrnOpnds.emplace_back(listInRegPrefix);
1324     Insn *asmInsn = &GetInsnBuilder()->BuildInsn(MOP_asm, intrnOpnds);
1325     GetCurBB()->AppendInsn(*asmInsn);
1326 
1327     /* process listOutputOpnd */
1328     for (size_t i = 0; i < node.asmOutputs.size(); ++i) {
1329         bool isOutputTempNode = false;
1330         RegOperand *rPOpnd = nullptr;
1331         /* process output constraint */
1332         std::string str = GlobalTables::GetUStrTable().GetStringFromStrIdx(node.outputConstraints[i]);
1333 
1334         listOutConstraint->stringList.push_back(static_cast<StringOperand *>(&CreateStringOperand(str)));
1335         if (str[0] == '+') {
1336             CHECK_FATAL(!rPlusOpnd.empty(), "Need r+ operand");
1337             rPOpnd = static_cast<RegOperand *>((rPlusOpnd.begin()->first));
1338             listOutputOpnd->PushOpnd(*rPOpnd);
1339             listOutRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1340                 &CreateStringOperand(GetRegPrefixFromPrimType(rPlusOpnd.begin()->second, rPOpnd->GetSize(), str))));
1341             if (!rPlusOpnd.empty()) {
1342                 rPlusOpnd.pop_front();
1343             }
1344             isOutputTempNode = true;
1345         }
1346         if (str.find("Q") != std::string::npos || str.find("m") != std::string::npos) {
1347             continue;
1348         }
1349         /* process output operands */
1350         StIdx stIdx = node.asmOutputs[i].first;
1351         RegFieldPair regFieldPair = node.asmOutputs[i].second;
1352         if (regFieldPair.IsReg()) {
1353             PregIdx pregIdx = static_cast<PregIdx>(regFieldPair.GetPregIdx());
1354             MIRPreg *mirPreg = mirModule.CurFunction()->GetPregTab()->PregFromPregIdx(pregIdx);
1355             RegOperand *outOpnd = isOutputTempNode
1356                                       ? rPOpnd
1357                                       : &GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx));
1358             PrimType srcType = mirPreg->GetPrimType();
1359             PrimType destType = srcType;
1360             if (GetPrimTypeBitSize(destType) < k32BitSize) {
1361                 destType = IsSignedInteger(destType) ? PTY_i32 : PTY_u32;
1362             }
1363             RegType rtype = GetRegTyFromPrimTy(srcType);
1364             RegOperand &opnd0 = isOutputTempNode
1365                                     ? GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx))
1366                                     : CreateVirtualRegisterOperand(NewVReg(rtype, GetPrimTypeSize(srcType)));
1367             SelectCopy(opnd0, destType, *outOpnd, srcType);
1368             if (!isOutputTempNode) {
1369                 listOutputOpnd->PushOpnd(static_cast<RegOperand &>(*outOpnd));
1370                 listOutRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1371                     &CreateStringOperand(GetRegPrefixFromPrimType(srcType, outOpnd->GetSize(), str))));
1372             }
1373         } else {
1374             MIRSymbol *var;
1375             if (stIdx.IsGlobal()) {
1376                 var = GlobalTables::GetGsymTable().GetSymbolFromStidx(stIdx.Idx());
1377             } else {
1378                 var = mirModule.CurFunction()->GetSymbolTabItem(stIdx.Idx());
1379             }
1380             CHECK_FATAL(var != nullptr, "var should not be nullptr");
1381             if (!noReplacement || var->GetAsmAttr() != UStrIdx(0)) {
1382                 RegOperand *outOpnd = nullptr;
1383                 PrimType pty = GlobalTables::GetTypeTable().GetTypeTable().at(var->GetTyIdx())->GetPrimType();
1384                 if (var->GetAsmAttr() != UStrIdx(0)) {
1385                     std::string regDesp = GlobalTables::GetUStrTable().GetStringFromStrIdx(var->GetAsmAttr());
1386                     outOpnd = &GetOrCreatePhysicalRegisterOperand(regDesp);
1387                 } else {
1388                     RegType rtype = GetRegTyFromPrimTy(pty);
1389                     outOpnd =
1390                         isOutputTempNode ? rPOpnd : &CreateVirtualRegisterOperand(NewVReg(rtype, GetPrimTypeSize(pty)));
1391                 }
1392                 SaveReturnValueInLocal(node.asmOutputs, i, PTY_a64, *outOpnd, node);
1393                 if (!isOutputTempNode) {
1394                     listOutputOpnd->PushOpnd(static_cast<RegOperand &>(*outOpnd));
1395                     listOutRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1396                         &CreateStringOperand(GetRegPrefixFromPrimType(pty, outOpnd->GetSize(), str))));
1397                 }
1398             }
1399         }
1400     }
1401     if (noReplacement) {
1402         return;
1403     }
1404 
1405     /* process listClobber */
1406     for (size_t i = 0; i < node.clobberList.size(); ++i) {
1407         std::string str = GlobalTables::GetUStrTable().GetStringFromStrIdx(node.clobberList[i]);
1408         auto regno = static_cast<regno_t>(str[1] - '0');
1409         if (str[2] >= '0' && str[2] <= '9') { // if third char (index 2) is num, add to regno
1410             regno = regno * kDecimalMax + static_cast<uint32>((str[2] - '0'));
1411         }
1412         RegOperand *reg;
1413         switch (str[0]) {
1414             case 'w': {
1415                 reg = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(regno + R0), k32BitSize, kRegTyInt);
1416                 listClobber->PushOpnd(*reg);
1417                 break;
1418             }
1419             case 'x': {
1420                 reg = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(regno + R0), k64BitSize, kRegTyInt);
1421                 listClobber->PushOpnd(*reg);
1422                 break;
1423             }
1424             case 's': {
1425                 reg = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(regno + V0), k32BitSize, kRegTyFloat);
1426                 listClobber->PushOpnd(*reg);
1427                 break;
1428             }
1429             case 'd': {
1430                 reg = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(regno + V0), k64BitSize, kRegTyFloat);
1431                 listClobber->PushOpnd(*reg);
1432                 break;
1433             }
1434             case 'v': {
1435                 reg = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(regno + V0), k64BitSize, kRegTyFloat);
1436                 listClobber->PushOpnd(*reg);
1437                 break;
1438             }
1439             case 'c': {
1440                 asmInsn->SetAsmDefCondCode();
1441                 break;
1442             }
1443             case 'm': {
1444                 asmInsn->SetAsmModMem();
1445                 break;
1446             }
1447             default:
1448                 CHECK_FATAL(0, "Inline asm clobber list not handled");
1449         }
1450     }
1451 }
1452 
SelectRegassign(RegassignNode & stmt,Operand & opnd0)1453 void AArch64CGFunc::SelectRegassign(RegassignNode &stmt, Operand &opnd0)
1454 {
1455     if (GetCG()->IsLmbc()) {
1456         PrimType lhsSize = stmt.GetPrimType();
1457         PrimType rhsSize = stmt.Opnd(0)->GetPrimType();
1458         if (lhsSize != rhsSize && stmt.Opnd(0)->GetOpCode() == OP_ireadoff) {
1459             Insn *prev = GetCurBB()->GetLastInsn();
1460             if (prev->GetMachineOpcode() == MOP_wldrsb || prev->GetMachineOpcode() == MOP_wldrsh) {
1461                 opnd0.SetSize(GetPrimTypeBitSize(stmt.GetPrimType()));
1462                 prev->SetMOP(AArch64CG::kMd[prev->GetMachineOpcode() == MOP_wldrsb ? MOP_xldrsb : MOP_xldrsh]);
1463             } else if (prev->GetMachineOpcode() == MOP_wldr && stmt.GetPrimType() == PTY_i64) {
1464                 opnd0.SetSize(GetPrimTypeBitSize(stmt.GetPrimType()));
1465                 prev->SetMOP(AArch64CG::kMd[MOP_xldrsw]);
1466             }
1467         }
1468     }
1469     RegOperand *regOpnd = nullptr;
1470     PregIdx pregIdx = stmt.GetRegIdx();
1471     if (IsSpecialPseudoRegister(pregIdx)) {
1472         if (GetCG()->IsLmbc() && stmt.GetPrimType() == PTY_agg) {
1473             if (static_cast<RegOperand &>(opnd0).IsOfIntClass()) {
1474                 regOpnd = &GetOrCreateSpecialRegisterOperand(-pregIdx, PTY_i64);
1475             } else if (opnd0.GetSize() <= k4ByteSize) {
1476                 regOpnd = &GetOrCreateSpecialRegisterOperand(-pregIdx, PTY_f32);
1477             } else {
1478                 regOpnd = &GetOrCreateSpecialRegisterOperand(-pregIdx, PTY_f64);
1479             }
1480         } else {
1481             regOpnd = &GetOrCreateSpecialRegisterOperand(-pregIdx, stmt.GetPrimType());
1482         }
1483     } else {
1484         regOpnd = &GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx));
1485     }
1486     /* look at rhs */
1487     PrimType rhsType = stmt.Opnd(0)->GetPrimType();
1488     if (GetCG()->IsLmbc() && rhsType == PTY_agg) {
1489         /* This occurs when a call returns a small struct */
1490         /* The subtree should already taken care of the agg type that is in excess of 8 bytes */
1491         rhsType = PTY_i64;
1492     }
1493     PrimType dtype = rhsType;
1494     if (GetPrimTypeBitSize(dtype) < k32BitSize) {
1495         DEBUG_ASSERT(IsPrimitiveInteger(dtype), "");
1496         dtype = IsSignedInteger(dtype) ? PTY_i32 : PTY_u32;
1497     }
1498     DEBUG_ASSERT(regOpnd != nullptr, "null ptr check!");
1499     SelectCopy(*regOpnd, dtype, opnd0, rhsType);
1500     if (GetCG()->GenerateVerboseCG()) {
1501         if (GetCurBB()->GetLastInsn()) {
1502             GetCurBB()->GetLastInsn()->AppendComment(" regassign %" + std::to_string(pregIdx) + "; ");
1503         } else if (GetCurBB()->GetPrev()->GetLastInsn()) {
1504             GetCurBB()->GetPrev()->GetLastInsn()->AppendComment(" regassign %" + std::to_string(pregIdx) + "; ");
1505         }
1506     }
1507 
1508     if ((Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) && (pregIdx >= 0)) {
1509         MemOperand *dest = GetPseudoRegisterSpillMemoryOperand(pregIdx);
1510         PrimType stype = GetTypeFromPseudoRegIdx(pregIdx);
1511         MIRPreg *preg = GetFunction().GetPregTab()->PregFromPregIdx(pregIdx);
1512         uint32 srcBitLength = GetPrimTypeSize(preg->GetPrimType()) * kBitsPerByte;
1513         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(srcBitLength, stype), *regOpnd, *dest));
1514     } else if (regOpnd->GetRegisterNumber() == R0 || regOpnd->GetRegisterNumber() == R1) {
1515         Insn &pseudo = GetInsnBuilder()->BuildInsn(MOP_pseudo_ret_int, *regOpnd);
1516         GetCurBB()->AppendInsn(pseudo);
1517     } else if (regOpnd->GetRegisterNumber() >= V0 && regOpnd->GetRegisterNumber() <= V3) {
1518         Insn &pseudo = GetInsnBuilder()->BuildInsn(MOP_pseudo_ret_float, *regOpnd);
1519         GetCurBB()->AppendInsn(pseudo);
1520     }
1521     if (stmt.GetPrimType() == PTY_ref) {
1522         regOpnd->SetIsReference(true);
1523         AddReferenceReg(regOpnd->GetRegisterNumber());
1524     }
1525     if (pregIdx > 0) {
1526         // special MIRPreg is not supported
1527         SetPregIdx2Opnd(pregIdx, *regOpnd);
1528     }
1529     const auto &derived2BaseRef = GetFunction().GetDerived2BaseRef();
1530     auto itr = derived2BaseRef.find(pregIdx);
1531     if (itr != derived2BaseRef.end()) {
1532         auto *opnd = GetOpndFromPregIdx(itr->first);
1533         CHECK_FATAL(opnd != nullptr, "pregIdx has not been assigned Operand");
1534         auto &derivedRegOpnd = static_cast<RegOperand &>(*opnd);
1535         opnd = GetOpndFromPregIdx(itr->second);
1536         CHECK_FATAL(opnd != nullptr, "pregIdx has not been assigned Operand");
1537         auto &baseRegOpnd = static_cast<RegOperand &>(*opnd);
1538         derivedRegOpnd.SetBaseRefOpnd(baseRegOpnd);
1539     }
1540 }
1541 
FixLargeMemOpnd(MemOperand & memOpnd,uint32 align)1542 MemOperand *AArch64CGFunc::FixLargeMemOpnd(MemOperand &memOpnd, uint32 align)
1543 {
1544     MemOperand *lhsMemOpnd = &memOpnd;
1545     if ((lhsMemOpnd->GetMemVaryType() == kNotVary) && IsImmediateOffsetOutOfRange(*lhsMemOpnd, align * kBitsPerByte)) {
1546         RegOperand *addReg = &CreateRegisterOperandOfType(PTY_i64);
1547         lhsMemOpnd = &SplitOffsetWithAddInstruction(*lhsMemOpnd, align * k8BitSize, addReg->GetRegisterNumber());
1548     }
1549     return lhsMemOpnd;
1550 }
1551 
FixLargeMemOpnd(MOperator mOp,MemOperand & memOpnd,uint32 dSize,uint32 opndIdx)1552 MemOperand *AArch64CGFunc::FixLargeMemOpnd(MOperator mOp, MemOperand &memOpnd, uint32 dSize, uint32 opndIdx)
1553 {
1554     auto *a64MemOpnd = &memOpnd;
1555     if ((a64MemOpnd->GetMemVaryType() == kNotVary) && !IsOperandImmValid(mOp, &memOpnd, opndIdx)) {
1556         if (opndIdx == kInsnSecondOpnd) {
1557             a64MemOpnd = &SplitOffsetWithAddInstruction(*a64MemOpnd, dSize);
1558         } else if (opndIdx == kInsnThirdOpnd) {
1559             a64MemOpnd =
1560                 &SplitOffsetWithAddInstruction(*a64MemOpnd, dSize, AArch64reg::kRinvalid, false, nullptr, true);
1561         } else {
1562             CHECK_FATAL(false, "NYI");
1563         }
1564     }
1565     return a64MemOpnd;
1566 }
1567 
GenLargeAggFormalMemOpnd(const MIRSymbol & sym,uint32 align,int64 offset,bool needLow12)1568 MemOperand *AArch64CGFunc::GenLargeAggFormalMemOpnd(const MIRSymbol &sym, uint32 align, int64 offset, bool needLow12)
1569 {
1570     MemOperand *memOpnd;
1571     if (sym.GetStorageClass() == kScFormal && GetBecommon().GetTypeSize(sym.GetTyIdx()) > k16ByteSize) {
1572         /* formal of size of greater than 16 is copied by the caller and the pointer to it is passed. */
1573         /* otherwise it is passed in register and is accessed directly. */
1574         memOpnd = &GetOrCreateMemOpnd(sym, 0, align * kBitsPerByte);
1575         RegOperand *vreg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
1576         Insn &ldInsn = GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_i64), *vreg, *memOpnd);
1577         GetCurBB()->AppendInsn(ldInsn);
1578         memOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, vreg, nullptr,
1579                                       &GetOrCreateOfstOpnd(static_cast<uint64>(offset), k32BitSize), nullptr);
1580     } else {
1581         memOpnd = &GetOrCreateMemOpnd(sym, offset, align * kBitsPerByte, false, needLow12);
1582     }
1583     return FixLargeMemOpnd(*memOpnd, align);
1584 }
1585 
PrepareMemcpyParamOpnd(bool isLo12,const MIRSymbol & symbol,int64 offsetVal,RegOperand & BaseReg)1586 RegOperand *AArch64CGFunc::PrepareMemcpyParamOpnd(bool isLo12, const MIRSymbol &symbol, int64 offsetVal,
1587                                                   RegOperand &BaseReg)
1588 {
1589     RegOperand *tgtAddr = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
1590     if (isLo12) {
1591         StImmOperand &stImm = CreateStImmOperand(symbol, 0, 0);
1592         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrpl12, *tgtAddr, BaseReg, stImm));
1593     } else {
1594         ImmOperand &imm = CreateImmOperand(offsetVal, k64BitSize, false);
1595         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *tgtAddr, BaseReg, imm));
1596     }
1597     return tgtAddr;
1598 }
1599 
PrepareMemcpyParamOpnd(int64 offset,Operand & exprOpnd)1600 RegOperand *AArch64CGFunc::PrepareMemcpyParamOpnd(int64 offset, Operand &exprOpnd)
1601 {
1602     RegOperand *tgtAddr = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
1603     OfstOperand *ofstOpnd = &GetOrCreateOfstOpnd(static_cast<uint64>(offset), k32BitSize);
1604     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *tgtAddr, exprOpnd, *ofstOpnd));
1605     return tgtAddr;
1606 }
1607 
PrepareMemcpyParamOpnd(uint64 copySize)1608 RegOperand *AArch64CGFunc::PrepareMemcpyParamOpnd(uint64 copySize)
1609 {
1610     RegOperand *vregMemcpySize = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
1611     ImmOperand *sizeOpnd = &CreateImmOperand(static_cast<int64>(copySize), k64BitSize, false);
1612     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wmovri32, *vregMemcpySize, *sizeOpnd));
1613     return vregMemcpySize;
1614 }
1615 
AggtStrLdrInsert(bool bothUnion,Insn * lastStrLdr,Insn & newStrLdr)1616 Insn *AArch64CGFunc::AggtStrLdrInsert(bool bothUnion, Insn *lastStrLdr, Insn &newStrLdr)
1617 {
1618     if (bothUnion) {
1619         if (lastStrLdr == nullptr) {
1620             GetCurBB()->AppendInsn(newStrLdr);
1621         } else {
1622             GetCurBB()->InsertInsnAfter(*lastStrLdr, newStrLdr);
1623         }
1624     } else {
1625         GetCurBB()->AppendInsn(newStrLdr);
1626     }
1627     return &newStrLdr;
1628 }
1629 
GetOrCreateLocator(CallConvKind cc)1630 CCImpl *AArch64CGFunc::GetOrCreateLocator(CallConvKind cc)
1631 {
1632     auto it = hashCCTable.find(cc);
1633     if (it != hashCCTable.end()) {
1634         it->second->Init();
1635         return it->second;
1636     }
1637     CCImpl *res = nullptr;
1638     if (cc == kCCall) {
1639         res = memPool->New<AArch64CallConvImpl>(GetBecommon());
1640     } else if (cc == kWebKitJS) {
1641         res = memPool->New<AArch64WebKitJSCC>(GetBecommon());
1642     } else if (cc == kGHC) {
1643         res = memPool->New<GHCCC>(GetBecommon());
1644     } else {
1645         CHECK_FATAL(false, "unsupported yet");
1646     }
1647     hashCCTable[cc] = res;
1648     return res;
1649 }
SelectAggDassign(DassignNode & stmt)1650 void AArch64CGFunc::SelectAggDassign(DassignNode &stmt)
1651 {
1652     MIRSymbol *lhsSymbol = GetFunction().GetLocalOrGlobalSymbol(stmt.GetStIdx());
1653     uint32 lhsOffset = 0;
1654     MIRType *lhsType = lhsSymbol->GetType();
1655     bool bothUnion = false;
1656     if (stmt.GetFieldID() != 0) {
1657         MIRStructType *structType = static_cast<MIRStructType *>(lhsSymbol->GetType());
1658         DEBUG_ASSERT(structType != nullptr, "SelectAggDassign: non-zero fieldID for non-structure");
1659         lhsType = structType->GetFieldType(stmt.GetFieldID());
1660         lhsOffset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, stmt.GetFieldID()).first);
1661         bothUnion |= (structType->GetKind() == kTypeUnion);
1662     }
1663     uint32 lhsAlign = GetBecommon().GetTypeAlign(lhsType->GetTypeIndex());
1664     uint64 lhsSize = GetBecommon().GetTypeSize(lhsType->GetTypeIndex());
1665 
1666     uint32 rhsAlign;
1667     uint32 alignUsed;
1668     uint32 rhsOffset = 0;
1669     if (stmt.GetRHS()->GetOpCode() == OP_dread) {
1670         AddrofNode *rhsDread = static_cast<AddrofNode *>(stmt.GetRHS());
1671         MIRSymbol *rhsSymbol = GetFunction().GetLocalOrGlobalSymbol(rhsDread->GetStIdx());
1672         MIRType *rhsType = rhsSymbol->GetType();
1673         if (rhsDread->GetFieldID() != 0) {
1674             MIRStructType *structType = static_cast<MIRStructType *>(rhsSymbol->GetType());
1675             DEBUG_ASSERT(structType != nullptr, "SelectAggDassign: non-zero fieldID for non-structure");
1676             rhsType = structType->GetFieldType(rhsDread->GetFieldID());
1677             rhsOffset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, rhsDread->GetFieldID()).first);
1678             bothUnion &= (structType->GetKind() == kTypeUnion);
1679         }
1680         bothUnion &= (rhsSymbol == lhsSymbol);
1681         rhsAlign = GetBecommon().GetTypeAlign(rhsType->GetTypeIndex());
1682         alignUsed = std::min(lhsAlign, rhsAlign);
1683         DEBUG_ASSERT(alignUsed != 0, "expect non-zero");
1684         uint32 copySize = GetAggCopySize(lhsOffset, rhsOffset, alignUsed);
1685         MemOperand *rhsBaseMemOpnd;
1686         if (IsParamStructCopy(*rhsSymbol)) {
1687             rhsBaseMemOpnd = &LoadStructCopyBase(*rhsSymbol, rhsOffset, static_cast<int>(copySize * k8BitSize));
1688         } else {
1689             rhsBaseMemOpnd = &GetOrCreateMemOpnd(*rhsSymbol, rhsOffset, copySize * k8BitSize, false, true);
1690             rhsBaseMemOpnd = FixLargeMemOpnd(*rhsBaseMemOpnd, copySize);
1691         }
1692         RegOperand *rhsBaseReg = rhsBaseMemOpnd->GetBaseRegister();
1693         int64 rhsOffsetVal = rhsBaseMemOpnd->GetOffsetOperand()->GetValue();
1694         MemOperand *lhsBaseMemOpnd = GenLargeAggFormalMemOpnd(*lhsSymbol, copySize, lhsOffset, true);
1695         RegOperand *lhsBaseReg = lhsBaseMemOpnd->GetBaseRegister();
1696         int64 lhsOffsetVal = lhsBaseMemOpnd->GetOffsetOperand()->GetValue();
1697         bool rhsIsLo12 = (rhsBaseMemOpnd->GetAddrMode() == MemOperand::kAddrModeLo12Li);
1698         bool lhsIsLo12 = (lhsBaseMemOpnd->GetAddrMode() == MemOperand::kAddrModeLo12Li);
1699         if (lhsSize > kParmMemcpySize) {
1700             std::vector<Operand *> opndVec;
1701             RegOperand *regResult = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
1702             opndVec.push_back(regResult); /* result */
1703 
1704             opndVec.push_back(PrepareMemcpyParamOpnd(lhsIsLo12, *lhsSymbol, lhsOffsetVal, *lhsBaseReg)); /* param 0 */
1705 
1706             opndVec.push_back(PrepareMemcpyParamOpnd(rhsIsLo12, *rhsSymbol, rhsOffsetVal, *rhsBaseReg)); /* param 1 */
1707 
1708             opndVec.push_back(PrepareMemcpyParamOpnd(lhsSize)); /* param 2 */
1709 
1710             SelectLibCall("memcpy", opndVec, PTY_a64, PTY_a64);
1711 
1712             return;
1713         }
1714         Insn *lastLdr = nullptr;
1715         Insn *lastStr = nullptr;
1716         for (uint32 i = 0; i < (lhsSize / copySize); i++) {
1717             uint64 rhsBaseOffset = i * copySize + static_cast<uint64>(rhsOffsetVal);
1718             uint64 lhsBaseOffset = i * copySize + static_cast<uint64>(lhsOffsetVal);
1719             MemOperand::AArch64AddressingMode addrMode =
1720                 rhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
1721             MIRSymbol *sym = rhsIsLo12 ? rhsSymbol : nullptr;
1722             OfstOperand &rhsOfstOpnd = GetOrCreateOfstOpnd(rhsBaseOffset, k32BitSize);
1723             /* generate the load */
1724             MemOperand *rhsMemOpnd =
1725                 &GetOrCreateMemOpnd(addrMode, copySize * k8BitSize, rhsBaseReg, nullptr, &rhsOfstOpnd, sym);
1726             /* generate the load */
1727             RegOperand &result = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, copySize)));
1728             bool doPair = (!rhsIsLo12 && !lhsIsLo12 && (copySize >= k4BitSize) && ((i + 1) < (lhsSize / copySize)));
1729             RegOperand *result1 = nullptr;
1730             Insn *newLoadInsn = nullptr;
1731             if (doPair) {
1732                 MOperator mOpLDP = (copySize == k4BitSize) ? MOP_wldp : MOP_xldp;
1733                 result1 = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, copySize)));
1734                 rhsMemOpnd = FixLargeMemOpnd(mOpLDP, *rhsMemOpnd, copySize * k8BitSize, kInsnThirdOpnd);
1735                 newLoadInsn = &GetInsnBuilder()->BuildInsn(mOpLDP, result, *result1, *rhsMemOpnd);
1736             } else {
1737                 MOperator mOp = PickLdInsn(copySize * k8BitSize, PTY_u32);
1738                 rhsMemOpnd = FixLargeMemOpnd(mOp, *rhsMemOpnd, copySize * k8BitSize, kInsnSecondOpnd);
1739                 newLoadInsn = &GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd);
1740             }
1741             DEBUG_ASSERT(newLoadInsn != nullptr, "build load instruction failed in SelectAggDassign");
1742             lastLdr = AggtStrLdrInsert(bothUnion, lastLdr, *newLoadInsn);
1743             /* generate the store */
1744             OfstOperand &lhsOfstOpnd = GetOrCreateOfstOpnd(lhsBaseOffset, k32BitSize);
1745             addrMode = lhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
1746             sym = lhsIsLo12 ? lhsSymbol : nullptr;
1747             Insn *newStoreInsn = nullptr;
1748             MemOperand *lhsMemOpnd =
1749                 &GetOrCreateMemOpnd(addrMode, copySize * k8BitSize, lhsBaseReg, nullptr, &lhsOfstOpnd, sym);
1750             if (doPair) {
1751                 MOperator mOpSTP = (copySize == k4BitSize) ? MOP_wstp : MOP_xstp;
1752                 lhsMemOpnd = FixLargeMemOpnd(mOpSTP, *lhsMemOpnd, copySize * k8BitSize, kInsnThirdOpnd);
1753                 DEBUG_ASSERT(result1 != nullptr, "result1 should not be nullptr");
1754                 newStoreInsn = &GetInsnBuilder()->BuildInsn(mOpSTP, result, *result1, *lhsMemOpnd);
1755                 i++;
1756             } else {
1757                 MOperator mOp = PickStInsn(copySize * k8BitSize, PTY_u32);
1758                 lhsMemOpnd = FixLargeMemOpnd(mOp, *lhsMemOpnd, copySize * k8BitSize, kInsnSecondOpnd);
1759                 newStoreInsn = &GetInsnBuilder()->BuildInsn(mOp, result, *lhsMemOpnd);
1760             }
1761             DEBUG_ASSERT(newStoreInsn != nullptr, "build store instruction failed in SelectAggDassign");
1762             lastStr = AggtStrLdrInsert(bothUnion, lastStr, *newStoreInsn);
1763         }
1764         /* take care of extra content at the end less than the unit */
1765         uint64 lhsSizeCovered = (lhsSize / copySize) * copySize;
1766         uint32 newAlignUsed = copySize;
1767         while (lhsSizeCovered < lhsSize) {
1768             newAlignUsed = newAlignUsed >> 1;
1769             CHECK_FATAL(newAlignUsed != 0, "expect non-zero");
1770             if ((lhsSizeCovered + newAlignUsed) > lhsSize) {
1771                 continue;
1772             }
1773             /* generate the load */
1774             MemOperand *rhsMemOpnd;
1775             MemOperand::AArch64AddressingMode addrMode =
1776                 rhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
1777             MIRSymbol *sym = rhsIsLo12 ? rhsSymbol : nullptr;
1778             OfstOperand &rhsOfstOpnd =
1779                 GetOrCreateOfstOpnd(lhsSizeCovered + static_cast<uint64>(rhsOffsetVal), k32BitSize);
1780             rhsMemOpnd =
1781                 &GetOrCreateMemOpnd(addrMode, newAlignUsed * k8BitSize, rhsBaseReg, nullptr, &rhsOfstOpnd, sym);
1782             rhsMemOpnd = FixLargeMemOpnd(*rhsMemOpnd, newAlignUsed);
1783             regno_t vRegNO = NewVReg(kRegTyInt, std::max(4u, newAlignUsed));
1784             RegOperand &result = CreateVirtualRegisterOperand(vRegNO);
1785             MOperator mOp = PickLdInsn(newAlignUsed * k8BitSize, PTY_u32);
1786             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd));
1787             /* generate the store */
1788             addrMode = lhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
1789             sym = lhsIsLo12 ? lhsSymbol : nullptr;
1790             OfstOperand &lhsOfstOpnd =
1791                 GetOrCreateOfstOpnd(lhsSizeCovered + static_cast<uint64>(lhsOffsetVal), k32BitSize);
1792             MemOperand *lhsMemOpnd;
1793             lhsMemOpnd =
1794                 &GetOrCreateMemOpnd(addrMode, newAlignUsed * k8BitSize, lhsBaseReg, nullptr, &lhsOfstOpnd, sym);
1795             lhsMemOpnd = FixLargeMemOpnd(*lhsMemOpnd, newAlignUsed);
1796             mOp = PickStInsn(newAlignUsed * k8BitSize, PTY_u32);
1797             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *lhsMemOpnd));
1798             lhsSizeCovered += newAlignUsed;
1799         }
1800     } else if (stmt.GetRHS()->GetOpCode() == OP_iread) {
1801         IreadNode *rhsIread = static_cast<IreadNode *>(stmt.GetRHS());
1802         RegOperand *addrOpnd = static_cast<RegOperand *>(HandleExpr(*rhsIread, *rhsIread->Opnd(0)));
1803         addrOpnd = &LoadIntoRegister(*addrOpnd, rhsIread->Opnd(0)->GetPrimType());
1804         MIRPtrType *rhsPointerType =
1805             static_cast<MIRPtrType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(rhsIread->GetTyIdx()));
1806         MIRType *rhsType = static_cast<MIRStructType *>(
1807             GlobalTables::GetTypeTable().GetTypeFromTyIdx(rhsPointerType->GetPointedTyIdx()));
1808         bool isRefField = false;
1809         if (rhsIread->GetFieldID() != 0) {
1810             MIRStructType *rhsStructType = static_cast<MIRStructType *>(rhsType);
1811             DEBUG_ASSERT(rhsStructType != nullptr, "SelectAggDassign: non-zero fieldID for non-structure");
1812             rhsType = rhsStructType->GetFieldType(rhsIread->GetFieldID());
1813             rhsOffset = static_cast<uint32>(GetBecommon().GetFieldOffset(*rhsStructType, rhsIread->GetFieldID()).first);
1814             isRefField = GetBecommon().IsRefField(*rhsStructType, rhsIread->GetFieldID());
1815         }
1816         rhsAlign = GetBecommon().GetTypeAlign(rhsType->GetTypeIndex());
1817         alignUsed = std::min(lhsAlign, rhsAlign);
1818         DEBUG_ASSERT(alignUsed != 0, "expect non-zero");
1819         uint32 copySize = GetAggCopySize(rhsOffset, lhsOffset, alignUsed);
1820         MemOperand *lhsBaseMemOpnd = GenLargeAggFormalMemOpnd(*lhsSymbol, copySize, lhsOffset, true);
1821         RegOperand *lhsBaseReg = lhsBaseMemOpnd->GetBaseRegister();
1822         int64 lhsOffsetVal = lhsBaseMemOpnd->GetOffsetOperand()->GetValue();
1823         bool lhsIsLo12 = (lhsBaseMemOpnd->GetAddrMode() == MemOperand::kAddrModeLo12Li);
1824         if (lhsSize > kParmMemcpySize) {
1825             std::vector<Operand *> opndVec;
1826             RegOperand *regResult = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
1827             opndVec.push_back(regResult); /* result */
1828 
1829             opndVec.push_back(PrepareMemcpyParamOpnd(lhsIsLo12, *lhsSymbol, lhsOffsetVal, *lhsBaseReg)); /* param 0 */
1830 
1831             opndVec.push_back(PrepareMemcpyParamOpnd(rhsOffset, *addrOpnd)); /* param 1 */
1832 
1833             opndVec.push_back(PrepareMemcpyParamOpnd(lhsSize)); /* param 2 */
1834 
1835             SelectLibCall("memcpy", opndVec, PTY_a64, PTY_a64);
1836 
1837             return;
1838         }
1839         for (uint32 i = 0; i < (lhsSize / copySize); i++) {
1840             uint64 rhsBaseOffset = rhsOffset + i * copySize;
1841             uint64 lhsBaseOffset = static_cast<uint64>(lhsOffsetVal) + i * copySize;
1842             /* generate the load */
1843             OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(rhsBaseOffset, k32BitSize);
1844             MemOperand *rhsMemOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, copySize * k8BitSize, addrOpnd,
1845                                                          nullptr, &ofstOpnd, nullptr);
1846             regno_t vRegNO = NewVReg(kRegTyInt, std::max(4u, copySize));
1847             RegOperand &result = CreateVirtualRegisterOperand(vRegNO);
1848             bool doPair = (!lhsIsLo12 && copySize >= k4BitSize) && ((i + 1) < (lhsSize / copySize));
1849             Insn *insn = nullptr;
1850             RegOperand *result1 = nullptr;
1851             if (doPair) {
1852                 MOperator mOpLDP = (copySize == k4BitSize) ? MOP_wldp : MOP_xldp;
1853                 regno_t vRegNO1 = NewVReg(kRegTyInt, std::max(4u, copySize));
1854                 result1 = &CreateVirtualRegisterOperand(vRegNO1);
1855                 rhsMemOpnd = FixLargeMemOpnd(mOpLDP, *rhsMemOpnd, copySize * k8BitSize, kInsnThirdOpnd);
1856                 insn = &GetInsnBuilder()->BuildInsn(mOpLDP, result, *result1, *rhsMemOpnd);
1857             } else {
1858                 MOperator mOp = PickLdInsn(copySize * k8BitSize, PTY_u32);
1859                 rhsMemOpnd = FixLargeMemOpnd(mOp, *rhsMemOpnd, copySize * k8BitSize, kInsnSecondOpnd);
1860                 insn = &GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd);
1861             }
1862             insn->MarkAsAccessRefField(isRefField);
1863             GetCurBB()->AppendInsn(*insn);
1864             /* generate the store */
1865             MemOperand::AArch64AddressingMode addrMode =
1866                 lhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
1867             MIRSymbol *sym = lhsIsLo12 ? lhsSymbol : nullptr;
1868             OfstOperand &lhsOfstOpnd = GetOrCreateOfstOpnd(lhsBaseOffset, k32BitSize);
1869             MemOperand *lhsMemOpnd =
1870                 &GetOrCreateMemOpnd(addrMode, copySize * k8BitSize, lhsBaseReg, nullptr, &lhsOfstOpnd, sym);
1871             if (doPair) {
1872                 MOperator mOpSTP = (copySize == k4BitSize) ? MOP_wstp : MOP_xstp;
1873                 lhsMemOpnd = FixLargeMemOpnd(mOpSTP, *lhsMemOpnd, copySize * k8BitSize, kInsnThirdOpnd);
1874                 DEBUG_ASSERT(result1 != nullptr, "result1 should not be nullptr");
1875                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpSTP, result, *result1, *lhsMemOpnd));
1876                 i++;
1877             } else {
1878                 MOperator mOp = PickStInsn(copySize * k8BitSize, PTY_u32);
1879                 lhsMemOpnd = FixLargeMemOpnd(mOp, *lhsMemOpnd, copySize * k8BitSize, kInsnSecondOpnd);
1880                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *lhsMemOpnd));
1881             }
1882         }
1883         /* take care of extra content at the end less than the unit of alignUsed */
1884         uint64 lhsSizeCovered = (lhsSize / copySize) * copySize;
1885         uint32 newAlignUsed = copySize;
1886         while (lhsSizeCovered < lhsSize) {
1887             newAlignUsed = newAlignUsed >> 1;
1888             CHECK_FATAL(newAlignUsed != 0, "expect non-zero");
1889             if ((lhsSizeCovered + newAlignUsed) > lhsSize) {
1890                 continue;
1891             }
1892             /* generate the load */
1893             OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(rhsOffset + lhsSizeCovered, k32BitSize);
1894             MemOperand *rhsMemOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, newAlignUsed * k8BitSize, addrOpnd,
1895                                                          nullptr, &ofstOpnd, nullptr);
1896             rhsMemOpnd = FixLargeMemOpnd(*rhsMemOpnd, newAlignUsed);
1897             RegOperand &result = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, newAlignUsed)));
1898             MOperator mOp = PickLdInsn(newAlignUsed * k8BitSize, PTY_u32);
1899             Insn &insn = GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd);
1900             insn.MarkAsAccessRefField(isRefField);
1901             GetCurBB()->AppendInsn(insn);
1902             /* generate the store */
1903             MemOperand::AArch64AddressingMode addrMode =
1904                 lhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
1905             MIRSymbol *sym = lhsIsLo12 ? lhsSymbol : nullptr;
1906             OfstOperand &lhsOfstOpnd =
1907                 GetOrCreateOfstOpnd(lhsSizeCovered + static_cast<uint64>(lhsOffsetVal), k32BitSize);
1908             MemOperand *lhsMemOpnd;
1909             lhsMemOpnd =
1910                 &GetOrCreateMemOpnd(addrMode, newAlignUsed * k8BitSize, lhsBaseReg, nullptr, &lhsOfstOpnd, sym);
1911             lhsMemOpnd = FixLargeMemOpnd(*lhsMemOpnd, newAlignUsed);
1912             mOp = PickStInsn(newAlignUsed * k8BitSize, PTY_u32);
1913             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *lhsMemOpnd));
1914             lhsSizeCovered += newAlignUsed;
1915         }
1916     } else {
1917         DEBUG_ASSERT(stmt.GetRHS()->op == OP_regread, "SelectAggDassign: NYI");
1918         bool isRet = false;
1919         if (lhsType->GetKind() == kTypeStruct || lhsType->GetKind() == kTypeUnion) {
1920             RegreadNode *rhsregread = static_cast<RegreadNode *>(stmt.GetRHS());
1921             PregIdx pregIdx = rhsregread->GetRegIdx();
1922             if (IsSpecialPseudoRegister(pregIdx)) {
1923                 if ((-pregIdx) == kSregRetval0) {
1924                     CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
1925                     AArch64CallConvImpl parmlocator(GetBecommon());
1926                     CCLocInfo pLoc;
1927                     PrimType retPtype;
1928                     RegType regType;
1929                     uint32 memSize;
1930                     uint32 regSize;
1931                     parmlocator.LocateRetVal(*lhsType, pLoc);
1932                     AArch64reg r[kFourRegister];
1933                     r[kFirstReg] = static_cast<AArch64reg>(pLoc.reg0);
1934                     r[kSecondReg] = static_cast<AArch64reg>(pLoc.reg1);
1935                     r[kThirdReg] = static_cast<AArch64reg>(pLoc.reg2);
1936                     r[kFourthReg] = static_cast<AArch64reg>(pLoc.reg3);
1937                     if (pLoc.numFpPureRegs) {
1938                         regSize = (pLoc.fpSize == k4ByteSize) ? k32BitSize : k64BitSize;
1939                         memSize = pLoc.fpSize;
1940                         retPtype = (pLoc.fpSize == k4ByteSize) ? PTY_f32 : PTY_f64;
1941                         regType = kRegTyFloat;
1942                     } else {
1943                         regSize = k64BitSize;
1944                         memSize = k8BitSize;
1945                         retPtype = PTY_u64;
1946                         regType = kRegTyInt;
1947                     }
1948                     for (uint32 i = 0; i < kFourRegister; ++i) {
1949                         if (r[i] == kRinvalid) {
1950                             break;
1951                         }
1952                         RegOperand &parm = GetOrCreatePhysicalRegisterOperand(r[i], regSize, regType);
1953                         Operand &mOpnd = GetOrCreateMemOpnd(*lhsSymbol, memSize * i, regSize);
1954                         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(regSize, retPtype), parm, mOpnd));
1955                     }
1956                     isRet = true;
1957                 }
1958             }
1959         }
1960         CHECK_FATAL(isRet, "SelectAggDassign: NYI");
1961     }
1962 }
1963 
GetPointedToType(const MIRPtrType & pointerType)1964 static MIRType *GetPointedToType(const MIRPtrType &pointerType)
1965 {
1966     MIRType *aType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(pointerType.GetPointedTyIdx());
1967     if (aType->GetKind() == kTypeArray) {
1968         MIRArrayType *arrayType = static_cast<MIRArrayType *>(aType);
1969         return GlobalTables::GetTypeTable().GetTypeFromTyIdx(arrayType->GetElemTyIdx());
1970     }
1971     if (aType->GetKind() == kTypeFArray || aType->GetKind() == kTypeJArray) {
1972         MIRFarrayType *farrayType = static_cast<MIRFarrayType *>(aType);
1973         return GlobalTables::GetTypeTable().GetTypeFromTyIdx(farrayType->GetElemTyIdx());
1974     }
1975     return aType;
1976 }
1977 
SelectIassign(IassignNode & stmt)1978 void AArch64CGFunc::SelectIassign(IassignNode &stmt)
1979 {
1980     int32 offset = 0;
1981     MIRPtrType *pointerType = static_cast<MIRPtrType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(stmt.GetTyIdx()));
1982     DEBUG_ASSERT(pointerType != nullptr, "expect a pointer type at iassign node");
1983     MIRType *pointedType = nullptr;
1984     bool isRefField = false;
1985     AArch64isa::MemoryOrdering memOrd = AArch64isa::kMoNone;
1986 
1987     if (stmt.GetFieldID() != 0) {
1988         MIRType *pointedTy = GlobalTables::GetTypeTable().GetTypeFromTyIdx(pointerType->GetPointedTyIdx());
1989         MIRStructType *structType = nullptr;
1990         if (pointedTy->GetKind() != kTypeJArray) {
1991             structType = static_cast<MIRStructType *>(pointedTy);
1992         } else {
1993             /* it's a Jarray type. using it's parent's field info: java.lang.Object */
1994             structType = static_cast<MIRJarrayType *>(pointedTy)->GetParentType();
1995         }
1996         DEBUG_ASSERT(structType != nullptr, "SelectIassign: non-zero fieldID for non-structure");
1997         pointedType = structType->GetFieldType(stmt.GetFieldID());
1998         offset = GetBecommon().GetFieldOffset(*structType, stmt.GetFieldID()).first;
1999         isRefField = GetBecommon().IsRefField(*structType, stmt.GetFieldID());
2000     } else {
2001         pointedType = GetPointedToType(*pointerType);
2002         if (GetFunction().IsJava() && (pointedType->GetKind() == kTypePointer)) {
2003             MIRType *nextPointedType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(
2004                 static_cast<MIRPtrType *>(pointedType)->GetPointedTyIdx());
2005             if (nextPointedType->GetKind() != kTypeScalar) {
2006                 isRefField = true; /* write into an object array or a high-dimensional array */
2007             }
2008         }
2009     }
2010 
2011     PrimType styp = stmt.GetRHS()->GetPrimType();
2012     Operand *valOpnd = HandleExpr(stmt, *stmt.GetRHS());
2013     Operand &srcOpnd = LoadIntoRegister(*valOpnd, (IsPrimitiveInteger(styp) || IsPrimitiveVectorInteger(styp)),
2014                                         GetPrimTypeBitSize(styp));
2015 
2016     PrimType destType = pointedType->GetPrimType();
2017     if (destType == PTY_agg) {
2018         destType = PTY_a64;
2019     }
2020     if (IsPrimitiveVector(styp)) { /* a vector type */
2021         destType = styp;
2022     }
2023     DEBUG_ASSERT(stmt.Opnd(0) != nullptr, "null ptr check");
2024     MemOperand &memOpnd = CreateMemOpnd(destType, stmt, *stmt.Opnd(0), offset);
2025     auto dataSize = GetPrimTypeBitSize(destType);
2026     memOpnd = memOpnd.IsOffsetMisaligned(dataSize) ? ConstraintOffsetToSafeRegion(dataSize, memOpnd) : memOpnd;
2027     if (isVolStore && memOpnd.GetAddrMode() == MemOperand::kAddrModeBOi) {
2028         memOrd = AArch64isa::kMoRelease;
2029         isVolStore = false;
2030     }
2031 
2032     if (memOrd == AArch64isa::kMoNone) {
2033         SelectCopy(memOpnd, destType, srcOpnd, destType);
2034     } else {
2035         AArch64CGFunc::SelectStoreRelease(memOpnd, destType, srcOpnd, destType, memOrd, false);
2036     }
2037     GetCurBB()->GetLastInsn()->MarkAsAccessRefField(isRefField);
2038 }
2039 
SelectIassignoff(IassignoffNode & stmt)2040 void AArch64CGFunc::SelectIassignoff(IassignoffNode &stmt)
2041 {
2042     int32 offset = stmt.GetOffset();
2043     PrimType destType = stmt.GetPrimType();
2044 
2045     MemOperand &memOpnd = CreateMemOpnd(destType, stmt, *stmt.GetBOpnd(0), offset);
2046     auto dataSize = GetPrimTypeBitSize(destType);
2047     memOpnd = memOpnd.IsOffsetMisaligned(dataSize) ? ConstraintOffsetToSafeRegion(dataSize, memOpnd) : memOpnd;
2048     Operand *valOpnd = HandleExpr(stmt, *stmt.GetBOpnd(1));
2049     Operand &srcOpnd = LoadIntoRegister(*valOpnd, true, GetPrimTypeBitSize(destType));
2050     SelectCopy(memOpnd, destType, srcOpnd, destType);
2051 }
2052 
GenLmbcFpMemOperand(int32 offset,uint32 byteSize,AArch64reg baseRegno)2053 MemOperand *AArch64CGFunc::GenLmbcFpMemOperand(int32 offset, uint32 byteSize, AArch64reg baseRegno)
2054 {
2055     MemOperand *memOpnd;
2056     RegOperand *rfp = &GetOrCreatePhysicalRegisterOperand(baseRegno, k64BitSize, kRegTyInt);
2057     uint32 bitlen = byteSize * kBitsPerByte;
2058     if (offset < 0 && offset < kNegative256BitSize) {
2059         RegOperand *baseOpnd = &CreateRegisterOperandOfType(PTY_a64);
2060         ImmOperand &immOpnd = CreateImmOperand(offset, k32BitSize, true);
2061         Insn &addInsn = GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *baseOpnd, *rfp, immOpnd);
2062         GetCurBB()->AppendInsn(addInsn);
2063         OfstOperand *offsetOpnd = &CreateOfstOpnd(0, k32BitSize);
2064         memOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, bitlen, baseOpnd, nullptr, offsetOpnd, nullptr);
2065     } else {
2066         OfstOperand *offsetOpnd = &CreateOfstOpnd(static_cast<uint64>(static_cast<int64>(offset)), k32BitSize);
2067         memOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, bitlen, rfp, nullptr, offsetOpnd, nullptr);
2068     }
2069     memOpnd->SetStackMem(true);
2070     return memOpnd;
2071 }
2072 
SelectIassignfpoff(IassignFPoffNode & stmt,Operand & opnd)2073 void AArch64CGFunc::SelectIassignfpoff(IassignFPoffNode &stmt, Operand &opnd)
2074 {
2075     int32 offset = stmt.GetOffset();
2076     PrimType primType = stmt.GetPrimType();
2077     MIRType *rType = GetLmbcCallReturnType();
2078     bool isPureFpStruct = false;
2079     uint32 numRegs = 0;
2080     if (rType && rType->GetPrimType() == PTY_agg && opnd.IsRegister() &&
2081         static_cast<RegOperand &>(opnd).IsPhysicalRegister()) {
2082         CHECK_FATAL(rType->GetSize() <= k16BitSize, "SelectIassignfpoff invalid agg size");
2083         uint32 fpSize;
2084         numRegs = FloatParamRegRequired(static_cast<MIRStructType *>(rType), fpSize);
2085         if (numRegs) {
2086             primType = (fpSize == k4ByteSize) ? PTY_f32 : PTY_f64;
2087             isPureFpStruct = true;
2088         }
2089     }
2090     uint32 byteSize = GetPrimTypeSize(primType);
2091     uint32 bitlen = byteSize * kBitsPerByte;
2092     if (isPureFpStruct) {
2093         for (uint32 i = 0; i < numRegs; ++i) {
2094             MemOperand *memOpnd = GenLmbcFpMemOperand(offset + static_cast<int32>(i * byteSize), byteSize);
2095             RegOperand &srcOpnd = GetOrCreatePhysicalRegisterOperand(AArch64reg(V0 + i), bitlen, kRegTyFloat);
2096             MOperator mOp = PickStInsn(bitlen, primType);
2097             Insn &store = GetInsnBuilder()->BuildInsn(mOp, srcOpnd, *memOpnd);
2098             GetCurBB()->AppendInsn(store);
2099         }
2100     } else {
2101         Operand &srcOpnd = LoadIntoRegister(opnd, primType);
2102         MemOperand *memOpnd = GenLmbcFpMemOperand(offset, byteSize);
2103         MOperator mOp = PickStInsn(bitlen, primType);
2104         Insn &store = GetInsnBuilder()->BuildInsn(mOp, srcOpnd, *memOpnd);
2105         GetCurBB()->AppendInsn(store);
2106     }
2107 }
2108 
2109 /* Load and assign to a new register. To be moved to the correct call register OR stack
2110    location in LmbcSelectParmList */
SelectIassignspoff(PrimType pTy,int32 offset,Operand & opnd)2111 void AArch64CGFunc::SelectIassignspoff(PrimType pTy, int32 offset, Operand &opnd)
2112 {
2113     if (GetLmbcArgInfo() == nullptr) {
2114         LmbcArgInfo *p = memPool->New<LmbcArgInfo>(*GetFuncScopeAllocator());
2115         SetLmbcArgInfo(p);
2116     }
2117     uint32 byteLen = GetPrimTypeSize(pTy);
2118     uint32 bitLen = byteLen * kBitsPerByte;
2119     RegType regTy = GetRegTyFromPrimTy(pTy);
2120     int32 curRegArgs = GetLmbcArgsInRegs(regTy);
2121     if (curRegArgs < static_cast<int32>(k8ByteSize)) {
2122         RegOperand *res = &CreateVirtualRegisterOperand(NewVReg(regTy, byteLen));
2123         SelectCopy(*res, pTy, opnd, pTy);
2124         SetLmbcArgInfo(res, pTy, offset, 1);
2125     } else {
2126         /* Move into allocated space */
2127         Operand &memOpd = CreateMemOpnd(RSP, offset, byteLen);
2128         Operand &reg = LoadIntoRegister(opnd, pTy);
2129         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(bitLen, pTy), reg, memOpd));
2130     }
2131     IncLmbcArgsInRegs(regTy); /* num of args in registers */
2132     IncLmbcTotalArgs();       /* num of args */
2133 }
2134 
2135 /* Search for CALL/ICALL/ICALLPROTO node, must be called from a blkassignoff node */
LmbcGetAggTyFromCallSite(StmtNode * stmt,std::vector<TyIdx> ** parmList) const2136 MIRType *AArch64CGFunc::LmbcGetAggTyFromCallSite(StmtNode *stmt, std::vector<TyIdx> **parmList) const
2137 {
2138     for (; stmt != nullptr; stmt = stmt->GetNext()) {
2139         if (stmt->GetOpCode() == OP_call || stmt->GetOpCode() == OP_icallproto) {
2140             break;
2141         }
2142     }
2143     CHECK_FATAL(stmt && (stmt->GetOpCode() == OP_call || stmt->GetOpCode() == OP_icallproto),
2144                 "blkassign sp not followed by call");
2145     uint32 nargs = GetLmbcTotalArgs();
2146     MIRType *ty = nullptr;
2147     if (stmt->GetOpCode() == OP_call) {
2148         CallNode *callNode = static_cast<CallNode *>(stmt);
2149         MIRFunction *fn = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(callNode->GetPUIdx());
2150         if (fn->GetFormalCount() > 0) {
2151             ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(fn->GetFormalDefVec()[nargs].formalTyIdx);
2152         }
2153         *parmList = &fn->GetParamTypes();
2154         // would return null if the actual parameter is bogus
2155     } else if (stmt->GetOpCode() == OP_icallproto) {
2156         IcallNode *icallproto = static_cast<IcallNode *>(stmt);
2157         MIRType *type = GlobalTables::GetTypeTable().GetTypeFromTyIdx(icallproto->GetRetTyIdx());
2158         MIRFuncType *fType = static_cast<MIRFuncType *>(type);
2159         ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(fType->GetNthParamType(nargs));
2160         *parmList = &fType->GetParamTypeList();
2161     } else {
2162         CHECK_FATAL(stmt->GetOpCode() == OP_icallproto, "LmbcGetAggTyFromCallSite:: unexpected call operator");
2163     }
2164     return ty;
2165 }
2166 
2167 /* return true if blkassignoff for return, false otherwise */
LmbcSmallAggForRet(const BlkassignoffNode & bNode,const Operand * src)2168 bool AArch64CGFunc::LmbcSmallAggForRet(const BlkassignoffNode &bNode, const Operand *src)
2169 {
2170     PrimType pTy;
2171     uint32 size = 0;
2172     AArch64reg regno = static_cast<AArch64reg>(static_cast<const RegOperand *>(src)->GetRegisterNumber());
2173     MIRFunction *func = &GetFunction();
2174 
2175     if (func->IsReturnStruct()) {
2176         /* This blkassignoff is for struct return? */
2177         uint32 loadSize;
2178         uint32 numRegs = 0;
2179         if (bNode.GetNext()->GetOpCode() == OP_return) {
2180             MIRStructType *ty = static_cast<MIRStructType *>(
2181                 GlobalTables::GetTypeTable().GetTypeFromTyIdx(func->GetFuncRetStructTyIdx()));
2182             uint32 fpregs = FloatParamRegRequired(ty, size);
2183             if (fpregs > 0) {
2184                 /* pure floating point in agg */
2185                 numRegs = fpregs;
2186                 pTy = (size == k4ByteSize) ? PTY_f32 : PTY_f64;
2187                 loadSize = GetPrimTypeSize(pTy) * kBitsPerByte;
2188                 for (uint32 i = 0; i < fpregs; i++) {
2189                     int32 s = (i == 0) ? 0 : static_cast<int32>(i * size);
2190                     MemOperand &mem = CreateMemOpnd(regno, s, size * kBitsPerByte);
2191                     AArch64reg reg = static_cast<AArch64reg>(V0 + i);
2192                     RegOperand *res = &GetOrCreatePhysicalRegisterOperand(reg, loadSize, kRegTyFloat);
2193                     SelectCopy(*res, pTy, mem, pTy);
2194                 }
2195             } else {
2196                 /* int/float mixed */
2197                 numRegs = kRegNum2;
2198                 pTy = PTY_i64;
2199                 size = k4ByteSize;
2200                 switch (bNode.blockSize) {
2201                     case k1ByteSize:
2202                         pTy = PTY_i8;
2203                         break;
2204                     case k2ByteSize:
2205                         pTy = PTY_i16;
2206                         break;
2207                     case k4ByteSize:
2208                         pTy = PTY_i32;
2209                         break;
2210                     default:
2211                         size = k8ByteSize; /* pTy remains i64 */
2212                         break;
2213                 }
2214                 loadSize = GetPrimTypeSize(pTy) * kBitsPerByte;
2215                 MemOperand &mem = CreateMemOpnd(regno, 0, size * kBitsPerByte);
2216                 RegOperand *res = &GetOrCreatePhysicalRegisterOperand(R0, loadSize, kRegTyInt);
2217                 SelectCopy(*res, pTy, mem, pTy);
2218                 if (bNode.blockSize > static_cast<int32>(k8ByteSize)) {
2219                     MemOperand &newMem = CreateMemOpnd(regno, k8ByteSize, size * kBitsPerByte);
2220                     res = &GetOrCreatePhysicalRegisterOperand(R1, loadSize, kRegTyInt);
2221                     SelectCopy(*res, pTy, newMem, pTy);
2222                 }
2223             }
2224             bool intReg = fpregs == 0;
2225             for (uint32 i = 0; i < numRegs; i++) {
2226                 AArch64reg preg = static_cast<AArch64reg>((intReg ? R0 : V0) + i);
2227                 MOperator mop = intReg ? MOP_pseudo_ret_int : MOP_pseudo_ret_float;
2228                 RegOperand &dest = GetOrCreatePhysicalRegisterOperand(preg, loadSize, intReg ? kRegTyInt : kRegTyFloat);
2229                 Insn &pseudo = GetInsnBuilder()->BuildInsn(mop, dest);
2230                 GetCurBB()->AppendInsn(pseudo);
2231             }
2232             return true;
2233         }
2234     }
2235     return false;
2236 }
2237 
2238 /* return true if blkassignoff for return, false otherwise */
LmbcSmallAggForCall(BlkassignoffNode & bNode,const Operand * src,std::vector<TyIdx> ** parmList)2239 bool AArch64CGFunc::LmbcSmallAggForCall(BlkassignoffNode &bNode, const Operand *src, std::vector<TyIdx> **parmList)
2240 {
2241     AArch64reg regno = static_cast<AArch64reg>(static_cast<const RegOperand *>(src)->GetRegisterNumber());
2242     if (IsBlkassignForPush(bNode)) {
2243         PrimType pTy = PTY_i64;
2244         MIRStructType *ty = static_cast<MIRStructType *>(LmbcGetAggTyFromCallSite(&bNode, parmList));
2245         uint32 size = 0;
2246         uint32 fpregs = ty ? FloatParamRegRequired(ty, size) : 0; /* fp size determined */
2247         if (fpregs > 0) {
2248             /* pure floating point in agg */
2249             pTy = (size == k4ByteSize) ? PTY_f32 : PTY_f64;
2250             for (uint32 i = 0; i < fpregs; i++) {
2251                 int32 s = (i == 0) ? 0 : static_cast<int32>(i * size);
2252                 MemOperand &mem = CreateMemOpnd(regno, s, size * kBitsPerByte);
2253                 RegOperand *res = &CreateVirtualRegisterOperand(NewVReg(kRegTyFloat, size));
2254                 SelectCopy(*res, pTy, mem, pTy);
2255                 SetLmbcArgInfo(res, pTy, 0, static_cast<int32>(fpregs));
2256                 IncLmbcArgsInRegs(kRegTyFloat);
2257             }
2258             IncLmbcTotalArgs();
2259             return true;
2260         } else if (bNode.blockSize <= static_cast<int32>(k16ByteSize)) {
2261             /* integer/mixed types in register/s */
2262             size = k4ByteSize;
2263             switch (bNode.blockSize) {
2264                 case k1ByteSize:
2265                     pTy = PTY_i8;
2266                     break;
2267                 case k2ByteSize:
2268                     pTy = PTY_i16;
2269                     break;
2270                 case k4ByteSize:
2271                     pTy = PTY_i32;
2272                     break;
2273                 default:
2274                     size = k8ByteSize; /* pTy remains i64 */
2275                     break;
2276             }
2277             MemOperand &mem = CreateMemOpnd(regno, 0, size * kBitsPerByte);
2278             RegOperand *res = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, size));
2279             SelectCopy(*res, pTy, mem, pTy);
2280             SetLmbcArgInfo(res, pTy, bNode.offset,
2281                            bNode.blockSize > static_cast<int32>(k8ByteSize) ? kThirdReg : kSecondReg);
2282             IncLmbcArgsInRegs(kRegTyInt);
2283             if (bNode.blockSize > static_cast<int32>(k8ByteSize)) {
2284                 MemOperand &newMem = CreateMemOpnd(regno, k8ByteSize, size * kBitsPerByte);
2285                 RegOperand *newRes = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, size));
2286                 SelectCopy(*newRes, pTy, newMem, pTy);
2287                 SetLmbcArgInfo(newRes, pTy, bNode.offset + k8ByteSizeInt, kThirdReg);
2288                 IncLmbcArgsInRegs(kRegTyInt);
2289             }
2290             IncLmbcTotalArgs();
2291             return true;
2292         }
2293     }
2294     return false;
2295 }
2296 
2297 /* This function is incomplete and may be removed when Lmbc IR is changed
2298    to have the lowerer figures out the address of the large agg to reside */
LmbcFindTotalStkUsed(std::vector<TyIdx> * paramList)2299 uint32 AArch64CGFunc::LmbcFindTotalStkUsed(std::vector<TyIdx> *paramList)
2300 {
2301     CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
2302     AArch64CallConvImpl parmlocator(GetBecommon());
2303     CCLocInfo pLoc;
2304     for (TyIdx tyIdx : *paramList) {
2305         MIRType *ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(tyIdx);
2306         (void)parmlocator.LocateNextParm(*ty, pLoc);
2307     }
2308     return 0;
2309 }
2310 
2311 /* All arguments passed as registers */
LmbcTotalRegsUsed()2312 uint32 AArch64CGFunc::LmbcTotalRegsUsed()
2313 {
2314     if (GetLmbcArgInfo() == nullptr) {
2315         return 0; /* no arg */
2316     }
2317     MapleVector<int32> &regs = GetLmbcCallArgNumOfRegs();
2318     MapleVector<PrimType> &types = GetLmbcCallArgTypes();
2319     uint32 iCnt = 0;
2320     uint32 fCnt = 0;
2321     for (uint32 i = 0; i < regs.size(); i++) {
2322         if (IsPrimitiveInteger(types[i])) {
2323             if ((iCnt + static_cast<uint32>(regs[i])) <= k8ByteSize) {
2324                 iCnt += static_cast<uint32>(regs[i]);
2325             };
2326         } else {
2327             if ((fCnt + static_cast<uint32>(regs[i])) <= k8ByteSize) {
2328                 fCnt += static_cast<uint32>(regs[i]);
2329             };
2330         }
2331     }
2332     return iCnt + fCnt;
2333 }
2334 
2335 /* If blkassignoff for argument, this function loads the agg arguments into
2336    virtual registers, disregard if there is sufficient physicall call
2337    registers. Argument > 16-bytes are copied to preset space and ptr
2338    result is loaded into virtual register.
2339    If blassign is not for argument, this function simply memcpy */
SelectBlkassignoff(BlkassignoffNode & bNode,Operand * src)2340 void AArch64CGFunc::SelectBlkassignoff(BlkassignoffNode &bNode, Operand *src)
2341 {
2342     CHECK_FATAL(src->GetKind() == Operand::kOpdRegister, "blkassign src type not in register");
2343     std::vector<TyIdx> *parmList;
2344     if (GetLmbcArgInfo() == nullptr) {
2345         LmbcArgInfo *p = memPool->New<LmbcArgInfo>(*GetFuncScopeAllocator());
2346         SetLmbcArgInfo(p);
2347     }
2348     if (LmbcSmallAggForRet(bNode, src)) {
2349         return;
2350     } else if (LmbcSmallAggForCall(bNode, src, &parmList)) {
2351         return;
2352     }
2353     Operand *dest = HandleExpr(bNode, *bNode.Opnd(0));
2354     RegOperand *regResult = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
2355     /* memcpy for agg assign OR large agg for arg/ret */
2356     int32 offset = bNode.offset;
2357     if (IsBlkassignForPush(bNode)) {
2358         /* large agg for call, addr to be pushed in SelectCall */
2359         offset = GetLmbcTotalStkUsed();
2360         if (offset < 0) {
2361             /* length of ALL stack based args for this call, this location is where the
2362                next large agg resides, its addr will then be passed */
2363             offset = LmbcFindTotalStkUsed(parmList) + LmbcTotalRegsUsed();
2364         }
2365         SetLmbcTotalStkUsed(offset + bNode.blockSize); /* next use */
2366         SetLmbcArgInfo(regResult, PTY_i64, 0, 1);      /* 1 reg for ptr */
2367         IncLmbcArgsInRegs(kRegTyInt);
2368         IncLmbcTotalArgs();
2369         /* copy large agg arg to offset below */
2370     }
2371     std::vector<Operand *> opndVec;
2372     opndVec.push_back(regResult);                                                                        /* result */
2373     opndVec.push_back(PrepareMemcpyParamOpnd(offset, *dest));                                            /* param 0 */
2374     opndVec.push_back(src);                                                                              /* param 1 */
2375     opndVec.push_back(PrepareMemcpyParamOpnd(static_cast<uint64>(static_cast<int64>(bNode.blockSize)))); /* param 2 */
2376     SelectLibCall("memcpy", opndVec, PTY_a64, PTY_a64);
2377 }
2378 
SelectAggIassign(IassignNode & stmt,Operand & AddrOpnd)2379 void AArch64CGFunc::SelectAggIassign(IassignNode &stmt, Operand &AddrOpnd)
2380 {
2381     DEBUG_ASSERT(stmt.Opnd(0) != nullptr, "null ptr check");
2382     Operand &lhsAddrOpnd = LoadIntoRegister(AddrOpnd, stmt.Opnd(0)->GetPrimType());
2383     uint32 lhsOffset = 0;
2384     MIRType *stmtType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(stmt.GetTyIdx());
2385     MIRPtrType *lhsPointerType = static_cast<MIRPtrType *>(stmtType);
2386     bool loadToRegs4StructReturn = false;
2387     if (mirModule.CurFunction()->StructReturnedInRegs()) {
2388         MIRSymbol *retSt = mirModule.CurFunction()->GetFormal(0);
2389         if (stmt.Opnd(0)->GetOpCode() == OP_dread) {
2390             DreadNode *dread = static_cast<DreadNode *>(stmt.Opnd(0));
2391             MIRSymbol *addrSym = mirModule.CurFunction()->GetLocalOrGlobalSymbol(dread->GetStIdx());
2392             loadToRegs4StructReturn = (retSt == addrSym);
2393         }
2394     }
2395     MIRType *lhsType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(lhsPointerType->GetPointedTyIdx());
2396     if (stmt.GetFieldID() != 0) {
2397         MIRStructType *structType = static_cast<MIRStructType *>(lhsType);
2398         DEBUG_ASSERT(structType != nullptr, "SelectAggIassign: non-zero fieldID for non-structure");
2399         lhsType = structType->GetFieldType(stmt.GetFieldID());
2400         lhsOffset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, stmt.GetFieldID()).first);
2401     } else if (lhsType->GetKind() == kTypeArray) {
2402 #if DEBUG
2403         MIRArrayType *arrayLhsType = static_cast<MIRArrayType *>(lhsType);
2404         /* access an array element */
2405         MIRType *lhsType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(arrayLhsType->GetElemTyIdx());
2406         MIRTypeKind typeKind = lhsType->GetKind();
2407         DEBUG_ASSERT(((typeKind == kTypeScalar) || (typeKind == kTypeStruct) || (typeKind == kTypeClass) ||
2408                       (typeKind == kTypePointer)),
2409                      "unexpected array element type in iassign");
2410 #endif
2411     } else if (lhsType->GetKind() == kTypeFArray) {
2412 #if DEBUG
2413         MIRFarrayType *farrayLhsType = static_cast<MIRFarrayType *>(lhsType);
2414         /* access an array element */
2415         MIRType *lhsElemType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(farrayLhsType->GetElemTyIdx());
2416         MIRTypeKind typeKind = lhsElemType->GetKind();
2417         DEBUG_ASSERT(((typeKind == kTypeScalar) || (typeKind == kTypeStruct) || (typeKind == kTypeClass) ||
2418                       (typeKind == kTypePointer)),
2419                      "unexpected array element type in iassign");
2420 #endif
2421     }
2422     uint32 lhsAlign = GetBecommon().GetTypeAlign(lhsType->GetTypeIndex());
2423     uint64 lhsSize = GetBecommon().GetTypeSize(lhsType->GetTypeIndex());
2424 
2425     uint32 rhsAlign;
2426     uint32 alignUsed;
2427     uint32 rhsOffset = 0;
2428     if (stmt.GetRHS()->GetOpCode() == OP_dread) {
2429         AddrofNode *rhsDread = static_cast<AddrofNode *>(stmt.GetRHS());
2430         MIRSymbol *rhsSymbol = GetFunction().GetLocalOrGlobalSymbol(rhsDread->GetStIdx());
2431         MIRType *rhsType = rhsSymbol->GetType();
2432         if (rhsDread->GetFieldID() != 0) {
2433             MIRStructType *structType = static_cast<MIRStructType *>(rhsSymbol->GetType());
2434             DEBUG_ASSERT(structType != nullptr, "SelectAggIassign: non-zero fieldID for non-structure");
2435             rhsType = structType->GetFieldType(rhsDread->GetFieldID());
2436             rhsOffset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, rhsDread->GetFieldID()).first);
2437         }
2438         if (loadToRegs4StructReturn) {
2439             /* generate move to regs for agg return */
2440             CHECK_FATAL(lhsSize <= k16ByteSize, "SelectAggIassign: illegal struct size");
2441             CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
2442             AArch64CallConvImpl parmlocator(GetBecommon());
2443             CCLocInfo pLoc;
2444             parmlocator.LocateNextParm(*lhsType, pLoc, true, GetBecommon().GetMIRModule().CurFunction());
2445             /* aggregates are 8 byte aligned. */
2446             Operand *rhsmemopnd = nullptr;
2447             RegOperand *result[kFourRegister]; /* up to 2 int or 4 fp */
2448             uint32 loadSize;
2449             uint32 numRegs;
2450             RegType regType;
2451             PrimType retPty;
2452             bool fpParm = false;
2453             if (pLoc.numFpPureRegs) {
2454                 loadSize = pLoc.fpSize;
2455                 numRegs = pLoc.numFpPureRegs;
2456                 fpParm = true;
2457                 regType = kRegTyFloat;
2458                 retPty = (pLoc.fpSize == k4ByteSize) ? PTY_f32 : PTY_f64;
2459             } else {
2460                 if (CGOptions::IsBigEndian()) {
2461                     loadSize = k8ByteSize;
2462                     numRegs = (lhsSize <= k8ByteSize) ? kOneRegister : kTwoRegister;
2463                     regType = kRegTyInt;
2464                     retPty = PTY_u64;
2465                 } else {
2466                     loadSize = (lhsSize <= k4ByteSize) ? k4ByteSize : k8ByteSize;
2467                     numRegs = (lhsSize <= k8ByteSize) ? kOneRegister : kTwoRegister;
2468                     regType = kRegTyInt;
2469                     retPty = PTY_u32;
2470                 }
2471             }
2472             bool parmCopy = IsParamStructCopy(*rhsSymbol);
2473             for (uint32 i = 0; i < numRegs; i++) {
2474                 if (parmCopy) {
2475                     rhsmemopnd = &LoadStructCopyBase(
2476                         *rhsSymbol, (rhsOffset + static_cast<int64>(i * (fpParm ? loadSize : k8ByteSize))),
2477                         static_cast<int>(loadSize * kBitsPerByte));
2478                 } else {
2479                     rhsmemopnd = &GetOrCreateMemOpnd(
2480                         *rhsSymbol, (rhsOffset + static_cast<int64>(i * (fpParm ? loadSize : k8ByteSize))),
2481                         (loadSize * kBitsPerByte));
2482                 }
2483                 result[i] = &CreateVirtualRegisterOperand(NewVReg(regType, loadSize));
2484                 MOperator mop1 = PickLdInsn(loadSize * kBitsPerByte, retPty);
2485                 Insn &ld = GetInsnBuilder()->BuildInsn(mop1, *(result[i]), *rhsmemopnd);
2486                 GetCurBB()->AppendInsn(ld);
2487             }
2488             AArch64reg regs[kFourRegister];
2489             regs[kFirstReg] = static_cast<AArch64reg>(pLoc.reg0);
2490             regs[kSecondReg] = static_cast<AArch64reg>(pLoc.reg1);
2491             regs[kThirdReg] = static_cast<AArch64reg>(pLoc.reg2);
2492             regs[kFourthReg] = static_cast<AArch64reg>(pLoc.reg3);
2493             for (uint32 i = 0; i < numRegs; i++) {
2494                 AArch64reg preg;
2495                 MOperator mop2;
2496                 if (fpParm) {
2497                     preg = regs[i];
2498                     mop2 = (loadSize == k4ByteSize) ? MOP_xvmovs : MOP_xvmovd;
2499                 } else {
2500                     preg = (i == 0 ? R0 : R1);
2501                     mop2 = (loadSize == k4ByteSize) ? MOP_wmovrr : MOP_xmovrr;
2502                 }
2503                 RegOperand &dest = GetOrCreatePhysicalRegisterOperand(preg, (loadSize * kBitsPerByte), regType);
2504                 Insn &mov = GetInsnBuilder()->BuildInsn(mop2, dest, *(result[i]));
2505                 GetCurBB()->AppendInsn(mov);
2506             }
2507             /* Create artificial dependency to extend the live range */
2508             for (uint32 i = 0; i < numRegs; i++) {
2509                 AArch64reg preg;
2510                 MOperator mop3;
2511                 if (fpParm) {
2512                     preg = regs[i];
2513                     mop3 = MOP_pseudo_ret_float;
2514                 } else {
2515                     preg = (i == 0 ? R0 : R1);
2516                     mop3 = MOP_pseudo_ret_int;
2517                 }
2518                 RegOperand &dest = GetOrCreatePhysicalRegisterOperand(preg, loadSize * kBitsPerByte, regType);
2519                 Insn &pseudo = GetInsnBuilder()->BuildInsn(mop3, dest);
2520                 GetCurBB()->AppendInsn(pseudo);
2521             }
2522             return;
2523         }
2524         rhsAlign = GetBecommon().GetTypeAlign(rhsType->GetTypeIndex());
2525         alignUsed = std::min(lhsAlign, rhsAlign);
2526         DEBUG_ASSERT(alignUsed != 0, "expect non-zero");
2527         uint32 copySize = GetAggCopySize(rhsOffset, lhsOffset, alignUsed);
2528         MemOperand *rhsBaseMemOpnd;
2529         if (IsParamStructCopy(*rhsSymbol)) {
2530             rhsBaseMemOpnd = &LoadStructCopyBase(*rhsSymbol, rhsOffset, static_cast<int>(copySize * k8BitSize));
2531         } else {
2532             rhsBaseMemOpnd = GenLargeAggFormalMemOpnd(*rhsSymbol, copySize, rhsOffset, true);
2533         }
2534         RegOperand *rhsBaseReg = rhsBaseMemOpnd->GetBaseRegister();
2535         int64 rhsOffsetVal = rhsBaseMemOpnd->GetOffsetOperand()->GetValue();
2536         bool rhsIsLo12 = (rhsBaseMemOpnd->GetAddrMode() == MemOperand::kAddrModeLo12Li);
2537         if (lhsSize > kParmMemcpySize) {
2538             std::vector<Operand *> opndVec;
2539             RegOperand *regResult = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
2540             opndVec.push_back(regResult); /* result */
2541 
2542             opndVec.push_back(PrepareMemcpyParamOpnd(static_cast<int64>(lhsOffset), lhsAddrOpnd)); /* param 0 */
2543 
2544             opndVec.push_back(PrepareMemcpyParamOpnd(rhsOffsetVal, *rhsBaseReg)); /* param 1 */
2545 
2546             opndVec.push_back(PrepareMemcpyParamOpnd(lhsSize)); /* param 2 */
2547 
2548             SelectLibCall("memcpy", opndVec, PTY_a64, PTY_a64);
2549 
2550             return;
2551         }
2552         for (uint32 i = 0; i < (lhsSize / copySize); ++i) {
2553             uint32 rhsBaseOffset = static_cast<uint32>(rhsOffsetVal + i * copySize);
2554             uint32 lhsBaseOffset = lhsOffset + i * copySize;
2555             MemOperand::AArch64AddressingMode addrMode =
2556                 rhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
2557             MIRSymbol *sym = rhsIsLo12 ? rhsSymbol : nullptr;
2558             OfstOperand &rhsOfstOpnd = GetOrCreateOfstOpnd(rhsBaseOffset, k32BitSize);
2559             MemOperand *rhsMemOpnd =
2560                 &GetOrCreateMemOpnd(addrMode, copySize * k8BitSize, rhsBaseReg, nullptr, &rhsOfstOpnd, sym);
2561             rhsMemOpnd = FixLargeMemOpnd(*rhsMemOpnd, copySize);
2562             /* generate the load */
2563             RegOperand &result = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, copySize)));
2564             MOperator mOpLDP = (copySize == k4BitSize) ? MOP_wldp : MOP_xldp;
2565             bool doPair = (!rhsIsLo12 && (copySize >= k4BitSize) && ((i + 1) < (lhsSize / copySize)));
2566             RegOperand *result1 = nullptr;
2567             if (doPair) {
2568                 regno_t vRegNO1 = NewVReg(kRegTyInt, std::max(4u, copySize));
2569                 result1 = &CreateVirtualRegisterOperand(vRegNO1);
2570                 rhsMemOpnd =
2571                     FixLargeMemOpnd(mOpLDP, *static_cast<MemOperand *>(rhsMemOpnd), result.GetSize(), kInsnThirdOpnd);
2572                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpLDP, result, *result1, *rhsMemOpnd));
2573             } else {
2574                 MOperator mOp = PickLdInsn(copySize * k8BitSize, PTY_u32);
2575                 rhsMemOpnd =
2576                     FixLargeMemOpnd(mOp, *static_cast<MemOperand *>(rhsMemOpnd), result.GetSize(), kInsnSecondOpnd);
2577                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd));
2578             }
2579             /* generate the store */
2580             OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(lhsBaseOffset, k32BitSize);
2581             MemOperand *lhsMemOpnd =
2582                 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, copySize * k8BitSize,
2583                                     static_cast<RegOperand *>(&lhsAddrOpnd), nullptr, &ofstOpnd, nullptr);
2584             if (doPair) {
2585                 MOperator mOpSTP = (copySize == k4BitSize) ? MOP_wstp : MOP_xstp;
2586                 lhsMemOpnd = FixLargeMemOpnd(mOpSTP, *lhsMemOpnd, result.GetSize(), kInsnThirdOpnd);
2587                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpSTP, result, *result1, *lhsMemOpnd));
2588                 i++;
2589             } else {
2590                 MOperator mOp = PickStInsn(copySize * k8BitSize, PTY_u32);
2591                 lhsMemOpnd = FixLargeMemOpnd(mOp, *lhsMemOpnd, copySize * k8BitSize, kInsnSecondOpnd);
2592                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *lhsMemOpnd));
2593             }
2594         }
2595         /* take care of extra content at the end less than the unit of alignUsed */
2596         uint64 lhsSizeCovered = (lhsSize / copySize) * copySize;
2597         uint32 newAlignUsed = copySize;
2598         while (lhsSizeCovered < lhsSize) {
2599             newAlignUsed = newAlignUsed >> 1;
2600             CHECK_FATAL(newAlignUsed != 0, "expect non-zero");
2601             if ((lhsSizeCovered + newAlignUsed) > lhsSize) {
2602                 continue;
2603             }
2604             MemOperand::AArch64AddressingMode addrMode =
2605                 rhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
2606             MIRSymbol *sym = rhsIsLo12 ? rhsSymbol : nullptr;
2607             OfstOperand &rhsOfstOpnd =
2608                 GetOrCreateOfstOpnd(lhsSizeCovered + static_cast<uint64>(rhsOffsetVal), k32BitSize);
2609             MemOperand *rhsMemOpnd =
2610                 &GetOrCreateMemOpnd(addrMode, newAlignUsed * k8BitSize, rhsBaseReg, nullptr, &rhsOfstOpnd, sym);
2611             /* generate the load */
2612             Operand &result = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, newAlignUsed)));
2613             MOperator mOp = PickLdInsn(newAlignUsed * k8BitSize, PTY_u32);
2614             rhsMemOpnd = FixLargeMemOpnd(mOp, *rhsMemOpnd, newAlignUsed * k8BitSize, kInsnSecondOpnd);
2615             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd));
2616             /* generate the store */
2617             OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(lhsOffset + lhsSizeCovered, k32BitSize);
2618             MemOperand &lhsMemOpnd = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, newAlignUsed * k8BitSize,
2619                                                         static_cast<RegOperand *>(&lhsAddrOpnd), nullptr, &ofstOpnd,
2620                                                         static_cast<MIRSymbol *>(nullptr));
2621             mOp = PickStInsn(newAlignUsed * k8BitSize, PTY_u32);
2622             lhsMemOpnd = *FixLargeMemOpnd(mOp, lhsMemOpnd, newAlignUsed * k8BitSize, kInsnSecondOpnd);
2623             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, lhsMemOpnd));
2624             lhsSizeCovered += newAlignUsed;
2625         }
2626     } else { /* rhs is iread */
2627         DEBUG_ASSERT(stmt.GetRHS()->GetOpCode() == OP_iread, "SelectAggDassign: NYI");
2628         IreadNode *rhsIread = static_cast<IreadNode *>(stmt.GetRHS());
2629         RegOperand *rhsAddrOpnd = static_cast<RegOperand *>(HandleExpr(*rhsIread, *rhsIread->Opnd(0)));
2630         rhsAddrOpnd = &LoadIntoRegister(*rhsAddrOpnd, rhsIread->Opnd(0)->GetPrimType());
2631         MIRPtrType *rhsPointerType =
2632             static_cast<MIRPtrType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(rhsIread->GetTyIdx()));
2633         MIRType *rhsType = static_cast<MIRStructType *>(
2634             GlobalTables::GetTypeTable().GetTypeFromTyIdx(rhsPointerType->GetPointedTyIdx()));
2635         bool isRefField = false;
2636         if (rhsIread->GetFieldID() != 0) {
2637             MIRStructType *rhsStructType = static_cast<MIRStructType *>(rhsType);
2638             DEBUG_ASSERT(rhsStructType, "SelectAggDassign: non-zero fieldID for non-structure");
2639             rhsType = rhsStructType->GetFieldType(rhsIread->GetFieldID());
2640             rhsOffset = static_cast<uint32>(GetBecommon().GetFieldOffset(*rhsStructType, rhsIread->GetFieldID()).first);
2641             isRefField = GetBecommon().IsRefField(*rhsStructType, rhsIread->GetFieldID());
2642         }
2643         if (loadToRegs4StructReturn) {
2644             /* generate move to regs. */
2645             CHECK_FATAL(lhsSize <= k16ByteSize, "SelectAggIassign: illegal struct size");
2646             RegOperand *result[kTwoRegister]; /* maximum 16 bytes, 2 registers */
2647             uint32 loadSize;
2648             if (CGOptions::IsBigEndian()) {
2649                 loadSize = k8ByteSize;
2650             } else {
2651                 loadSize = (lhsSize <= k4ByteSize) ? k4ByteSize : k8ByteSize;
2652             }
2653             uint32 numRegs = (lhsSize <= k8ByteSize) ? kOneRegister : kTwoRegister;
2654             for (uint32 i = 0; i < numRegs; i++) {
2655                 OfstOperand *rhsOffOpnd = &GetOrCreateOfstOpnd(rhsOffset + i * loadSize, loadSize * kBitsPerByte);
2656                 Operand &rhsmemopnd = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, loadSize * kBitsPerByte, rhsAddrOpnd,
2657                                                          nullptr, rhsOffOpnd, nullptr);
2658                 result[i] = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, loadSize));
2659                 MOperator mop1 = PickLdInsn(loadSize * kBitsPerByte, PTY_u32);
2660                 Insn &ld = GetInsnBuilder()->BuildInsn(mop1, *(result[i]), rhsmemopnd);
2661                 ld.MarkAsAccessRefField(isRefField);
2662                 GetCurBB()->AppendInsn(ld);
2663             }
2664             for (uint32 i = 0; i < numRegs; i++) {
2665                 AArch64reg preg = (i == 0 ? R0 : R1);
2666                 RegOperand &dest = GetOrCreatePhysicalRegisterOperand(preg, loadSize * kBitsPerByte, kRegTyInt);
2667                 Insn &mov = GetInsnBuilder()->BuildInsn(MOP_xmovrr, dest, *(result[i]));
2668                 GetCurBB()->AppendInsn(mov);
2669             }
2670             /* Create artificial dependency to extend the live range */
2671             for (uint32 i = 0; i < numRegs; i++) {
2672                 AArch64reg preg = (i == 0 ? R0 : R1);
2673                 RegOperand &dest = GetOrCreatePhysicalRegisterOperand(preg, loadSize * kBitsPerByte, kRegTyInt);
2674                 Insn &pseudo = GetInsnBuilder()->BuildInsn(MOP_pseudo_ret_int, dest);
2675                 GetCurBB()->AppendInsn(pseudo);
2676             }
2677             return;
2678         }
2679         rhsAlign = GetBecommon().GetTypeAlign(rhsType->GetTypeIndex());
2680         alignUsed = std::min(lhsAlign, rhsAlign);
2681         DEBUG_ASSERT(alignUsed != 0, "expect non-zero");
2682         uint32 copySize = GetAggCopySize(rhsOffset, lhsOffset, alignUsed);
2683         if (lhsSize > kParmMemcpySize) {
2684             std::vector<Operand *> opndVec;
2685             RegOperand *regResult = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
2686             opndVec.push_back(regResult); /* result */
2687 
2688             opndVec.push_back(PrepareMemcpyParamOpnd(static_cast<int64>(lhsOffset), lhsAddrOpnd)); /* param 0 */
2689 
2690             opndVec.push_back(PrepareMemcpyParamOpnd(static_cast<int64>(rhsOffset), *rhsAddrOpnd)); /* param 1 */
2691 
2692             opndVec.push_back(PrepareMemcpyParamOpnd(lhsSize)); /* param 2 */
2693 
2694             SelectLibCall("memcpy", opndVec, PTY_a64, PTY_a64);
2695 
2696             return;
2697         }
2698         DEBUG_ASSERT(copySize != 0, "expect non-zero");
2699         for (uint32 i = 0; i < (lhsSize / copySize); i++) {
2700             /* generate the load */
2701             uint32 operandSize = copySize * k8BitSize;
2702             OfstOperand &rhsOfstOpnd = GetOrCreateOfstOpnd(rhsOffset + i * copySize, k32BitSize);
2703             MemOperand *rhsMemOpnd =
2704                 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, operandSize, static_cast<RegOperand *>(rhsAddrOpnd),
2705                                     nullptr, &rhsOfstOpnd, nullptr);
2706             RegOperand &result = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, copySize)));
2707             bool doPair = ((copySize >= k4BitSize) && ((i + 1) < (lhsSize / copySize)));
2708             Insn *insn = nullptr;
2709             RegOperand *result1 = nullptr;
2710             if (doPair) {
2711                 MOperator mOpLDP = (copySize == k4BitSize) ? MOP_wldp : MOP_xldp;
2712                 rhsMemOpnd =
2713                     FixLargeMemOpnd(mOpLDP, *static_cast<MemOperand *>(rhsMemOpnd), operandSize, kInsnThirdOpnd);
2714                 result1 = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, copySize)));
2715                 insn = &GetInsnBuilder()->BuildInsn(mOpLDP, result, *result1, *rhsMemOpnd);
2716             } else {
2717                 MOperator mOp = PickLdInsn(operandSize, PTY_u32);
2718                 rhsMemOpnd = FixLargeMemOpnd(mOp, *static_cast<MemOperand *>(rhsMemOpnd), operandSize, kInsnSecondOpnd);
2719                 insn = &GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd);
2720             }
2721             insn->MarkAsAccessRefField(isRefField);
2722             GetCurBB()->AppendInsn(*insn);
2723             /* generate the store */
2724             OfstOperand &lhsOfstOpnd = GetOrCreateOfstOpnd(lhsOffset + i * copySize, k32BitSize);
2725             MemOperand *lhsMemOpnd =
2726                 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, operandSize, static_cast<RegOperand *>(&lhsAddrOpnd),
2727                                     nullptr, &lhsOfstOpnd, nullptr);
2728             if (doPair) {
2729                 MOperator mOpSTP = (copySize == k4BitSize) ? MOP_wstp : MOP_xstp;
2730                 lhsMemOpnd =
2731                     FixLargeMemOpnd(mOpSTP, *static_cast<MemOperand *>(lhsMemOpnd), operandSize, kInsnThirdOpnd);
2732                 DEBUG_ASSERT(result1 != nullptr, "result1 should not be nullptr");
2733                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpSTP, result, *result1, *lhsMemOpnd));
2734                 i++;
2735             } else {
2736                 MOperator mOp = PickStInsn(operandSize, PTY_u32);
2737                 lhsMemOpnd = FixLargeMemOpnd(mOp, *static_cast<MemOperand *>(lhsMemOpnd), operandSize, kInsnSecondOpnd);
2738                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *lhsMemOpnd));
2739             }
2740         }
2741         /* take care of extra content at the end less than the unit */
2742         uint64 lhsSizeCovered = (lhsSize / copySize) * copySize;
2743         uint32 newAlignUsed = copySize;
2744         while (lhsSizeCovered < lhsSize) {
2745             newAlignUsed = newAlignUsed >> 1;
2746             CHECK_FATAL(newAlignUsed != 0, "expect non-zero");
2747             if ((lhsSizeCovered + newAlignUsed) > lhsSize) {
2748                 continue;
2749             }
2750             /* generate the load */
2751             OfstOperand &rhsOfstOpnd = GetOrCreateOfstOpnd(rhsOffset + lhsSizeCovered, k32BitSize);
2752             uint32 memOpndSize = newAlignUsed * k8BitSize;
2753             MemOperand *rhsMemOpnd =
2754                 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, memOpndSize, static_cast<RegOperand *>(rhsAddrOpnd),
2755                                     nullptr, &rhsOfstOpnd, nullptr);
2756             regno_t vRegNO = NewVReg(kRegTyInt, std::max(4u, newAlignUsed));
2757             RegOperand &result = CreateVirtualRegisterOperand(vRegNO);
2758             MOperator mOpLD = PickLdInsn(memOpndSize, PTY_u32);
2759             rhsMemOpnd = FixLargeMemOpnd(mOpLD, *rhsMemOpnd, memOpndSize, static_cast<uint32>(kInsnSecondOpnd));
2760             Insn &insn = GetInsnBuilder()->BuildInsn(mOpLD, result, *rhsMemOpnd);
2761             insn.MarkAsAccessRefField(isRefField);
2762             GetCurBB()->AppendInsn(insn);
2763             /* generate the store */
2764             OfstOperand &lhsOfstOpnd = GetOrCreateOfstOpnd(lhsOffset + lhsSizeCovered, k32BitSize);
2765             MemOperand *lhsMemOpnd =
2766                 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, memOpndSize, static_cast<RegOperand *>(&lhsAddrOpnd),
2767                                     nullptr, &lhsOfstOpnd, nullptr);
2768             MOperator mOpST = PickStInsn(memOpndSize, PTY_u32);
2769             lhsMemOpnd = FixLargeMemOpnd(mOpST, *lhsMemOpnd, memOpndSize, static_cast<uint32>(kInsnSecondOpnd));
2770             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpST, result, *lhsMemOpnd));
2771             lhsSizeCovered += newAlignUsed;
2772         }
2773     }
2774 }
2775 
SelectReturnSendOfStructInRegs(BaseNode * x)2776 void AArch64CGFunc::SelectReturnSendOfStructInRegs(BaseNode *x)
2777 {
2778     uint32 offset = 0;
2779     if (x->GetOpCode() == OP_dread) {
2780         DreadNode *dread = static_cast<DreadNode *>(x);
2781         MIRSymbol *sym = GetFunction().GetLocalOrGlobalSymbol(dread->GetStIdx());
2782         MIRType *mirType = sym->GetType();
2783         if (dread->GetFieldID() != 0) {
2784             MIRStructType *structType = static_cast<MIRStructType *>(mirType);
2785             mirType = structType->GetFieldType(dread->GetFieldID());
2786             offset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, dread->GetFieldID()).first);
2787         }
2788         uint32 typeSize = GetBecommon().GetTypeSize(mirType->GetTypeIndex());
2789         /* generate move to regs for agg return */
2790         CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
2791         AArch64CallConvImpl parmlocator(GetBecommon());
2792         CCLocInfo pLoc;
2793         (void)parmlocator.LocateNextParm(*mirType, pLoc, true, GetBecommon().GetMIRModule().CurFunction());
2794         /* aggregates are 8 byte aligned. */
2795         Operand *rhsmemopnd = nullptr;
2796         RegOperand *result[kFourRegister]; /* up to 2 int or 4 fp */
2797         uint32 loadSize;
2798         uint32 numRegs;
2799         RegType regType;
2800         PrimType retPty;
2801         bool fpParm = false;
2802         if (pLoc.numFpPureRegs) {
2803             loadSize = pLoc.fpSize;
2804             numRegs = pLoc.numFpPureRegs;
2805             fpParm = true;
2806             regType = kRegTyFloat;
2807             retPty = (pLoc.fpSize == k4ByteSize) ? PTY_f32 : PTY_f64;
2808         } else {
2809             if (CGOptions::IsBigEndian()) {
2810                 loadSize = k8ByteSize;
2811                 numRegs = (typeSize <= k8ByteSize) ? kOneRegister : kTwoRegister;
2812                 regType = kRegTyInt;
2813                 retPty = PTY_u64;
2814             } else {
2815                 loadSize = (typeSize <= k4ByteSize) ? k4ByteSize : k8ByteSize;
2816                 numRegs = (typeSize <= k8ByteSize) ? kOneRegister : kTwoRegister;
2817                 regType = kRegTyInt;
2818                 retPty = PTY_u32;
2819             }
2820         }
2821         bool parmCopy = IsParamStructCopy(*sym);
2822         for (uint32 i = 0; i < numRegs; i++) {
2823             if (parmCopy) {
2824                 rhsmemopnd =
2825                     &LoadStructCopyBase(*sym, (offset + static_cast<int64>(i * (fpParm ? loadSize : k8ByteSize))),
2826                                         static_cast<int>(loadSize * kBitsPerByte));
2827             } else {
2828                 rhsmemopnd =
2829                     &GetOrCreateMemOpnd(*sym, (offset + static_cast<int64>(i * (fpParm ? loadSize : k8ByteSize))),
2830                                         (loadSize * kBitsPerByte));
2831             }
2832             result[i] = &CreateVirtualRegisterOperand(NewVReg(regType, loadSize));
2833             MOperator mop1 = PickLdInsn(loadSize * kBitsPerByte, retPty);
2834             Insn &ld = GetInsnBuilder()->BuildInsn(mop1, *(result[i]), *rhsmemopnd);
2835             GetCurBB()->AppendInsn(ld);
2836         }
2837         AArch64reg regs[kFourRegister];
2838         regs[kFirstReg] = static_cast<AArch64reg>(pLoc.reg0);
2839         regs[kSecondReg] = static_cast<AArch64reg>(pLoc.reg1);
2840         regs[kThirdReg] = static_cast<AArch64reg>(pLoc.reg2);
2841         regs[kFourthReg] = static_cast<AArch64reg>(pLoc.reg3);
2842         RegOperand *dest;
2843         for (uint32 i = 0; i < numRegs; i++) {
2844             AArch64reg preg;
2845             MOperator mop2;
2846             if (fpParm) {
2847                 preg = regs[i];
2848                 mop2 = (loadSize == k4ByteSize) ? MOP_xvmovs : MOP_xvmovd;
2849             } else {
2850                 preg = (i == 0 ? R0 : R1);
2851                 mop2 = (loadSize == k4ByteSize) ? MOP_wmovrr : MOP_xmovrr;
2852             }
2853             dest = &GetOrCreatePhysicalRegisterOperand(preg, (loadSize * kBitsPerByte), regType);
2854             Insn &mov = GetInsnBuilder()->BuildInsn(mop2, *dest, *(result[i]));
2855             GetCurBB()->AppendInsn(mov);
2856         }
2857         /* Create artificial dependency to extend the live range */
2858         for (uint32 i = 0; i < numRegs; i++) {
2859             AArch64reg preg;
2860             MOperator mop3;
2861             if (fpParm) {
2862                 preg = regs[i];
2863                 mop3 = MOP_pseudo_ret_float;
2864             } else {
2865                 preg = (i == 0 ? R0 : R1);
2866                 mop3 = MOP_pseudo_ret_int;
2867             }
2868             dest = &GetOrCreatePhysicalRegisterOperand(preg, loadSize * kBitsPerByte, regType);
2869             Insn &pseudo = GetInsnBuilder()->BuildInsn(mop3, *dest);
2870             GetCurBB()->AppendInsn(pseudo);
2871         }
2872         return;
2873     } else if (x->GetOpCode() == OP_iread) {
2874         IreadNode *iread = static_cast<IreadNode *>(x);
2875         RegOperand *rhsAddrOpnd = static_cast<RegOperand *>(HandleExpr(*iread, *iread->Opnd(0)));
2876         rhsAddrOpnd = &LoadIntoRegister(*rhsAddrOpnd, iread->Opnd(0)->GetPrimType());
2877         MIRPtrType *ptrType =
2878             static_cast<MIRPtrType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(iread->GetTyIdx()));
2879         MIRType *mirType = static_cast<MIRStructType *>(ptrType->GetPointedType());
2880         bool isRefField = false;
2881         if (iread->GetFieldID() != 0) {
2882             MIRStructType *structType = static_cast<MIRStructType *>(mirType);
2883             mirType = structType->GetFieldType(iread->GetFieldID());
2884             offset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, iread->GetFieldID()).first);
2885             isRefField = GetBecommon().IsRefField(*structType, iread->GetFieldID());
2886         }
2887         uint32 typeSize = GetBecommon().GetTypeSize(mirType->GetTypeIndex());
2888         /* generate move to regs. */
2889         RegOperand *result[kTwoRegister]; /* maximum 16 bytes, 2 registers */
2890         uint32 loadSize;
2891         if (CGOptions::IsBigEndian()) {
2892             loadSize = k8ByteSize;
2893         } else {
2894             loadSize = (typeSize <= k4ByteSize) ? k4ByteSize : k8ByteSize;
2895         }
2896         uint32 numRegs = (typeSize <= k8ByteSize) ? kOneRegister : kTwoRegister;
2897         for (uint32 i = 0; i < numRegs; i++) {
2898             OfstOperand *rhsOffOpnd = &GetOrCreateOfstOpnd(offset + i * loadSize, loadSize * kBitsPerByte);
2899             Operand &rhsmemopnd = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, loadSize * kBitsPerByte, rhsAddrOpnd,
2900                                                      nullptr, rhsOffOpnd, nullptr);
2901             result[i] = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, loadSize));
2902             MOperator mop1 = PickLdInsn(loadSize * kBitsPerByte, PTY_u32);
2903             Insn &ld = GetInsnBuilder()->BuildInsn(mop1, *(result[i]), rhsmemopnd);
2904             ld.MarkAsAccessRefField(isRefField);
2905             GetCurBB()->AppendInsn(ld);
2906         }
2907         RegOperand *dest;
2908         for (uint32 i = 0; i < numRegs; i++) {
2909             AArch64reg preg = (i == 0 ? R0 : R1);
2910             dest = &GetOrCreatePhysicalRegisterOperand(preg, loadSize * kBitsPerByte, kRegTyInt);
2911             Insn &mov = GetInsnBuilder()->BuildInsn(MOP_xmovrr, *dest, *(result[i]));
2912             GetCurBB()->AppendInsn(mov);
2913         }
2914         /* Create artificial dependency to extend the live range */
2915         for (uint32 i = 0; i < numRegs; i++) {
2916             AArch64reg preg = (i == 0 ? R0 : R1);
2917             dest = &GetOrCreatePhysicalRegisterOperand(preg, loadSize * kBitsPerByte, kRegTyInt);
2918             Insn &pseudo = GetInsnBuilder()->BuildInsn(MOP_pseudo_ret_int, *dest);
2919             GetCurBB()->AppendInsn(pseudo);
2920         }
2921         return;
2922     } else {  // dummy return of 0 inserted by front-end at absence of return
2923         DEBUG_ASSERT(x->GetOpCode() == OP_constval, "SelectReturnSendOfStructInRegs: unexpected return operand");
2924         uint32 typeSize = GetPrimTypeSize(x->GetPrimType());
2925         RegOperand &dest = GetOrCreatePhysicalRegisterOperand(R0, typeSize * kBitsPerByte, kRegTyInt);
2926         ImmOperand &src = CreateImmOperand(0, k16BitSize, false);
2927         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wmovri32, dest, src));
2928         return;
2929     }
2930 }
2931 
SelectDread(const BaseNode & parent,DreadNode & expr)2932 Operand *AArch64CGFunc::SelectDread(const BaseNode &parent, DreadNode &expr)
2933 {
2934     MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(expr.GetStIdx());
2935     auto itr = stIdx2OverflowResult.find(expr.GetStIdx());
2936     if (itr != stIdx2OverflowResult.end()) {
2937         /* add_with_overflow / sub_with_overflow:
2938          * reg1: param1
2939          * reg2: param2
2940          * adds/subs reg3, reg1, reg2
2941          * cset reg4, vs
2942          * result is saved in std::pair<RegOperand*, RegOperand*>(reg3, reg4)
2943          */
2944         if (expr.GetFieldID() == 1) {
2945             return itr->second.first;
2946         } else {
2947             DEBUG_ASSERT(expr.GetFieldID() == 2, "only has 2 fileds for intrinsic overflow call result");
2948             return itr->second.second;
2949         }
2950     }
2951     if (symbol->IsEhIndex()) {
2952         CHECK_FATAL(false, "should not go here");
2953         MIRType *type = GlobalTables::GetTypeTable().GetTypeFromTyIdx(static_cast<TyIdx>(PTY_i32));
2954         /* use the second register return by __builtin_eh_return(). */
2955         AArch64CallConvImpl retLocator(GetBecommon());
2956         CCLocInfo retMech;
2957         retLocator.InitReturnInfo(*type, retMech);
2958         retLocator.SetupSecondRetReg(*type, retMech);
2959         return &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(retMech.GetReg1()), k64BitSize, kRegTyInt);
2960     }
2961 
2962     PrimType symType = symbol->GetType()->GetPrimType();
2963     uint32 offset = 0;
2964     bool parmCopy = false;
2965     if (expr.GetFieldID() != 0) {
2966         MIRStructType *structType = static_cast<MIRStructType *>(symbol->GetType());
2967         DEBUG_ASSERT(structType != nullptr, "SelectDread: non-zero fieldID for non-structure");
2968         symType = structType->GetFieldType(expr.GetFieldID())->GetPrimType();
2969         offset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, expr.GetFieldID()).first);
2970         parmCopy = IsParamStructCopy(*symbol);
2971     }
2972 
2973     uint32 dataSize = GetPrimTypeBitSize(symType);
2974     uint32 aggSize = 0;
2975     if (symType == PTY_agg) {
2976         if (expr.GetPrimType() == PTY_agg) {
2977             aggSize = static_cast<uint32>(GetBecommon().GetTypeSize(symbol->GetType()->GetTypeIndex().GetIdx()));
2978             dataSize = ((expr.GetFieldID() == 0) ? GetPointerSize() : aggSize) << k8BitShift;
2979         } else {
2980             dataSize = GetPrimTypeBitSize(expr.GetPrimType());
2981         }
2982     }
2983     MemOperand *memOpnd = nullptr;
2984     if (aggSize > k8ByteSize) {
2985         if (parent.op == OP_eval) {
2986             if (symbol->GetAttr(ATTR_volatile)) {
2987                 /* Need to generate loads for the upper parts of the struct. */
2988                 Operand &dest = GetZeroOpnd(k64BitSize);
2989                 uint32 numLoads = static_cast<uint32>(RoundUp(aggSize, k64BitSize) / k64BitSize);
2990                 for (uint32 o = 0; o < numLoads; ++o) {
2991                     if (parmCopy) {
2992                         memOpnd = &LoadStructCopyBase(*symbol, offset + o * GetPointerSize(), GetPointerSize());
2993                     } else {
2994                         memOpnd = &GetOrCreateMemOpnd(*symbol, offset + o * GetPointerSize(), GetPointerSize());
2995                     }
2996                     if (IsImmediateOffsetOutOfRange(*memOpnd, GetPointerSize())) {
2997                         memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, GetPointerSize());
2998                     }
2999                     SelectCopy(dest, PTY_u64, *memOpnd, PTY_u64);
3000                 }
3001             } else {
3002                 /* No side-effects.  No need to generate anything for eval. */
3003             }
3004         } else {
3005             if (expr.GetFieldID() != 0) {
3006                 CHECK_FATAL(false, "SelectDread: Illegal agg size");
3007             }
3008         }
3009     }
3010     if (parmCopy) {
3011         memOpnd = &LoadStructCopyBase(*symbol, offset, static_cast<int>(dataSize));
3012     } else {
3013         memOpnd = &GetOrCreateMemOpnd(*symbol, offset, dataSize);
3014     }
3015     if ((memOpnd->GetMemVaryType() == kNotVary) && IsImmediateOffsetOutOfRange(*memOpnd, dataSize)) {
3016         memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, dataSize);
3017     }
3018 
3019     PrimType resultType = expr.GetPrimType();
3020     RegOperand &resOpnd = GetOrCreateResOperand(parent, symType);
3021     /* a local register variable defined with a specified register */
3022     if (symbol->GetAsmAttr() != UStrIdx(0) && symbol->GetStorageClass() != kScPstatic &&
3023         symbol->GetStorageClass() != kScFstatic) {
3024         std::string regDesp = GlobalTables::GetUStrTable().GetStringFromStrIdx(symbol->GetAsmAttr());
3025         RegOperand &specifiedOpnd = GetOrCreatePhysicalRegisterOperand(regDesp);
3026         return &specifiedOpnd;
3027     }
3028     memOpnd = memOpnd->IsOffsetMisaligned(dataSize) ? &ConstraintOffsetToSafeRegion(dataSize, *memOpnd) : memOpnd;
3029     SelectCopy(resOpnd, resultType, *memOpnd, symType);
3030     return &resOpnd;
3031 }
3032 
SelectRegread(RegreadNode & expr)3033 RegOperand *AArch64CGFunc::SelectRegread(RegreadNode &expr)
3034 {
3035     PregIdx pregIdx = expr.GetRegIdx();
3036     if (IsSpecialPseudoRegister(pregIdx)) {
3037         /* if it is one of special registers */
3038         return &GetOrCreateSpecialRegisterOperand(-pregIdx, expr.GetPrimType());
3039     }
3040     RegOperand &reg = GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx));
3041     if (GetOpndFromPregIdx(pregIdx) == nullptr) {
3042         SetPregIdx2Opnd(pregIdx, reg);
3043     }
3044     if (expr.GetPrimType() == PTY_ref) {
3045         reg.SetIsReference(true);
3046         AddReferenceReg(reg.GetRegisterNumber());
3047     }
3048     if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) {
3049         MemOperand *src = GetPseudoRegisterSpillMemoryOperand(pregIdx);
3050         MIRPreg *preg = GetFunction().GetPregTab()->PregFromPregIdx(pregIdx);
3051         PrimType stype = preg->GetPrimType();
3052         uint32 srcBitLength = GetPrimTypeSize(stype) * kBitsPerByte;
3053         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(srcBitLength, stype), reg, *src));
3054     }
3055     return &reg;
3056 }
3057 
SelectAddrof(Operand & result,StImmOperand & stImm,FieldID field)3058 void AArch64CGFunc::SelectAddrof(Operand &result, StImmOperand &stImm, FieldID field)
3059 {
3060     const MIRSymbol *symbol = stImm.GetSymbol();
3061     if (symbol->GetStorageClass() == kScAuto) {
3062         SetStackProtectInfo(kAddrofStack);
3063     }
3064     if ((symbol->GetStorageClass() == kScAuto) || (symbol->GetStorageClass() == kScFormal)) {
3065         if (!CGOptions::IsQuiet()) {
3066             maple::LogInfo::MapleLogger(kLlErr)
3067                 << "Warning: we expect AddrOf with StImmOperand is not used for local variables";
3068         }
3069         AArch64SymbolAlloc *symLoc =
3070             static_cast<AArch64SymbolAlloc *>(GetMemlayout()->GetSymAllocInfo(symbol->GetStIndex()));
3071         ImmOperand *offset = nullptr;
3072         if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsArgsStkPassed) {
3073             offset = &CreateImmOperand(GetBaseOffset(*symLoc) + stImm.GetOffset(), k64BitSize, false, kUnAdjustVary);
3074         } else if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsRefLocals) {
3075             auto it = immOpndsRequiringOffsetAdjustmentForRefloc.find(symLoc);
3076             if (it != immOpndsRequiringOffsetAdjustmentForRefloc.end()) {
3077                 offset = (*it).second;
3078             } else {
3079                 offset = &CreateImmOperand(GetBaseOffset(*symLoc) + stImm.GetOffset(), k64BitSize, false);
3080                 immOpndsRequiringOffsetAdjustmentForRefloc[symLoc] = offset;
3081             }
3082         } else if (mirModule.IsJavaModule()) {
3083             auto it = immOpndsRequiringOffsetAdjustment.find(symLoc);
3084             if ((it != immOpndsRequiringOffsetAdjustment.end()) && (symbol->GetType()->GetPrimType() != PTY_agg)) {
3085                 offset = (*it).second;
3086             } else {
3087                 offset = &CreateImmOperand(GetBaseOffset(*symLoc) + stImm.GetOffset(), k64BitSize, false);
3088                 if (symbol->GetType()->GetKind() != kTypeClass) {
3089                     immOpndsRequiringOffsetAdjustment[symLoc] = offset;
3090                 }
3091             }
3092         } else {
3093             /* Do not cache modified symbol location */
3094             offset = &CreateImmOperand(GetBaseOffset(*symLoc) + stImm.GetOffset(), k64BitSize, false);
3095         }
3096 
3097         SelectAdd(result, *GetBaseReg(*symLoc), *offset, PTY_u64);
3098         if (GetCG()->GenerateVerboseCG()) {
3099             /* Add a comment */
3100             Insn *insn = GetCurBB()->GetLastInsn();
3101             std::string comm = "local/formal var: ";
3102             comm += symbol->GetName();
3103             insn->SetComment(comm);
3104         }
3105     } else if (symbol->IsThreadLocal()) {
3106         SelectAddrofThreadLocal(result, stImm);
3107         return;
3108     } else {
3109         Operand *srcOpnd = &result;
3110         if (!IsAfterRegAlloc()) {
3111             // Create a new vreg/preg for the upper bits of the address
3112             PregIdx pregIdx = GetFunction().GetPregTab()->CreatePreg(PTY_a64);
3113             MIRPreg *tmpPreg = GetFunction().GetPregTab()->PregFromPregIdx(pregIdx);
3114             regno_t vRegNO = NewVReg(kRegTyInt, GetPrimTypeSize(PTY_a64));
3115             RegOperand &tmpreg = GetOrCreateVirtualRegisterOperand(vRegNO);
3116 
3117             // Register this vreg mapping
3118             RegisterVregMapping(vRegNO, pregIdx);
3119 
3120             // Store rematerialization info in the preg
3121             tmpPreg->SetOp(OP_addrof);
3122             tmpPreg->rematInfo.sym = symbol;
3123             tmpPreg->fieldID = field;
3124             tmpPreg->addrUpper = true;
3125 
3126             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrp, tmpreg, stImm));
3127             srcOpnd = &tmpreg;
3128         } else {
3129             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrp, result, stImm));
3130         }
3131         if (CGOptions::IsPIC() && symbol->NeedPIC()) {
3132             /* ldr     x0, [x0, #:got_lo12:Ljava_2Flang_2FSystem_3B_7Cout] */
3133             OfstOperand &offset = CreateOfstOpnd(*stImm.GetSymbol(), stImm.GetOffset(), stImm.GetRelocs());
3134 
3135             auto size = GetPointerSize() * kBitsPerByte;
3136             MemOperand &memOpnd = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, size, static_cast<RegOperand *>(srcOpnd),
3137                                                      nullptr, &offset, nullptr);
3138             GetCurBB()->AppendInsn(
3139                 GetInsnBuilder()->BuildInsn(size == k64BitSize ? MOP_xldr : MOP_wldr, result, memOpnd));
3140 
3141             if (stImm.GetOffset() > 0) {
3142                 ImmOperand &immOpnd = CreateImmOperand(stImm.GetOffset(), result.GetSize(), false);
3143                 SelectAdd(result, result, immOpnd, PTY_u64);
3144             }
3145         } else {
3146             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrpl12, result, *srcOpnd, stImm));
3147         }
3148     }
3149 }
3150 
SelectAddrof(Operand & result,MemOperand & memOpnd,FieldID field)3151 void AArch64CGFunc::SelectAddrof(Operand &result, MemOperand &memOpnd, FieldID field)
3152 {
3153     const MIRSymbol *symbol = memOpnd.GetSymbol();
3154     if (symbol->GetStorageClass() == kScAuto) {
3155         auto *offsetOpnd = static_cast<OfstOperand *>(memOpnd.GetOffsetImmediate());
3156         Operand &immOpnd = CreateImmOperand(offsetOpnd->GetOffsetValue(), PTY_u32, false);
3157         DEBUG_ASSERT(memOpnd.GetBaseRegister() != nullptr, "nullptr check");
3158         SelectAdd(result, *memOpnd.GetBaseRegister(), immOpnd, PTY_u32);
3159         SetStackProtectInfo(kAddrofStack);
3160     } else if (!IsAfterRegAlloc()) {
3161         // Create a new vreg/preg for the upper bits of the address
3162         PregIdx pregIdx = GetFunction().GetPregTab()->CreatePreg(PTY_a64);
3163         MIRPreg *tmpPreg = GetFunction().GetPregTab()->PregFromPregIdx(pregIdx);
3164         regno_t vRegNO = NewVReg(kRegTyInt, GetPrimTypeSize(PTY_a64));
3165         RegOperand &tmpreg = GetOrCreateVirtualRegisterOperand(vRegNO);
3166 
3167         // Register this vreg mapping
3168         RegisterVregMapping(vRegNO, pregIdx);
3169 
3170         // Store rematerialization info in the preg
3171         tmpPreg->SetOp(OP_addrof);
3172         tmpPreg->rematInfo.sym = symbol;
3173         tmpPreg->fieldID = field;
3174         tmpPreg->addrUpper = true;
3175 
3176         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrp, tmpreg, memOpnd));
3177         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrpl12, result, tmpreg, memOpnd));
3178     } else {
3179         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrp, result, memOpnd));
3180         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrpl12, result, result, memOpnd));
3181     }
3182 }
3183 
SelectAddrof(AddrofNode & expr,const BaseNode & parent,bool isAddrofoff)3184 Operand *AArch64CGFunc::SelectAddrof(AddrofNode &expr, const BaseNode &parent, bool isAddrofoff)
3185 {
3186     MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(expr.GetStIdx());
3187     int32 offset = 0;
3188     AddrofoffNode &addrofoffExpr = static_cast<AddrofoffNode &>(static_cast<BaseNode &>(expr));
3189     if (isAddrofoff) {
3190         offset = addrofoffExpr.offset;
3191     } else {
3192         if (expr.GetFieldID() != 0) {
3193             MIRStructType *structType = static_cast<MIRStructType *>(symbol->GetType());
3194             /* with array of structs, it is possible to have nullptr */
3195             if (structType != nullptr) {
3196                 offset = GetBecommon().GetFieldOffset(*structType, expr.GetFieldID()).first;
3197             }
3198         }
3199     }
3200     if ((symbol->GetStorageClass() == kScFormal) && (symbol->GetSKind() == kStVar) &&
3201         ((!isAddrofoff && expr.GetFieldID() != 0) ||
3202          (GetBecommon().GetTypeSize(symbol->GetType()->GetTypeIndex().GetIdx()) > k16ByteSize))) {
3203         /*
3204          * Struct param is copied on the stack by caller if struct size > 16.
3205          * Else if size < 16 then struct param is copied into one or two registers.
3206          */
3207         RegOperand *stackAddr = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
3208         /* load the base address of the struct copy from stack. */
3209         SelectAddrof(*stackAddr, CreateStImmOperand(*symbol, 0, 0));
3210         Operand *structAddr;
3211         if (GetBecommon().GetTypeSize(symbol->GetType()->GetTypeIndex().GetIdx()) <= k16ByteSize) {
3212             isAggParamInReg = true;
3213             structAddr = stackAddr;
3214         } else {
3215             OfstOperand *offopnd = &CreateOfstOpnd(0, k32BitSize);
3216             MemOperand *mo = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, GetPointerSize() * kBitsPerByte, stackAddr,
3217                                                  nullptr, offopnd, nullptr);
3218             structAddr = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
3219             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xldr, *structAddr, *mo));
3220         }
3221         if (offset == 0) {
3222             return structAddr;
3223         } else {
3224             /* add the struct offset to the base address */
3225             Operand *result = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
3226             ImmOperand *imm = &CreateImmOperand(PTY_a64, offset);
3227             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *result, *structAddr, *imm));
3228             return result;
3229         }
3230     }
3231     PrimType ptype = expr.GetPrimType();
3232     Operand &result = GetOrCreateResOperand(parent, ptype);
3233     if (symbol->IsReflectionClassInfo() && !symbol->IsReflectionArrayClassInfo() && !GetCG()->IsLibcore()) {
3234         /*
3235          * Turn addrof __cinf_X  into a load of _PTR__cinf_X
3236          * adrp    x1, _PTR__cinf_Ljava_2Flang_2FSystem_3B
3237          * ldr     x1, [x1, #:lo12:_PTR__cinf_Ljava_2Flang_2FSystem_3B]
3238          */
3239         std::string ptrName = namemangler::kPtrPrefixStr + symbol->GetName();
3240         MIRType *ptrType = GlobalTables::GetTypeTable().GetPtr();
3241         symbol = GetMirModule().GetMIRBuilder()->GetOrCreateGlobalDecl(ptrName, *ptrType);
3242         symbol->SetStorageClass(kScFstatic);
3243 
3244         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_adrp_ldr, result, CreateStImmOperand(*symbol, 0, 0)));
3245         /* make it un rematerializable. */
3246         MIRPreg *preg = GetPseudoRegFromVirtualRegNO(static_cast<RegOperand &>(result).GetRegisterNumber());
3247         if (preg) {
3248             preg->SetOp(OP_undef);
3249         }
3250         return &result;
3251     }
3252 
3253     SelectAddrof(result, CreateStImmOperand(*symbol, offset, 0), isAddrofoff ? 0 : expr.GetFieldID());
3254     return &result;
3255 }
3256 
SelectAddrofoff(AddrofoffNode & expr,const BaseNode & parent)3257 Operand *AArch64CGFunc::SelectAddrofoff(AddrofoffNode &expr, const BaseNode &parent)
3258 {
3259     return SelectAddrof(static_cast<AddrofNode &>(static_cast<BaseNode &>(expr)), parent, true);
3260 }
3261 
SelectAddrofFunc(AddroffuncNode & expr,const BaseNode & parent)3262 Operand &AArch64CGFunc::SelectAddrofFunc(AddroffuncNode &expr, const BaseNode &parent)
3263 {
3264     uint32 instrSize = static_cast<uint32>(expr.SizeOfInstr());
3265     PrimType primType = (instrSize == k8ByteSize)
3266                             ? PTY_u64
3267                             : (instrSize == k4ByteSize) ? PTY_u32 : (instrSize == k2ByteSize) ? PTY_u16 : PTY_u8;
3268     Operand &operand = GetOrCreateResOperand(parent, primType);
3269     MIRFunction *mirFunction = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(expr.GetPUIdx());
3270     SelectAddrof(operand, CreateStImmOperand(*mirFunction->GetFuncSymbol(), 0, 0));
3271     return operand;
3272 }
3273 
3274 /* For an entire aggregate that can fit inside a single 8 byte register.  */
GetDestTypeFromAggSize(uint32 bitSize) const3275 PrimType AArch64CGFunc::GetDestTypeFromAggSize(uint32 bitSize) const
3276 {
3277     PrimType primType;
3278     switch (bitSize) {
3279         case k8BitSize: {
3280             primType = PTY_u8;
3281             break;
3282         }
3283         case k16BitSize: {
3284             primType = PTY_u16;
3285             break;
3286         }
3287         case k32BitSize: {
3288             primType = PTY_u32;
3289             break;
3290         }
3291         case k64BitSize: {
3292             primType = PTY_u64;
3293             break;
3294         }
3295         default:
3296             CHECK_FATAL(false, "aggregate of unhandled size");
3297     }
3298     return primType;
3299 }
3300 
SelectAddrofLabel(AddroflabelNode & expr,const BaseNode & parent)3301 Operand &AArch64CGFunc::SelectAddrofLabel(AddroflabelNode &expr, const BaseNode &parent)
3302 {
3303     /* adrp reg, label-id */
3304     uint32 instrSize = static_cast<uint32>(expr.SizeOfInstr());
3305     PrimType primType = (instrSize == k8ByteSize)
3306                             ? PTY_u64
3307                             : (instrSize == k4ByteSize) ? PTY_u32 : (instrSize == k2ByteSize) ? PTY_u16 : PTY_u8;
3308     Operand &dst = GetOrCreateResOperand(parent, primType);
3309     Operand &immOpnd = CreateImmOperand(expr.GetOffset(), k64BitSize, false);
3310     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_adrp_label, dst, immOpnd));
3311     return dst;
3312 }
3313 
SelectIreadoff(const BaseNode & parent,IreadoffNode & ireadoff)3314 Operand *AArch64CGFunc::SelectIreadoff(const BaseNode &parent, IreadoffNode &ireadoff)
3315 {
3316     auto offset = ireadoff.GetOffset();
3317     auto primType = ireadoff.GetPrimType();
3318     auto bitSize = GetPrimTypeBitSize(primType);
3319     auto *baseAddr = ireadoff.Opnd(0);
3320     auto *result = &CreateRegisterOperandOfType(primType);
3321     auto *addrOpnd = HandleExpr(ireadoff, *baseAddr);
3322     auto &memOpnd = CreateMemOpnd(LoadIntoRegister(*addrOpnd, PTY_a64), offset, bitSize);
3323     auto mop = PickLdInsn(bitSize, primType);
3324     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mop, *result, memOpnd));
3325     return result;
3326 }
3327 
GenLmbcParamLoad(int32 offset,uint32 byteSize,RegType regType,PrimType primType,AArch64reg baseRegno)3328 RegOperand *AArch64CGFunc::GenLmbcParamLoad(int32 offset, uint32 byteSize, RegType regType, PrimType primType,
3329                                             AArch64reg baseRegno)
3330 {
3331     MemOperand *memOpnd = GenLmbcFpMemOperand(offset, byteSize, baseRegno);
3332     RegOperand *result = &GetOrCreateVirtualRegisterOperand(NewVReg(regType, byteSize));
3333     MOperator mOp = PickLdInsn(byteSize * kBitsPerByte, primType);
3334     Insn &load = GetInsnBuilder()->BuildInsn(mOp, *result, *memOpnd);
3335     GetCurBB()->AppendInsn(load);
3336     return result;
3337 }
3338 
LmbcStructReturnLoad(int32 offset)3339 RegOperand *AArch64CGFunc::LmbcStructReturnLoad(int32 offset)
3340 {
3341     RegOperand *result = nullptr;
3342     MIRFunction &func = GetFunction();
3343     CHECK_FATAL(func.IsReturnStruct(), "LmbcStructReturnLoad: not struct return");
3344     MIRType *ty = func.GetReturnType();
3345     uint32 sz = GetBecommon().GetTypeSize(ty->GetTypeIndex());
3346     uint32 fpSize;
3347     uint32 numFpRegs = FloatParamRegRequired(static_cast<MIRStructType *>(ty), fpSize);
3348     if (numFpRegs > 0) {
3349         PrimType pType = (fpSize <= k4ByteSize) ? PTY_f32 : PTY_f64;
3350         for (int32 i = (numFpRegs - kOneRegister); i > 0; --i) {
3351             result = GenLmbcParamLoad(offset + (i * static_cast<int32>(fpSize)), fpSize, kRegTyFloat, pType);
3352             AArch64reg regNo = static_cast<AArch64reg>(V0 + static_cast<uint32>(i));
3353             RegOperand *reg = &GetOrCreatePhysicalRegisterOperand(regNo, fpSize * kBitsPerByte, kRegTyFloat);
3354             SelectCopy(*reg, pType, *result, pType);
3355             Insn &pseudo = GetInsnBuilder()->BuildInsn(MOP_pseudo_ret_float, *reg);
3356             GetCurBB()->AppendInsn(pseudo);
3357         }
3358         result = GenLmbcParamLoad(offset, fpSize, kRegTyFloat, pType);
3359     } else if (sz <= k4ByteSize) {
3360         result = GenLmbcParamLoad(offset, k4ByteSize, kRegTyInt, PTY_u32);
3361     } else if (sz <= k8ByteSize) {
3362         result = GenLmbcParamLoad(offset, k8ByteSize, kRegTyInt, PTY_i64);
3363     } else if (sz <= k16ByteSize) {
3364         result = GenLmbcParamLoad(offset + k8ByteSizeInt, k8ByteSize, kRegTyInt, PTY_i64);
3365         RegOperand *r1 = &GetOrCreatePhysicalRegisterOperand(R1, k8ByteSize * kBitsPerByte, kRegTyInt);
3366         SelectCopy(*r1, PTY_i64, *result, PTY_i64);
3367         Insn &pseudo = GetInsnBuilder()->BuildInsn(MOP_pseudo_ret_int, *r1);
3368         GetCurBB()->AppendInsn(pseudo);
3369         result = GenLmbcParamLoad(offset, k8ByteSize, kRegTyInt, PTY_i64);
3370     }
3371     return result;
3372 }
3373 
SelectIreadfpoff(const BaseNode & parent,IreadFPoffNode & ireadoff)3374 Operand *AArch64CGFunc::SelectIreadfpoff(const BaseNode &parent, IreadFPoffNode &ireadoff)
3375 {
3376     uint32 offset = ireadoff.GetOffset();
3377     PrimType primType = ireadoff.GetPrimType();
3378     uint32 bytelen = GetPrimTypeSize(primType);
3379     uint32 bitlen = bytelen * kBitsPerByte;
3380     RegType regty = GetRegTyFromPrimTy(primType);
3381     RegOperand *result = nullptr;
3382     if (offset >= 0) {
3383         LmbcFormalParamInfo *info = GetLmbcFormalParamInfo(static_cast<uint32>(offset));
3384         if (info->GetPrimType() == PTY_agg) {
3385             if (info->IsOnStack()) {
3386                 result = GenLmbcParamLoad(info->GetOnStackOffset(), GetPrimTypeSize(PTY_a64), kRegTyInt, PTY_a64);
3387                 regno_t baseRegno = result->GetRegisterNumber();
3388                 result = GenLmbcParamLoad(offset - static_cast<int32>(info->GetOffset()), bytelen, regty, primType,
3389                                           (AArch64reg)baseRegno);
3390             } else if (primType == PTY_agg) {
3391                 CHECK_FATAL(parent.GetOpCode() == OP_regassign, "SelectIreadfpoff of agg");
3392                 result = LmbcStructReturnLoad(offset);
3393             } else {
3394                 result = GenLmbcParamLoad(offset, bytelen, regty, primType);
3395             }
3396         } else {
3397             CHECK_FATAL(primType == info->GetPrimType(), "Incorrect primtype");
3398             CHECK_FATAL(offset == info->GetOffset(), "Incorrect offset");
3399             if (info->GetRegNO() == 0 || !info->HasRegassign()) {
3400                 result = GenLmbcParamLoad(offset, bytelen, regty, primType);
3401             } else {
3402                 result = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(info->GetRegNO()), bitlen, regty);
3403             }
3404         }
3405     } else {
3406         if (primType == PTY_agg) {
3407             CHECK_FATAL(parent.GetOpCode() == OP_regassign, "SelectIreadfpoff of agg");
3408             result = LmbcStructReturnLoad(offset);
3409         } else {
3410             result = GenLmbcParamLoad(offset, bytelen, regty, primType);
3411         }
3412     }
3413     return result;
3414 }
3415 
SelectIread(const BaseNode & parent,IreadNode & expr,int extraOffset,PrimType finalBitFieldDestType)3416 Operand *AArch64CGFunc::SelectIread(const BaseNode &parent, IreadNode &expr, int extraOffset,
3417                                     PrimType finalBitFieldDestType)
3418 {
3419     int32 offset = 0;
3420     MIRType *type = GlobalTables::GetTypeTable().GetTypeFromTyIdx(expr.GetTyIdx());
3421     MIRPtrType *pointerType = static_cast<MIRPtrType *>(type);
3422     DEBUG_ASSERT(pointerType != nullptr, "expect a pointer type at iread node");
3423     MIRType *pointedType = nullptr;
3424     bool isRefField = false;
3425     AArch64isa::MemoryOrdering memOrd = AArch64isa::kMoNone;
3426 
3427     if (expr.GetFieldID() != 0) {
3428         MIRType *pointedTy = GlobalTables::GetTypeTable().GetTypeFromTyIdx(pointerType->GetPointedTyIdx());
3429         MIRStructType *structType = nullptr;
3430         if (pointedTy->GetKind() != kTypeJArray) {
3431             structType = static_cast<MIRStructType *>(pointedTy);
3432         } else {
3433             /* it's a Jarray type. using it's parent's field info: java.lang.Object */
3434             structType = static_cast<MIRJarrayType *>(pointedTy)->GetParentType();
3435         }
3436 
3437         DEBUG_ASSERT(structType != nullptr, "SelectIread: non-zero fieldID for non-structure");
3438         pointedType = structType->GetFieldType(expr.GetFieldID());
3439         offset = GetBecommon().GetFieldOffset(*structType, expr.GetFieldID()).first;
3440         isRefField = GetBecommon().IsRefField(*structType, expr.GetFieldID());
3441     } else {
3442         pointedType = GetPointedToType(*pointerType);
3443         if (GetFunction().IsJava() && (pointedType->GetKind() == kTypePointer)) {
3444             MIRType *nextPointedType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(
3445                 static_cast<MIRPtrType *>(pointedType)->GetPointedTyIdx());
3446             if (nextPointedType->GetKind() != kTypeScalar) {
3447                 isRefField = true; /* read from an object array, or an high-dimentional array */
3448             }
3449         }
3450     }
3451 
3452     RegType regType = GetRegTyFromPrimTy(expr.GetPrimType());
3453     uint32 regSize = GetPrimTypeSize(expr.GetPrimType());
3454     if (expr.GetFieldID() == 0 && pointedType->GetPrimType() == PTY_agg) {
3455         /* Maple IR can passing small struct to be loaded into a single register. */
3456         if (regType == kRegTyFloat) {
3457             /* regsize is correct */
3458         } else {
3459             uint32 sz = GetBecommon().GetTypeSize(pointedType->GetTypeIndex().GetIdx());
3460             regSize = (sz <= k4ByteSize) ? k4ByteSize : k8ByteSize;
3461         }
3462     } else if (regSize < k4ByteSize) {
3463         regSize = k4ByteSize; /* 32-bit */
3464     }
3465     Operand *result = nullptr;
3466     if (parent.GetOpCode() == OP_eval) {
3467         /* regSize << 3, that is regSize * 8, change bytes to bits */
3468         result = &GetZeroOpnd(regSize << 3);
3469     } else {
3470         result = &GetOrCreateResOperand(parent, expr.GetPrimType());
3471     }
3472 
3473     PrimType destType = pointedType->GetPrimType();
3474 
3475     uint32 bitSize = 0;
3476     if ((pointedType->GetKind() == kTypeStructIncomplete) || (pointedType->GetKind() == kTypeClassIncomplete) ||
3477         (pointedType->GetKind() == kTypeInterfaceIncomplete)) {
3478         bitSize = GetPrimTypeBitSize(expr.GetPrimType());
3479         maple::LogInfo::MapleLogger(kLlErr) << "Warning: objsize is zero! \n";
3480     } else {
3481         if (pointedType->IsStructType()) {
3482             MIRStructType *structType = static_cast<MIRStructType *>(pointedType);
3483             /* size << 3, that is size * 8, change bytes to bits */
3484             bitSize = std::min(structType->GetSize(), static_cast<size_t>(GetPointerSize())) << 3;
3485         } else {
3486             bitSize = GetPrimTypeBitSize(destType);
3487         }
3488         if (regType == kRegTyFloat) {
3489             destType = expr.GetPrimType();
3490             bitSize = GetPrimTypeBitSize(destType);
3491         } else if (destType == PTY_agg) {
3492             switch (bitSize) {
3493                 case k8BitSize:
3494                     destType = PTY_u8;
3495                     break;
3496                 case k16BitSize:
3497                     destType = PTY_u16;
3498                     break;
3499                 case k32BitSize:
3500                     destType = PTY_u32;
3501                     break;
3502                 case k64BitSize:
3503                     destType = PTY_u64;
3504                     break;
3505                 default:
3506                     destType = PTY_u64;  // when eval agg . a way to round up
3507                     break;
3508             }
3509         }
3510     }
3511 
3512     MemOperand *memOpnd =
3513         CreateMemOpndOrNull(destType, expr, *expr.Opnd(0), static_cast<int64>(offset) + extraOffset, memOrd);
3514     if (aggParamReg != nullptr) {
3515         isAggParamInReg = false;
3516         return aggParamReg;
3517     }
3518     DEBUG_ASSERT(memOpnd != nullptr, "memOpnd should not be nullptr");
3519     if (isVolLoad && (memOpnd->GetAddrMode() == MemOperand::kAddrModeBOi)) {
3520         memOrd = AArch64isa::kMoAcquire;
3521         isVolLoad = false;
3522     }
3523 
3524     memOpnd = memOpnd->IsOffsetMisaligned(bitSize) ? &ConstraintOffsetToSafeRegion(bitSize, *memOpnd) : memOpnd;
3525     if (memOrd == AArch64isa::kMoNone) {
3526         MOperator mOp = 0;
3527         if (finalBitFieldDestType == kPtyInvalid) {
3528             mOp = PickLdInsn(bitSize, destType);
3529         } else {
3530             mOp = PickLdInsn(GetPrimTypeBitSize(finalBitFieldDestType), finalBitFieldDestType);
3531         }
3532         if ((memOpnd->GetMemVaryType() == kNotVary) && !IsOperandImmValid(mOp, memOpnd, 1)) {
3533             memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, bitSize);
3534         }
3535         Insn &insn = GetInsnBuilder()->BuildInsn(mOp, *result, *memOpnd);
3536         if (parent.GetOpCode() == OP_eval && result->IsRegister() &&
3537             static_cast<RegOperand *>(result)->GetRegisterNumber() == RZR) {
3538             insn.SetComment("null-check");
3539         }
3540         GetCurBB()->AppendInsn(insn);
3541 
3542         if (parent.op != OP_eval) {
3543             const InsnDesc *md = &AArch64CG::kMd[insn.GetMachineOpcode()];
3544             auto *prop = md->GetOpndDes(0);
3545             if ((prop->GetSize()) < insn.GetOperand(0).GetSize()) {
3546                 switch (destType) {
3547                     case PTY_i8:
3548                         mOp = MOP_xsxtb64;
3549                         break;
3550                     case PTY_i16:
3551                         mOp = MOP_xsxth64;
3552                         break;
3553                     case PTY_i32:
3554                         mOp = MOP_xsxtw64;
3555                         break;
3556                     case PTY_u8:
3557                         mOp = MOP_xuxtb32;
3558                         break;
3559                     case PTY_u16:
3560                         mOp = MOP_xuxth32;
3561                         break;
3562                     case PTY_u32:
3563                         mOp = MOP_xuxtw64;
3564                         break;
3565                     default:
3566                         break;
3567                 }
3568                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, insn.GetOperand(0), insn.GetOperand(0)));
3569             }
3570         }
3571     } else {
3572         if ((memOpnd->GetMemVaryType() == kNotVary) && IsImmediateOffsetOutOfRange(*memOpnd, bitSize)) {
3573             memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, bitSize);
3574         }
3575         AArch64CGFunc::SelectLoadAcquire(*result, destType, *memOpnd, destType, memOrd, false);
3576     }
3577     GetCurBB()->GetLastInsn()->MarkAsAccessRefField(isRefField);
3578     return result;
3579 }
3580 
SelectIntConst(const MIRIntConst & intConst)3581 Operand *AArch64CGFunc::SelectIntConst(const MIRIntConst &intConst)
3582 {
3583     return &CreateImmOperand(intConst.GetExtValue(), GetPrimTypeSize(intConst.GetType().GetPrimType()) * kBitsPerByte,
3584                              false);
3585 }
3586 
3587 template <typename T>
SelectLiteral(T * c,MIRFunction * func,uint32 labelIdx,AArch64CGFunc * cgFunc)3588 Operand *SelectLiteral(T *c, MIRFunction *func, uint32 labelIdx, AArch64CGFunc *cgFunc)
3589 {
3590     MIRSymbol *st = func->GetSymTab()->CreateSymbol(kScopeLocal);
3591     std::string lblStr(".LB_");
3592     MIRSymbol *funcSt = GlobalTables::GetGsymTable().GetSymbolFromStidx(func->GetStIdx().Idx());
3593     std::string funcName = funcSt->GetName();
3594     lblStr += funcName;
3595     lblStr += std::to_string(labelIdx);
3596     st->SetNameStrIdx(lblStr);
3597     st->SetStorageClass(kScPstatic);
3598     st->SetSKind(kStConst);
3599     st->SetKonst(c);
3600     cgFunc->SetLocalSymLabelIndex(*st, labelIdx);
3601     PrimType primType = c->GetType().GetPrimType();
3602     st->SetTyIdx(TyIdx(primType));
3603     uint32 typeBitSize = GetPrimTypeBitSize(primType);
3604 
3605     if ((T::GetPrimType() == PTY_f32 || T::GetPrimType() == PTY_f64)) {
3606         return static_cast<Operand *>(&cgFunc->GetOrCreateMemOpnd(*st, 0, typeBitSize));
3607     } else {
3608         CHECK_FATAL(false, "Unsupported const type");
3609     }
3610     return nullptr;
3611 }
3612 
HandleFmovImm(PrimType stype,int64 val,MIRConst & mirConst,const BaseNode & parent)3613 Operand *AArch64CGFunc::HandleFmovImm(PrimType stype, int64 val, MIRConst &mirConst, const BaseNode &parent)
3614 {
3615     Operand *result;
3616     bool is64Bits = (GetPrimTypeBitSize(stype) == k64BitSize);
3617     uint64 canRepreset = is64Bits ? (val & 0xffffffffffff) : (val & 0x7ffff);
3618     uint32 val1 = is64Bits ? (val >> 61) & 0x3 : (val >> 29) & 0x3;
3619     uint32 val2 = is64Bits ? (val >> 54) & 0xff : (val >> 25) & 0x1f;
3620     bool isSame = is64Bits ? ((val2 == 0) || (val2 == 0xff)) : ((val2 == 0) || (val2 == 0x1f));
3621     canRepreset = (canRepreset == 0) && ((val1 & 0x1) ^ ((val1 & 0x2) >> 1)) && isSame;
3622     if (canRepreset) {
3623         uint64 temp1 = is64Bits ? (val >> 63) << 7 : (val >> 31) << 7;
3624         uint64 temp2 = is64Bits ? val >> 48 : val >> 19;
3625         int64 imm8 = (temp2 & 0x7f) | temp1;
3626         Operand *newOpnd0 = &CreateImmOperand(imm8, k8BitSize, true, kNotVary, true);
3627         result = &GetOrCreateResOperand(parent, stype);
3628         MOperator mopFmov = (is64Bits ? MOP_xdfmovri : MOP_wsfmovri);
3629         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopFmov, *result, *newOpnd0));
3630     } else {
3631         if (is64Bits) {  // For DoubleConst, use ldr .literal
3632             uint32 labelIdxTmp = GetLabelIdx();
3633             result = SelectLiteral(static_cast<MIRDoubleConst *>(&mirConst), &GetFunction(), labelIdxTmp++, this);
3634             SetLabelIdx(labelIdxTmp);
3635             return result;
3636         }
3637         Operand *newOpnd0 = &CreateImmOperand(val, GetPrimTypeSize(stype) * kBitsPerByte, false);
3638         PrimType itype = (stype == PTY_f32) ? PTY_i32 : PTY_i64;
3639         RegOperand &regOpnd = LoadIntoRegister(*newOpnd0, itype);
3640 
3641         result = &GetOrCreateResOperand(parent, stype);
3642         MOperator mopFmov = (is64Bits ? MOP_xvmovdr : MOP_xvmovsr);
3643         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopFmov, *result, regOpnd));
3644     }
3645     return result;
3646 }
3647 
SelectFloatConst(MIRFloatConst & floatConst,const BaseNode & parent)3648 Operand *AArch64CGFunc::SelectFloatConst(MIRFloatConst &floatConst, const BaseNode &parent)
3649 {
3650     PrimType stype = floatConst.GetType().GetPrimType();
3651     int32 val = floatConst.GetIntValue();
3652     /* according to aarch64 encoding format, convert int to float expression */
3653     Operand *result;
3654     result = HandleFmovImm(stype, val, floatConst, parent);
3655     return result;
3656 }
3657 
SelectDoubleConst(MIRDoubleConst & doubleConst,const BaseNode & parent)3658 Operand *AArch64CGFunc::SelectDoubleConst(MIRDoubleConst &doubleConst, const BaseNode &parent)
3659 {
3660     PrimType stype = doubleConst.GetType().GetPrimType();
3661     int64 val = doubleConst.GetIntValue();
3662     /* according to aarch64 encoding format, convert int to float expression */
3663     Operand *result;
3664     result = HandleFmovImm(stype, val, doubleConst, parent);
3665     return result;
3666 }
3667 
3668 template <typename T>
SelectStrLiteral(T & c,AArch64CGFunc & cgFunc)3669 Operand *SelectStrLiteral(T &c, AArch64CGFunc &cgFunc)
3670 {
3671     std::string labelStr;
3672     if (c.GetKind() == kConstStrConst) {
3673         labelStr += ".LUstr_";
3674     } else if (c.GetKind() == kConstStr16Const) {
3675         labelStr += ".LUstr16_";
3676     } else {
3677         CHECK_FATAL(false, "Unsupported literal type");
3678     }
3679     labelStr += std::to_string(c.GetValue());
3680 
3681     MIRSymbol *labelSym =
3682         GlobalTables::GetGsymTable().GetSymbolFromStrIdx(GlobalTables::GetStrTable().GetStrIdxFromName(labelStr));
3683     if (labelSym == nullptr) {
3684         labelSym = cgFunc.GetMirModule().GetMIRBuilder()->CreateGlobalDecl(labelStr, c.GetType());
3685         labelSym->SetStorageClass(kScFstatic);
3686         labelSym->SetSKind(kStConst);
3687         /* c may be local, we need a global node here */
3688         labelSym->SetKonst(cgFunc.NewMirConst(c));
3689     }
3690 
3691     if (c.GetPrimType() == PTY_ptr) {
3692         StImmOperand &stOpnd = cgFunc.CreateStImmOperand(*labelSym, 0, 0);
3693         RegOperand &addrOpnd = cgFunc.CreateRegisterOperandOfType(PTY_a64);
3694         cgFunc.SelectAddrof(addrOpnd, stOpnd);
3695         return &addrOpnd;
3696     }
3697     CHECK_FATAL(false, "Unsupported const string type");
3698     return nullptr;
3699 }
3700 
SelectStrConst(MIRStrConst & strConst)3701 Operand *AArch64CGFunc::SelectStrConst(MIRStrConst &strConst)
3702 {
3703     return SelectStrLiteral(strConst, *this);
3704 }
3705 
SelectStr16Const(MIRStr16Const & str16Const)3706 Operand *AArch64CGFunc::SelectStr16Const(MIRStr16Const &str16Const)
3707 {
3708     return SelectStrLiteral(str16Const, *this);
3709 }
3710 
AppendInstructionTo(Insn & i,CGFunc & f)3711 static inline void AppendInstructionTo(Insn &i, CGFunc &f)
3712 {
3713     f.GetCurBB()->AppendInsn(i);
3714 }
3715 
3716 /*
3717  * Returns the number of leading 0-bits in x, starting at the most significant bit position.
3718  * If x is 0, the result is -1.
3719  */
GetHead0BitNum(int64 val)3720 static int32 GetHead0BitNum(int64 val)
3721 {
3722     uint32 bitNum = 0;
3723     for (; bitNum < k64BitSize; bitNum++) {
3724         if ((0x8000000000000000ULL >> static_cast<uint32>(bitNum)) & static_cast<uint64>(val)) {
3725             break;
3726         }
3727     }
3728     if (bitNum == k64BitSize) {
3729         return -1;
3730     }
3731     return bitNum;
3732 }
3733 
3734 /*
3735  * Returns the number of trailing 0-bits in x, starting at the least significant bit position.
3736  * If x is 0, the result is -1.
3737  */
GetTail0BitNum(int64 val)3738 static int32 GetTail0BitNum(int64 val)
3739 {
3740     uint32 bitNum = 0;
3741     for (; bitNum < k64BitSize; bitNum++) {
3742         if ((static_cast<uint64>(1) << static_cast<uint32>(bitNum)) & static_cast<uint64>(val)) {
3743             break;
3744         }
3745     }
3746     if (bitNum == k64BitSize) {
3747         return -1;
3748     }
3749     return bitNum;
3750 }
3751 
3752 /*
3753  * If the input integer is power of 2, return log2(input)
3754  * else return -1
3755  */
GetLog2(uint64 val)3756 static inline int32 GetLog2(uint64 val)
3757 {
3758     if (__builtin_popcountll(val) == 1) {
3759         return __builtin_ffsll(static_cast<int64>(val)) - 1;
3760     }
3761     return -1;
3762 }
3763 
PickJmpInsn(Opcode brOp,Opcode cmpOp,bool isFloat,bool isSigned) const3764 MOperator AArch64CGFunc::PickJmpInsn(Opcode brOp, Opcode cmpOp, bool isFloat, bool isSigned) const
3765 {
3766     switch (cmpOp) {
3767         case OP_ne:
3768             return (brOp == OP_brtrue) ? MOP_bne : MOP_beq;
3769         case OP_eq:
3770             return (brOp == OP_brtrue) ? MOP_beq : MOP_bne;
3771         case OP_lt:
3772             return (brOp == OP_brtrue) ? (isSigned ? MOP_blt : MOP_blo)
3773                                        : (isFloat ? MOP_bpl : (isSigned ? MOP_bge : MOP_bhs));
3774         case OP_le:
3775             return (brOp == OP_brtrue) ? (isSigned ? MOP_ble : MOP_bls)
3776                                        : (isFloat ? MOP_bhi : (isSigned ? MOP_bgt : MOP_bhi));
3777         case OP_gt:
3778             return (brOp == OP_brtrue) ? (isFloat ? MOP_bgt : (isSigned ? MOP_bgt : MOP_bhi))
3779                                        : (isSigned ? MOP_ble : MOP_bls);
3780         case OP_ge:
3781             return (brOp == OP_brtrue) ? (isFloat ? MOP_bpl : (isSigned ? MOP_bge : MOP_bhs))
3782                                        : (isSigned ? MOP_blt : MOP_blo);
3783         default:
3784             CHECK_FATAL(false, "PickJmpInsn error");
3785     }
3786 }
3787 
GenerateCompareWithZeroInstruction(Opcode jmpOp,Opcode cmpOp,bool is64Bits,PrimType primType,LabelOperand & targetOpnd,Operand & opnd0)3788 bool AArch64CGFunc::GenerateCompareWithZeroInstruction(Opcode jmpOp, Opcode cmpOp, bool is64Bits, PrimType primType,
3789                                                        LabelOperand &targetOpnd, Operand &opnd0)
3790 {
3791     bool finish = true;
3792     MOperator mOpCode = MOP_undef;
3793     switch (cmpOp) {
3794         case OP_ne: {
3795             if (jmpOp == OP_brtrue) {
3796                 mOpCode = is64Bits ? MOP_xcbnz : MOP_wcbnz;
3797             } else {
3798                 mOpCode = is64Bits ? MOP_xcbz : MOP_wcbz;
3799             }
3800             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, opnd0, targetOpnd));
3801             break;
3802         }
3803         case OP_eq: {
3804             if (jmpOp == OP_brtrue) {
3805                 mOpCode = is64Bits ? MOP_xcbz : MOP_wcbz;
3806             } else {
3807                 mOpCode = is64Bits ? MOP_xcbnz : MOP_wcbnz;
3808             }
3809             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, opnd0, targetOpnd));
3810             break;
3811         }
3812         /*
3813          * TBZ/TBNZ instruction have a range of +/-32KB, need to check if the jump target is reachable in a later
3814          * phase. If the branch target is not reachable, then we change tbz/tbnz into combination of ubfx and
3815          * cbz/cbnz, which will clobber one extra register. With LSRA under O2, we can use of the reserved registers
3816          * for that purpose.
3817          */
3818         case OP_lt: {
3819             if (primType == PTY_u64 || primType == PTY_u32) {
3820                 return false;
3821             }
3822             ImmOperand &signBit =
3823                 CreateImmOperand(is64Bits ? kHighestBitOf64Bits : kHighestBitOf32Bits, k8BitSize, false);
3824             if (jmpOp == OP_brtrue) {
3825                 mOpCode = is64Bits ? MOP_xtbnz : MOP_wtbnz;
3826             } else {
3827                 mOpCode = is64Bits ? MOP_xtbz : MOP_wtbz;
3828             }
3829             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, opnd0, signBit, targetOpnd));
3830             break;
3831         }
3832         case OP_ge: {
3833             if (primType == PTY_u64 || primType == PTY_u32) {
3834                 return false;
3835             }
3836             ImmOperand &signBit =
3837                 CreateImmOperand(is64Bits ? kHighestBitOf64Bits : kHighestBitOf32Bits, k8BitSize, false);
3838             if (jmpOp == OP_brtrue) {
3839                 mOpCode = is64Bits ? MOP_xtbz : MOP_wtbz;
3840             } else {
3841                 mOpCode = is64Bits ? MOP_xtbnz : MOP_wtbnz;
3842             }
3843             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, opnd0, signBit, targetOpnd));
3844             break;
3845         }
3846         default:
3847             finish = false;
3848             break;
3849     }
3850     return finish;
3851 }
3852 
SelectIgoto(Operand * opnd0)3853 void AArch64CGFunc::SelectIgoto(Operand *opnd0)
3854 {
3855     Operand *srcOpnd = opnd0;
3856     if (opnd0->GetKind() == Operand::kOpdMem) {
3857         Operand *dst = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
3858         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xldr, *dst, *opnd0));
3859         srcOpnd = dst;
3860     }
3861     GetCurBB()->SetKind(BB::kBBIgoto);
3862     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xbr, *srcOpnd));
3863 }
3864 
SelectCondGoto(LabelOperand & targetOpnd,Opcode jmpOp,Opcode cmpOp,Operand & origOpnd0,Operand & origOpnd1,PrimType primType,bool signedCond)3865 void AArch64CGFunc::SelectCondGoto(LabelOperand &targetOpnd, Opcode jmpOp, Opcode cmpOp, Operand &origOpnd0,
3866                                    Operand &origOpnd1, PrimType primType, bool signedCond)
3867 {
3868     Operand *opnd0 = &origOpnd0;
3869     Operand *opnd1 = &origOpnd1;
3870     opnd0 = &LoadIntoRegister(origOpnd0, primType);
3871 
3872     bool is64Bits = GetPrimTypeBitSize(primType) == k64BitSize;
3873     bool isFloat = IsPrimitiveFloat(primType);
3874     Operand &rflag = GetOrCreateRflag();
3875     if (isFloat) {
3876         opnd1 = &LoadIntoRegister(origOpnd1, primType);
3877         MOperator mOp =
3878             is64Bits ? MOP_dcmperr : ((GetPrimTypeBitSize(primType) == k32BitSize) ? MOP_scmperr : MOP_hcmperr);
3879         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, rflag, *opnd0, *opnd1));
3880     } else {
3881         bool isImm = ((origOpnd1.GetKind() == Operand::kOpdImmediate) || (origOpnd1.GetKind() == Operand::kOpdOffset));
3882         if ((origOpnd1.GetKind() != Operand::kOpdRegister) && !isImm) {
3883             opnd1 = &SelectCopy(origOpnd1, primType, primType);
3884         }
3885         MOperator mOp = is64Bits ? MOP_xcmprr : MOP_wcmprr;
3886 
3887         if (isImm) {
3888             /* Special cases, i.e., comparing with zero
3889              * Do not perform optimization for C, unlike Java which has no unsigned int.
3890              */
3891             if (static_cast<ImmOperand *>(opnd1)->IsZero() &&
3892                 (Globals::GetInstance()->GetOptimLevel() > CGOptions::kLevel0)) {
3893                 bool finish = GenerateCompareWithZeroInstruction(jmpOp, cmpOp, is64Bits, primType, targetOpnd, *opnd0);
3894                 if (finish) {
3895                     return;
3896                 }
3897             }
3898 
3899             /*
3900              * aarch64 assembly takes up to 24-bits immediate, generating
3901              * either cmp or cmp with shift 12 encoding
3902              */
3903             ImmOperand *immOpnd = static_cast<ImmOperand *>(opnd1);
3904             if (immOpnd->IsInBitSize(kMaxImmVal12Bits, 0) || immOpnd->IsInBitSize(kMaxImmVal12Bits, kMaxImmVal12Bits)) {
3905                 mOp = is64Bits ? MOP_xcmpri : MOP_wcmpri;
3906             } else {
3907                 opnd1 = &SelectCopy(*opnd1, primType, primType);
3908             }
3909         }
3910         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, rflag, *opnd0, *opnd1));
3911     }
3912 
3913     bool isSigned = IsPrimitiveInteger(primType) ? IsSignedInteger(primType) : (signedCond ? true : false);
3914     MOperator jmpOperator = PickJmpInsn(jmpOp, cmpOp, isFloat, isSigned);
3915     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(jmpOperator, rflag, targetOpnd));
3916 }
3917 
3918 /*
3919  *   brtrue @label0 (ge u8 i32 (
3920  *   cmp i32 i64 (dread i64 %Reg2_J, dread i64 %Reg4_J),
3921  *   constval i32 0))
3922  *  ===>
3923  *   cmp r1, r2
3924  *   bge Cond, label0
3925  */
SelectCondSpecialCase1(CondGotoNode & stmt,BaseNode & expr)3926 void AArch64CGFunc::SelectCondSpecialCase1(CondGotoNode &stmt, BaseNode &expr)
3927 {
3928     DEBUG_ASSERT(expr.GetOpCode() == OP_cmp, "unexpect opcode");
3929     Operand *opnd0 = HandleExpr(expr, *expr.Opnd(0));
3930     Operand *opnd1 = HandleExpr(expr, *expr.Opnd(1));
3931     CompareNode *node = static_cast<CompareNode *>(&expr);
3932     bool isFloat = IsPrimitiveFloat(node->GetOpndType());
3933     opnd0 = &LoadIntoRegister(*opnd0, node->GetOpndType());
3934     /*
3935      * most of FP constants are passed as MemOperand
3936      * except 0.0 which is passed as kOpdFPImmediate
3937      */
3938     Operand::OperandType opnd1Type = opnd1->GetKind();
3939     if ((opnd1Type != Operand::kOpdImmediate) && (opnd1Type != Operand::kOpdFPImmediate) &&
3940         (opnd1Type != Operand::kOpdOffset)) {
3941         opnd1 = &LoadIntoRegister(*opnd1, node->GetOpndType());
3942     }
3943     SelectAArch64Cmp(*opnd0, *opnd1, !isFloat, GetPrimTypeBitSize(node->GetOpndType()));
3944     /* handle condgoto now. */
3945     LabelIdx labelIdx = stmt.GetOffset();
3946     BaseNode *condNode = stmt.Opnd(0);
3947     LabelOperand &targetOpnd = GetOrCreateLabelOperand(labelIdx);
3948     Opcode cmpOp = condNode->GetOpCode();
3949     PrimType pType = static_cast<CompareNode *>(condNode)->GetOpndType();
3950     isFloat = IsPrimitiveFloat(pType);
3951     Operand &rflag = GetOrCreateRflag();
3952     bool isSigned =
3953         IsPrimitiveInteger(pType) ? IsSignedInteger(pType) : (IsSignedInteger(condNode->GetPrimType()) ? true : false);
3954     MOperator jmpOp = PickJmpInsn(stmt.GetOpCode(), cmpOp, isFloat, isSigned);
3955     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(jmpOp, rflag, targetOpnd));
3956 }
3957 
3958 /*
3959  * Special case:
3960  * brfalse(ge (cmpg (op0, op1), 0) ==>
3961  * fcmp op1, op2
3962  * blo
3963  */
SelectCondSpecialCase2(const CondGotoNode & stmt,BaseNode & expr)3964 void AArch64CGFunc::SelectCondSpecialCase2(const CondGotoNode &stmt, BaseNode &expr)
3965 {
3966     auto &cmpNode = static_cast<CompareNode &>(expr);
3967     Operand *opnd0 = HandleExpr(cmpNode, *cmpNode.Opnd(0));
3968     Operand *opnd1 = HandleExpr(cmpNode, *cmpNode.Opnd(1));
3969     PrimType operandType = cmpNode.GetOpndType();
3970     opnd0 = opnd0->IsRegister() ? static_cast<RegOperand *>(opnd0) : &SelectCopy(*opnd0, operandType, operandType);
3971     Operand::OperandType opnd1Type = opnd1->GetKind();
3972     if ((opnd1Type != Operand::kOpdImmediate) && (opnd1Type != Operand::kOpdFPImmediate) &&
3973         (opnd1Type != Operand::kOpdOffset)) {
3974         opnd1 = opnd1->IsRegister() ? static_cast<RegOperand *>(opnd1) : &SelectCopy(*opnd1, operandType, operandType);
3975     }
3976 #ifdef DEBUG
3977     bool isFloat = IsPrimitiveFloat(operandType);
3978     if (!isFloat) {
3979         DEBUG_ASSERT(false, "incorrect operand types");
3980     }
3981 #endif
3982     SelectTargetFPCmpQuiet(*opnd0, *opnd1, GetPrimTypeBitSize(operandType));
3983     Operand &rFlag = GetOrCreateRflag();
3984     LabelIdx tempLabelIdx = stmt.GetOffset();
3985     LabelOperand &targetOpnd = GetOrCreateLabelOperand(tempLabelIdx);
3986     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_blo, rFlag, targetOpnd));
3987 }
3988 
SelectCondGoto(CondGotoNode & stmt,Operand & opnd0,Operand & opnd1)3989 void AArch64CGFunc::SelectCondGoto(CondGotoNode &stmt, Operand &opnd0, Operand &opnd1)
3990 {
3991     /*
3992      * handle brfalse/brtrue op, opnd0 can be a compare node or non-compare node
3993      * such as a dread for example
3994      */
3995     LabelIdx labelIdx = stmt.GetOffset();
3996     BaseNode *condNode = stmt.Opnd(0);
3997     LabelOperand &targetOpnd = GetOrCreateLabelOperand(labelIdx);
3998     Opcode cmpOp;
3999 
4000     if (opnd0.IsRegister() && (static_cast<RegOperand *>(&opnd0)->GetValidBitsNum() == 1) &&
4001         (condNode->GetOpCode() == OP_lior)) {
4002         ImmOperand &condBit = CreateImmOperand(0, k8BitSize, false);
4003         if (stmt.GetOpCode() == OP_brtrue) {
4004             GetCurBB()->AppendInsn(
4005                 GetInsnBuilder()->BuildInsn(MOP_wtbnz, static_cast<RegOperand &>(opnd0), condBit, targetOpnd));
4006         } else {
4007             GetCurBB()->AppendInsn(
4008                 GetInsnBuilder()->BuildInsn(MOP_wtbz, static_cast<RegOperand &>(opnd0), condBit, targetOpnd));
4009         }
4010         return;
4011     }
4012 
4013     PrimType pType;
4014     if (kOpcodeInfo.IsCompare(condNode->GetOpCode())) {
4015         cmpOp = condNode->GetOpCode();
4016         pType = static_cast<CompareNode *>(condNode)->GetOpndType();
4017     } else {
4018         /* not a compare node; dread for example, take its pType */
4019         cmpOp = OP_ne;
4020         pType = condNode->GetPrimType();
4021     }
4022     bool signedCond = IsSignedInteger(pType) || IsPrimitiveFloat(pType);
4023     SelectCondGoto(targetOpnd, stmt.GetOpCode(), cmpOp, opnd0, opnd1, pType, signedCond);
4024 }
4025 
SelectGoto(GotoNode & stmt)4026 void AArch64CGFunc::SelectGoto(GotoNode &stmt)
4027 {
4028     Operand &targetOpnd = GetOrCreateLabelOperand(stmt.GetOffset());
4029     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xuncond, targetOpnd));
4030     GetCurBB()->SetKind(BB::kBBGoto);
4031 }
4032 
SelectAdd(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4033 Operand *AArch64CGFunc::SelectAdd(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4034 {
4035     PrimType dtype = node.GetPrimType();
4036     bool isSigned = IsSignedInteger(dtype);
4037     uint32 dsize = GetPrimTypeBitSize(dtype);
4038     bool is64Bits = (dsize == k64BitSize);
4039     bool isFloat = IsPrimitiveFloat(dtype);
4040     RegOperand *resOpnd = nullptr;
4041     if (!IsPrimitiveVector(dtype)) {
4042         /* promoted type */
4043         PrimType primType =
4044             isFloat ? dtype : ((is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32)));
4045         if (parent.GetOpCode() == OP_regassign) {
4046             auto &regAssignNode = static_cast<const RegassignNode &>(parent);
4047             PregIdx pregIdx = regAssignNode.GetRegIdx();
4048             if (IsSpecialPseudoRegister(pregIdx)) {
4049                 resOpnd = &GetOrCreateSpecialRegisterOperand(-pregIdx, dtype);
4050             } else {
4051                 resOpnd = &GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx));
4052             }
4053         } else {
4054             resOpnd = &CreateRegisterOperandOfType(primType);
4055         }
4056         SelectAdd(*resOpnd, opnd0, opnd1, primType);
4057     } else {
4058         /* vector operands */
4059         resOpnd =
4060             SelectVectorBinOp(dtype, &opnd0, node.Opnd(0)->GetPrimType(), &opnd1, node.Opnd(1)->GetPrimType(), OP_add);
4061     }
4062     return resOpnd;
4063 }
4064 
SelectAdd(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)4065 void AArch64CGFunc::SelectAdd(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
4066 {
4067     Operand::OperandType opnd0Type = opnd0.GetKind();
4068     Operand::OperandType opnd1Type = opnd1.GetKind();
4069     uint32 dsize = GetPrimTypeBitSize(primType);
4070     bool is64Bits = (dsize == k64BitSize);
4071     if (opnd0Type != Operand::kOpdRegister) {
4072         /* add #imm, #imm */
4073         if (opnd1Type != Operand::kOpdRegister) {
4074             SelectAdd(resOpnd, SelectCopy(opnd0, primType, primType), opnd1, primType);
4075             return;
4076         }
4077         /* add #imm, reg */
4078         SelectAdd(resOpnd, opnd1, opnd0, primType); /* commutative */
4079         return;
4080     }
4081     /* add reg, reg */
4082     if (opnd1Type == Operand::kOpdRegister) {
4083         DEBUG_ASSERT(IsPrimitiveFloat(primType) || IsPrimitiveInteger(primType), "NYI add");
4084         MOperator mOp =
4085             IsPrimitiveFloat(primType) ? (is64Bits ? MOP_dadd : MOP_sadd) : (is64Bits ? MOP_xaddrrr : MOP_waddrrr);
4086         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, opnd1));
4087         return;
4088     } else if (!((opnd1Type == Operand::kOpdImmediate) || (opnd1Type == Operand::kOpdOffset))) {
4089         /* add reg, otheregType */
4090         SelectAdd(resOpnd, opnd0, SelectCopy(opnd1, primType, primType), primType);
4091         return;
4092     } else {
4093         /* add reg, #imm */
4094         ImmOperand *immOpnd = static_cast<ImmOperand *>(&opnd1);
4095         if (immOpnd->IsNegative()) {
4096             immOpnd->Negate();
4097             SelectSub(resOpnd, opnd0, *immOpnd, primType);
4098             return;
4099         }
4100         if (immOpnd->IsInBitSize(kMaxImmVal24Bits, 0)) {
4101             /*
4102              * ADD Wd|WSP, Wn|WSP, #imm{, shift} ; 32-bit general registers
4103              * ADD Xd|SP,  Xn|SP,  #imm{, shift} ; 64-bit general registers
4104              * imm : 0 ~ 4095, shift: none, LSL #0, or LSL #12
4105              * aarch64 assembly takes up to 24-bits, if the lower 12 bits is all 0
4106              */
4107             MOperator mOpCode = MOP_undef;
4108             Operand *newOpnd0 = &opnd0;
4109             if (!(immOpnd->IsInBitSize(kMaxImmVal12Bits, 0) ||
4110                   immOpnd->IsInBitSize(kMaxImmVal12Bits, kMaxImmVal12Bits))) {
4111                 /* process higher 12 bits */
4112                 ImmOperand &immOpnd2 =
4113                     CreateImmOperand(static_cast<int64>(static_cast<uint64>(immOpnd->GetValue()) >> kMaxImmVal12Bits),
4114                                      immOpnd->GetSize(), immOpnd->IsSignedValue());
4115                 mOpCode = is64Bits ? MOP_xaddrri24 : MOP_waddrri24;
4116                 Operand *tmpRes = IsAfterRegAlloc() ? &resOpnd : &CreateRegisterOperandOfType(primType);
4117                 BitShiftOperand &shiftopnd = CreateBitShiftOperand(BitShiftOperand::kLSL, kShiftAmount12, k64BitSize);
4118                 Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, *tmpRes, opnd0, immOpnd2, shiftopnd);
4119                 GetCurBB()->AppendInsn(newInsn);
4120                 immOpnd->ModuloByPow2(static_cast<int32>(kMaxImmVal12Bits));
4121                 newOpnd0 = tmpRes;
4122             }
4123             /* process lower 12  bits */
4124             mOpCode = is64Bits ? MOP_xaddrri12 : MOP_waddrri12;
4125             Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, *newOpnd0, *immOpnd);
4126             GetCurBB()->AppendInsn(newInsn);
4127             return;
4128         }
4129         /* load into register */
4130         int64 immVal = immOpnd->GetValue();
4131         int32 tail0bitNum = GetTail0BitNum(immVal);
4132         int32 head0bitNum = GetHead0BitNum(immVal);
4133         const int32 bitNum = (k64BitSizeInt - head0bitNum) - tail0bitNum;
4134         RegOperand &regOpnd = CreateRegisterOperandOfType(primType);
4135         if (isAfterRegAlloc) {
4136             RegType regty = GetRegTyFromPrimTy(primType);
4137             uint32 bytelen = GetPrimTypeSize(primType);
4138             regOpnd = GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(R16), bytelen, regty);
4139         }
4140         regno_t regNO0 = static_cast<RegOperand &>(opnd0).GetRegisterNumber();
4141         /* addrrrs do not support sp */
4142         if (bitNum <= k16ValidBit && regNO0 != RSP) {
4143             int64 newImm = (static_cast<uint64>(immVal) >> static_cast<uint32>(tail0bitNum)) & 0xFFFF;
4144             ImmOperand &immOpnd1 = CreateImmOperand(newImm, k16BitSize, false);
4145             SelectCopyImm(regOpnd, immOpnd1, primType);
4146             uint32 mopBadd = is64Bits ? MOP_xaddrrrs : MOP_waddrrrs;
4147             int32 bitLen = is64Bits ? kBitLenOfShift64Bits : kBitLenOfShift32Bits;
4148             BitShiftOperand &bitShiftOpnd =
4149                 CreateBitShiftOperand(BitShiftOperand::kLSL, static_cast<uint32>(tail0bitNum), bitLen);
4150             Insn &newInsn = GetInsnBuilder()->BuildInsn(mopBadd, resOpnd, opnd0, regOpnd, bitShiftOpnd);
4151             GetCurBB()->AppendInsn(newInsn);
4152             return;
4153         }
4154 
4155         SelectCopyImm(regOpnd, *immOpnd, primType);
4156         MOperator mOpCode = is64Bits ? MOP_xaddrrr : MOP_waddrrr;
4157         Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, opnd0, regOpnd);
4158         GetCurBB()->AppendInsn(newInsn);
4159     }
4160 }
4161 
SelectMadd(BinaryNode & node,Operand & opndM0,Operand & opndM1,Operand & opnd1,const BaseNode & parent)4162 Operand *AArch64CGFunc::SelectMadd(BinaryNode &node, Operand &opndM0, Operand &opndM1, Operand &opnd1,
4163                                    const BaseNode &parent)
4164 {
4165     PrimType dtype = node.GetPrimType();
4166     bool isSigned = IsSignedInteger(dtype);
4167     uint32 dsize = GetPrimTypeBitSize(dtype);
4168     bool is64Bits = (dsize == k64BitSize);
4169     /* promoted type */
4170     PrimType primType = is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32);
4171     RegOperand &resOpnd = GetOrCreateResOperand(parent, primType);
4172     SelectMadd(resOpnd, opndM0, opndM1, opnd1, primType);
4173     return &resOpnd;
4174 }
4175 
SelectMadd(Operand & resOpnd,Operand & opndM0,Operand & opndM1,Operand & opnd1,PrimType primType)4176 void AArch64CGFunc::SelectMadd(Operand &resOpnd, Operand &opndM0, Operand &opndM1, Operand &opnd1, PrimType primType)
4177 {
4178     Operand::OperandType opndM0Type = opndM0.GetKind();
4179     Operand::OperandType opndM1Type = opndM1.GetKind();
4180     Operand::OperandType opnd1Type = opnd1.GetKind();
4181     uint32 dsize = GetPrimTypeBitSize(primType);
4182     bool is64Bits = (dsize == k64BitSize);
4183 
4184     if (opndM0Type != Operand::kOpdRegister) {
4185         SelectMadd(resOpnd, SelectCopy(opndM0, primType, primType), opndM1, opnd1, primType);
4186         return;
4187     } else if (opndM1Type != Operand::kOpdRegister) {
4188         SelectMadd(resOpnd, opndM0, SelectCopy(opndM1, primType, primType), opnd1, primType);
4189         return;
4190     } else if (opnd1Type != Operand::kOpdRegister) {
4191         SelectMadd(resOpnd, opndM0, opndM1, SelectCopy(opnd1, primType, primType), primType);
4192         return;
4193     }
4194 
4195     DEBUG_ASSERT(IsPrimitiveInteger(primType), "NYI MAdd");
4196     MOperator mOp = is64Bits ? MOP_xmaddrrrr : MOP_wmaddrrrr;
4197     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opndM0, opndM1, opnd1));
4198 }
4199 
SelectCGArrayElemAdd(BinaryNode & node,const BaseNode & parent)4200 Operand &AArch64CGFunc::SelectCGArrayElemAdd(BinaryNode &node, const BaseNode &parent)
4201 {
4202     BaseNode *opnd0 = node.Opnd(0);
4203     BaseNode *opnd1 = node.Opnd(1);
4204     DEBUG_ASSERT(opnd1->GetOpCode() == OP_constval, "Internal error, opnd1->op should be OP_constval.");
4205 
4206     switch (opnd0->op) {
4207         case OP_regread: {
4208             RegreadNode *regreadNode = static_cast<RegreadNode *>(opnd0);
4209             return *SelectRegread(*regreadNode);
4210         }
4211         case OP_addrof: {
4212             AddrofNode *addrofNode = static_cast<AddrofNode *>(opnd0);
4213             MIRSymbol &symbol = *mirModule.CurFunction()->GetLocalOrGlobalSymbol(addrofNode->GetStIdx());
4214             DEBUG_ASSERT(addrofNode->GetFieldID() == 0, "For debug SelectCGArrayElemAdd.");
4215 
4216             Operand &result = GetOrCreateResOperand(parent, PTY_a64);
4217 
4218             /* OP_constval */
4219             ConstvalNode *constvalNode = static_cast<ConstvalNode *>(opnd1);
4220             MIRConst *mirConst = constvalNode->GetConstVal();
4221             MIRIntConst *mirIntConst = static_cast<MIRIntConst *>(mirConst);
4222             SelectAddrof(result, CreateStImmOperand(symbol, mirIntConst->GetExtValue(), 0));
4223 
4224             return result;
4225         }
4226         default:
4227             CHECK_FATAL(0, "Internal error, cannot handle opnd0.");
4228     }
4229 }
4230 
SelectSub(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)4231 void AArch64CGFunc::SelectSub(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
4232 {
4233     Operand::OperandType opnd1Type = opnd1.GetKind();
4234     uint32 dsize = GetPrimTypeBitSize(primType);
4235     bool is64Bits = (dsize == k64BitSize);
4236     bool isFloat = IsPrimitiveFloat(primType);
4237     Operand *opnd0Bak = &LoadIntoRegister(opnd0, primType);
4238     if (opnd1Type == Operand::kOpdRegister) {
4239         MOperator mOp = isFloat ? (is64Bits ? MOP_dsub : MOP_ssub) : (is64Bits ? MOP_xsubrrr : MOP_wsubrrr);
4240         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, *opnd0Bak, opnd1));
4241         return;
4242     }
4243 
4244     if ((opnd1Type != Operand::kOpdImmediate) && (opnd1Type != Operand::kOpdOffset)) {
4245         SelectSub(resOpnd, *opnd0Bak, SelectCopy(opnd1, primType, primType), primType);
4246         return;
4247     }
4248 
4249     ImmOperand *immOpnd = static_cast<ImmOperand *>(&opnd1);
4250     if (immOpnd->IsNegative()) {
4251         immOpnd->Negate();
4252         SelectAdd(resOpnd, *opnd0Bak, *immOpnd, primType);
4253         return;
4254     }
4255 
4256     int64 higher12BitVal = static_cast<int64>(static_cast<uint64>(immOpnd->GetValue()) >> kMaxImmVal12Bits);
4257     if (immOpnd->IsInBitSize(kMaxImmVal24Bits, 0) && higher12BitVal + 1 <= kMaxPimm8) {
4258         /*
4259          * SUB Wd|WSP, Wn|WSP, #imm{, shift} ; 32-bit general registers
4260          * SUB Xd|SP,  Xn|SP,  #imm{, shift} ; 64-bit general registers
4261          * imm : 0 ~ 4095, shift: none, LSL #0, or LSL #12
4262          * aarch64 assembly takes up to 24-bits, if the lower 12 bits is all 0
4263          * large offset is treated as sub (higher 12 bits + 4096) + add
4264          * it gives opportunities for combining add + ldr due to the characteristics of aarch64's load/store
4265          */
4266         MOperator mOpCode = MOP_undef;
4267         bool isSplitSub = false;
4268         if (!(immOpnd->IsInBitSize(kMaxImmVal12Bits, 0) || immOpnd->IsInBitSize(kMaxImmVal12Bits, kMaxImmVal12Bits))) {
4269             isSplitSub = true;
4270             /* process higher 12 bits */
4271             ImmOperand &immOpnd2 = CreateImmOperand(higher12BitVal + 1, immOpnd->GetSize(), immOpnd->IsSignedValue());
4272 
4273             mOpCode = is64Bits ? MOP_xsubrri24 : MOP_wsubrri24;
4274             BitShiftOperand &shiftopnd = CreateBitShiftOperand(BitShiftOperand::kLSL, kShiftAmount12, k64BitSize);
4275             Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, *opnd0Bak, immOpnd2, shiftopnd);
4276             GetCurBB()->AppendInsn(newInsn);
4277             immOpnd->ModuloByPow2(static_cast<int64>(kMaxImmVal12Bits));
4278             immOpnd->SetValue(static_cast<int64>(kMax12UnsignedImm) - immOpnd->GetValue());
4279             opnd0Bak = &resOpnd;
4280         }
4281         /* process lower 12 bits */
4282         mOpCode = isSplitSub ? (is64Bits ? MOP_xaddrri12 : MOP_waddrri12) : (is64Bits ? MOP_xsubrri12 : MOP_wsubrri12);
4283         Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, *opnd0Bak, *immOpnd);
4284         GetCurBB()->AppendInsn(newInsn);
4285         return;
4286     }
4287 
4288     /* load into register */
4289     int64 immVal = immOpnd->GetValue();
4290     int32 tail0bitNum = GetTail0BitNum(immVal);
4291     int32 head0bitNum = GetHead0BitNum(immVal);
4292     const int32 bitNum = (k64BitSizeInt - head0bitNum) - tail0bitNum;
4293     RegOperand &regOpnd = CreateRegisterOperandOfType(primType);
4294     if (isAfterRegAlloc) {
4295         RegType regty = GetRegTyFromPrimTy(primType);
4296         uint32 bytelen = GetPrimTypeSize(primType);
4297         regOpnd = GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(R16), bytelen, regty);
4298     }
4299 
4300     if (bitNum <= k16ValidBit) {
4301         int64 newImm = (static_cast<uint64>(immVal) >> static_cast<uint32>(tail0bitNum)) & 0xFFFF;
4302         ImmOperand &immOpnd1 = CreateImmOperand(newImm, k16BitSize, false);
4303         SelectCopyImm(regOpnd, immOpnd1, primType);
4304         uint32 mopBsub = is64Bits ? MOP_xsubrrrs : MOP_wsubrrrs;
4305         int32 bitLen = is64Bits ? kBitLenOfShift64Bits : kBitLenOfShift32Bits;
4306         BitShiftOperand &bitShiftOpnd =
4307             CreateBitShiftOperand(BitShiftOperand::kLSL, static_cast<uint32>(tail0bitNum), bitLen);
4308         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBsub, resOpnd, *opnd0Bak, regOpnd, bitShiftOpnd));
4309         return;
4310     }
4311 
4312     SelectCopyImm(regOpnd, *immOpnd, primType);
4313     MOperator mOpCode = is64Bits ? MOP_xsubrrr : MOP_wsubrrr;
4314     Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, *opnd0Bak, regOpnd);
4315     GetCurBB()->AppendInsn(newInsn);
4316 }
4317 
SelectSub(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4318 Operand *AArch64CGFunc::SelectSub(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4319 {
4320     PrimType dtype = node.GetPrimType();
4321     bool isSigned = IsSignedInteger(dtype);
4322     uint32 dsize = GetPrimTypeBitSize(dtype);
4323     bool is64Bits = (dsize == k64BitSize);
4324     bool isFloat = IsPrimitiveFloat(dtype);
4325     RegOperand *resOpnd = nullptr;
4326     if (!IsPrimitiveVector(dtype)) {
4327         /* promoted type */
4328         PrimType primType =
4329             isFloat ? dtype : ((is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32)));
4330         resOpnd = &GetOrCreateResOperand(parent, primType);
4331         SelectSub(*resOpnd, opnd0, opnd1, primType);
4332     } else {
4333         /* vector operands */
4334         resOpnd =
4335             SelectVectorBinOp(dtype, &opnd0, node.Opnd(0)->GetPrimType(), &opnd1, node.Opnd(1)->GetPrimType(), OP_sub);
4336     }
4337     return resOpnd;
4338 }
4339 
SelectMpy(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4340 Operand *AArch64CGFunc::SelectMpy(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4341 {
4342     PrimType dtype = node.GetPrimType();
4343     bool isSigned = IsSignedInteger(dtype);
4344     uint32 dsize = GetPrimTypeBitSize(dtype);
4345     bool is64Bits = (dsize == k64BitSize);
4346     bool isFloat = IsPrimitiveFloat(dtype);
4347     RegOperand *resOpnd = nullptr;
4348     if (!IsPrimitiveVector(dtype)) {
4349         /* promoted type */
4350         PrimType primType =
4351             isFloat ? dtype : ((is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32)));
4352         resOpnd = &GetOrCreateResOperand(parent, primType);
4353         SelectMpy(*resOpnd, opnd0, opnd1, primType);
4354     } else {
4355         resOpnd =
4356             SelectVectorBinOp(dtype, &opnd0, node.Opnd(0)->GetPrimType(), &opnd1, node.Opnd(1)->GetPrimType(), OP_mul);
4357     }
4358     return resOpnd;
4359 }
4360 
SelectMpy(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)4361 void AArch64CGFunc::SelectMpy(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
4362 {
4363     Operand::OperandType opnd0Type = opnd0.GetKind();
4364     Operand::OperandType opnd1Type = opnd1.GetKind();
4365     uint32 dsize = GetPrimTypeBitSize(primType);
4366     bool is64Bits = (dsize == k64BitSize);
4367 
4368     if (((opnd0Type == Operand::kOpdImmediate) || (opnd0Type == Operand::kOpdOffset) ||
4369          (opnd1Type == Operand::kOpdImmediate) || (opnd1Type == Operand::kOpdOffset)) &&
4370         IsPrimitiveInteger(primType)) {
4371         ImmOperand *imm = ((opnd0Type == Operand::kOpdImmediate) || (opnd0Type == Operand::kOpdOffset))
4372                               ? static_cast<ImmOperand *>(&opnd0)
4373                               : static_cast<ImmOperand *>(&opnd1);
4374         Operand *otherOp =
4375             ((opnd0Type == Operand::kOpdImmediate) || (opnd0Type == Operand::kOpdOffset)) ? &opnd1 : &opnd0;
4376         int64 immValue = llabs(imm->GetValue());
4377         if (immValue != 0 && (static_cast<uint64>(immValue) & (static_cast<uint64>(immValue) - 1)) == 0) {
4378             /* immValue is 1 << n */
4379             if (otherOp->GetKind() != Operand::kOpdRegister) {
4380                 otherOp = &SelectCopy(*otherOp, primType, primType);
4381             }
4382             int64 shiftVal = __builtin_ffsll(immValue);
4383             ImmOperand &shiftNum = CreateImmOperand(shiftVal - 1, dsize, false);
4384             SelectShift(resOpnd, *otherOp, shiftNum, kShiftLeft, primType);
4385             bool reachSignBit = (is64Bits && (shiftVal == k64BitSize)) || (!is64Bits && (shiftVal == k32BitSize));
4386             if (imm->GetValue() < 0 && !reachSignBit) {
4387                 SelectNeg(resOpnd, resOpnd, primType);
4388             }
4389 
4390             return;
4391         } else if (immValue > 2) { // immValue should larger than 2
4392             uint32 zeroNum = __builtin_ffsll(immValue) - 1;
4393             int64 headVal = static_cast<uint64>(immValue) >> zeroNum;
4394             /*
4395              * if (headVal - 1) & (headVal - 2) == 0, that is (immVal >> zeroNum) - 1 == 1 << n
4396              * otherOp * immVal = (otherOp * (immVal >> zeroNum) * (1 << zeroNum)
4397              * = (otherOp * ((immVal >> zeroNum) - 1) + otherOp) * (1 << zeroNum)
4398              */
4399             if (((static_cast<uint64>(headVal) - 1) & (static_cast<uint64>(headVal) - 2)) == 0) { // 2 see comment above
4400                 if (otherOp->GetKind() != Operand::kOpdRegister) {
4401                     otherOp = &SelectCopy(*otherOp, primType, primType);
4402                 }
4403                 ImmOperand &shiftNum1 = CreateImmOperand(__builtin_ffsll(headVal - 1) - 1, dsize, false);
4404                 RegOperand &tmpOpnd = CreateRegisterOperandOfType(primType);
4405                 SelectShift(tmpOpnd, *otherOp, shiftNum1, kShiftLeft, primType);
4406                 SelectAdd(resOpnd, *otherOp, tmpOpnd, primType);
4407                 ImmOperand &shiftNum2 = CreateImmOperand(zeroNum, dsize, false);
4408                 SelectShift(resOpnd, resOpnd, shiftNum2, kShiftLeft, primType);
4409                 if (imm->GetValue() < 0) {
4410                     SelectNeg(resOpnd, resOpnd, primType);
4411                 }
4412 
4413                 return;
4414             }
4415         }
4416     }
4417 
4418     if ((opnd0Type != Operand::kOpdRegister) && (opnd1Type != Operand::kOpdRegister)) {
4419         SelectMpy(resOpnd, SelectCopy(opnd0, primType, primType), opnd1, primType);
4420     } else if ((opnd0Type == Operand::kOpdRegister) && (opnd1Type != Operand::kOpdRegister)) {
4421         SelectMpy(resOpnd, opnd0, SelectCopy(opnd1, primType, primType), primType);
4422     } else if ((opnd0Type != Operand::kOpdRegister) && (opnd1Type == Operand::kOpdRegister)) {
4423         SelectMpy(resOpnd, opnd1, opnd0, primType);
4424     } else {
4425         DEBUG_ASSERT(IsPrimitiveFloat(primType) || IsPrimitiveInteger(primType), "NYI Mpy");
4426         MOperator mOp =
4427             IsPrimitiveFloat(primType) ? (is64Bits ? MOP_xvmuld : MOP_xvmuls) : (is64Bits ? MOP_xmulrrr : MOP_wmulrrr);
4428         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, opnd1));
4429     }
4430 }
4431 
SelectDiv(Operand & resOpnd,Operand & origOpnd0,Operand & opnd1,PrimType primType)4432 void AArch64CGFunc::SelectDiv(Operand &resOpnd, Operand &origOpnd0, Operand &opnd1, PrimType primType)
4433 {
4434     Operand &opnd0 = LoadIntoRegister(origOpnd0, primType);
4435     Operand::OperandType opnd0Type = opnd0.GetKind();
4436     Operand::OperandType opnd1Type = opnd1.GetKind();
4437     uint32 dsize = GetPrimTypeBitSize(primType);
4438     bool is64Bits = (dsize == k64BitSize);
4439 
4440     if (Globals::GetInstance()->GetOptimLevel() > CGOptions::kLevel0) {
4441         if (((opnd1Type == Operand::kOpdImmediate) || (opnd1Type == Operand::kOpdOffset)) &&
4442             IsSignedInteger(primType)) {
4443             ImmOperand *imm = static_cast<ImmOperand *>(&opnd1);
4444             int64 immValue = llabs(imm->GetValue());
4445             if ((immValue != 0) && (static_cast<uint64>(immValue) & (static_cast<uint64>(immValue) - 1)) == 0) {
4446                 if (immValue == 1) {
4447                     if (imm->GetValue() > 0) {
4448                         uint32 mOp = is64Bits ? MOP_xmovrr : MOP_wmovrr;
4449                         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0));
4450                     } else {
4451                         SelectNeg(resOpnd, opnd0, primType);
4452                     }
4453 
4454                     return;
4455                 }
4456                 int32 shiftNumber = __builtin_ffsll(immValue) - 1;
4457                 ImmOperand &shiftNum = CreateImmOperand(shiftNumber, dsize, false);
4458                 Operand &tmpOpnd = CreateRegisterOperandOfType(primType);
4459                 SelectShift(tmpOpnd, opnd0, CreateImmOperand(dsize - 1, dsize, false), kShiftAright, primType);
4460                 uint32 mopBadd = is64Bits ? MOP_xaddrrrs : MOP_waddrrrs;
4461                 int32 bitLen = is64Bits ? kBitLenOfShift64Bits : kBitLenOfShift32Bits;
4462                 BitShiftOperand &shiftOpnd = CreateBitShiftOperand(BitShiftOperand::kLSR, dsize - shiftNumber, bitLen);
4463                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBadd, tmpOpnd, opnd0, tmpOpnd, shiftOpnd));
4464                 SelectShift(resOpnd, tmpOpnd, shiftNum, kShiftAright, primType);
4465                 if (imm->GetValue() < 0) {
4466                     SelectNeg(resOpnd, resOpnd, primType);
4467                 }
4468 
4469                 return;
4470             }
4471         } else if (((opnd1Type == Operand::kOpdImmediate) || (opnd1Type == Operand::kOpdOffset)) &&
4472                    IsUnsignedInteger(primType)) {
4473             ImmOperand *imm = static_cast<ImmOperand *>(&opnd1);
4474             if (imm->GetValue() != 0) {
4475                 if ((imm->GetValue() > 0) &&
4476                     ((static_cast<uint64>(imm->GetValue()) & (static_cast<uint64>(imm->GetValue()) - 1)) == 0)) {
4477                     ImmOperand &shiftNum = CreateImmOperand(__builtin_ffsll(imm->GetValue()) - 1, dsize, false);
4478                     SelectShift(resOpnd, opnd0, shiftNum, kShiftLright, primType);
4479 
4480                     return;
4481                 } else if (imm->GetValue() < 0) {
4482                     SelectAArch64Cmp(opnd0, *imm, true, dsize);
4483                     SelectAArch64CSet(resOpnd, GetCondOperand(CC_CS), is64Bits);
4484 
4485                     return;
4486                 }
4487             }
4488         }
4489     }
4490 
4491     if (opnd0Type != Operand::kOpdRegister) {
4492         SelectDiv(resOpnd, SelectCopy(opnd0, primType, primType), opnd1, primType);
4493     } else if (opnd1Type != Operand::kOpdRegister) {
4494         SelectDiv(resOpnd, opnd0, SelectCopy(opnd1, primType, primType), primType);
4495     } else {
4496         DEBUG_ASSERT(IsPrimitiveFloat(primType) || IsPrimitiveInteger(primType), "NYI Div");
4497         MOperator mOp = IsPrimitiveFloat(primType)
4498                             ? (is64Bits ? MOP_ddivrrr : MOP_sdivrrr)
4499                             : (IsSignedInteger(primType) ? (is64Bits ? MOP_xsdivrrr : MOP_wsdivrrr)
4500                                                          : (is64Bits ? MOP_xudivrrr : MOP_wudivrrr));
4501         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, opnd1));
4502     }
4503 }
4504 
SelectDiv(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4505 Operand *AArch64CGFunc::SelectDiv(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4506 {
4507     PrimType dtype = node.GetPrimType();
4508     bool isSigned = IsSignedInteger(dtype);
4509     uint32 dsize = GetPrimTypeBitSize(dtype);
4510     bool is64Bits = (dsize == k64BitSize);
4511     bool isFloat = IsPrimitiveFloat(dtype);
4512     CHECK_FATAL(!IsPrimitiveVector(dtype), "NYI DIV vector operands");
4513     /* promoted type */
4514     PrimType primType =
4515         isFloat ? dtype : ((is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32)));
4516     RegOperand &resOpnd = GetOrCreateResOperand(parent, primType);
4517     SelectDiv(resOpnd, opnd0, opnd1, primType);
4518     return &resOpnd;
4519 }
4520 
SelectRem(Operand & resOpnd,Operand & lhsOpnd,Operand & rhsOpnd,PrimType primType,bool isSigned,bool is64Bits)4521 void AArch64CGFunc::SelectRem(Operand &resOpnd, Operand &lhsOpnd, Operand &rhsOpnd, PrimType primType, bool isSigned,
4522                               bool is64Bits)
4523 {
4524     Operand &opnd0 = LoadIntoRegister(lhsOpnd, primType);
4525     Operand &opnd1 = LoadIntoRegister(rhsOpnd, primType);
4526 
4527     DEBUG_ASSERT(IsPrimitiveInteger(primType), "Wrong type for REM");
4528     /*
4529      * printf("%d \n", 29 % 7 );
4530      * -> 1
4531      * printf("%u %d \n", (unsigned)-7, (unsigned)(-7) % 7 );
4532      * -> 4294967289 4
4533      * printf("%d \n", (-7) % 7 );
4534      * -> 0
4535      * printf("%d \n", 237 % -7 );
4536      * 6->
4537      * printf("implicit i->u conversion %d \n", ((unsigned)237) % -7 );
4538      * implicit conversion 237
4539 
4540      * http://stackoverflow.com/questions/35351470/obtaining-remainder-using-single-aarch64-instruction
4541      * input: x0=dividend, x1=divisor
4542      * udiv|sdiv x2, x0, x1
4543      * msub x3, x2, x1, x0  -- multply-sub : x3 <- x0 - x2*x1
4544      * result: x2=quotient, x3=remainder
4545      *
4546      * allocate temporary register
4547      */
4548     RegOperand &temp = CreateRegisterOperandOfType(primType);
4549     /*
4550      * mov     w1, #2
4551      * sdiv    wTemp, w0, w1
4552      * msub    wRespond, wTemp, w1, w0
4553      * ========>
4554      * asr     wTemp, w0, #31
4555      * lsr     wTemp, wTemp, #31  (#30 for 4, #29 for 8, ...)
4556      * add     wRespond, w0, wTemp
4557      * and     wRespond, wRespond, #1   (#3 for 4, #7 for 8, ...)
4558      * sub     wRespond, wRespond, w2
4559      *
4560      * if divde by 2
4561      * ========>
4562      * lsr     wTemp, w0, #31
4563      * add     wRespond, w0, wTemp
4564      * and     wRespond, wRespond, #1
4565      * sub     wRespond, wRespond, w2
4566      *
4567      * for unsigned rem op, just use and
4568      */
4569     if ((Globals::GetInstance()->GetOptimLevel() >= CGOptions::kLevel2)) {
4570         ImmOperand *imm = nullptr;
4571         Insn *movImmInsn = GetCurBB()->GetLastInsn();
4572         if (movImmInsn &&
4573             ((movImmInsn->GetMachineOpcode() == MOP_wmovri32) || (movImmInsn->GetMachineOpcode() == MOP_xmovri64)) &&
4574             movImmInsn->GetOperand(0).Equals(opnd1)) {
4575             /*
4576              * mov w1, #2
4577              * rem res, w0, w1
4578              */
4579             imm = static_cast<ImmOperand *>(&movImmInsn->GetOperand(kInsnSecondOpnd));
4580         } else if (opnd1.IsImmediate()) {
4581             /*
4582              * rem res, w0, #2
4583              */
4584             imm = static_cast<ImmOperand *>(&opnd1);
4585         }
4586         /* positive or negative do not have effect on the result */
4587         int64 dividor = 0;
4588         if (imm && (imm->GetValue() != LONG_MIN)) {
4589             dividor = abs(imm->GetValue());
4590         }
4591         const int64 Log2OfDividor = GetLog2(static_cast<uint64>(dividor));
4592         if ((dividor != 0) && (Log2OfDividor > 0)) {
4593             if (is64Bits) {
4594                 CHECK_FATAL(Log2OfDividor < k64BitSize, "imm out of bound");
4595                 if (isSigned) {
4596                     ImmOperand &rightShiftValue = CreateImmOperand(k64BitSize - Log2OfDividor, k64BitSize, isSigned);
4597                     if (Log2OfDividor != 1) {
4598                         /* 63->shift ALL , 32 ->32bit register */
4599                         ImmOperand &rightShiftAll = CreateImmOperand(63, k64BitSize, isSigned);
4600                         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xasrrri6, temp, opnd0, rightShiftAll));
4601 
4602                         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xlsrrri6, temp, temp, rightShiftValue));
4603                     } else {
4604                         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xlsrrri6, temp, opnd0, rightShiftValue));
4605                     }
4606                     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrrr, resOpnd, opnd0, temp));
4607                     ImmOperand &remBits = CreateImmOperand(dividor - 1, k64BitSize, isSigned);
4608                     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xandrri13, resOpnd, resOpnd, remBits));
4609                     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xsubrrr, resOpnd, resOpnd, temp));
4610                     return;
4611                 } else if (imm && imm->GetValue() > 0) {
4612                     ImmOperand &remBits = CreateImmOperand(dividor - 1, k64BitSize, isSigned);
4613                     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xandrri13, resOpnd, opnd0, remBits));
4614                     return;
4615                 }
4616             } else {
4617                 CHECK_FATAL(Log2OfDividor < k32BitSize, "imm out of bound");
4618                 if (isSigned) {
4619                     ImmOperand &rightShiftValue = CreateImmOperand(k32BitSize - Log2OfDividor, k32BitSize, isSigned);
4620                     if (Log2OfDividor != 1) {
4621                         /* 31->shift ALL , 32 ->32bit register */
4622                         ImmOperand &rightShiftAll = CreateImmOperand(31, k32BitSize, isSigned);
4623                         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wasrrri5, temp, opnd0, rightShiftAll));
4624 
4625                         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wlsrrri5, temp, temp, rightShiftValue));
4626                     } else {
4627                         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wlsrrri5, temp, opnd0, rightShiftValue));
4628                     }
4629 
4630                     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_waddrrr, resOpnd, opnd0, temp));
4631                     ImmOperand &remBits = CreateImmOperand(dividor - 1, k32BitSize, isSigned);
4632                     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wandrri12, resOpnd, resOpnd, remBits));
4633 
4634                     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wsubrrr, resOpnd, resOpnd, temp));
4635                     return;
4636                 } else if (imm && imm->GetValue() > 0) {
4637                     ImmOperand &remBits = CreateImmOperand(dividor - 1, k32BitSize, isSigned);
4638                     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wandrri12, resOpnd, opnd0, remBits));
4639                     return;
4640                 }
4641             }
4642         }
4643     }
4644 
4645     uint32 mopDiv = is64Bits ? (isSigned ? MOP_xsdivrrr : MOP_xudivrrr) : (isSigned ? MOP_wsdivrrr : MOP_wudivrrr);
4646     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopDiv, temp, opnd0, opnd1));
4647 
4648     uint32 mopSub = is64Bits ? MOP_xmsubrrrr : MOP_wmsubrrrr;
4649     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopSub, resOpnd, temp, opnd1, opnd0));
4650 }
4651 
SelectRem(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4652 Operand *AArch64CGFunc::SelectRem(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4653 {
4654     PrimType dtype = node.GetPrimType();
4655     DEBUG_ASSERT(IsPrimitiveInteger(dtype), "wrong type for rem");
4656     bool isSigned = IsSignedInteger(dtype);
4657     uint32 dsize = GetPrimTypeBitSize(dtype);
4658     bool is64Bits = (dsize == k64BitSize);
4659     CHECK_FATAL(!IsPrimitiveVector(dtype), "NYI DIV vector operands");
4660 
4661     /* promoted type */
4662     PrimType primType = ((is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32)));
4663     RegOperand &resOpnd = GetOrCreateResOperand(parent, primType);
4664     SelectRem(resOpnd, opnd0, opnd1, primType, isSigned, is64Bits);
4665     return &resOpnd;
4666 }
4667 
SelectLand(BinaryNode & node,Operand & lhsOpnd,Operand & rhsOpnd,const BaseNode & parent)4668 Operand *AArch64CGFunc::SelectLand(BinaryNode &node, Operand &lhsOpnd, Operand &rhsOpnd, const BaseNode &parent)
4669 {
4670     PrimType primType = node.GetPrimType();
4671     DEBUG_ASSERT(IsPrimitiveInteger(primType), "Land should be integer type");
4672     bool is64Bits = (GetPrimTypeBitSize(primType) == k64BitSize);
4673     RegOperand &resOpnd = GetOrCreateResOperand(parent, is64Bits ? PTY_u64 : PTY_u32);
4674     /*
4675      * OP0 band Op1
4676      * cmp  OP0, 0     # compare X0 with 0, sets Z bit
4677      * ccmp OP1, 0, 4 //==0100b, ne     # if(OP0!=0) cmp Op1 and 0, else NZCV <- 0100 makes OP0==0
4678      * cset RES, ne     # if Z==1(i.e., OP0==0||OP1==0) RES<-0, RES<-1
4679      */
4680     Operand &opnd0 = LoadIntoRegister(lhsOpnd, primType);
4681     SelectAArch64Cmp(opnd0, CreateImmOperand(0, primType, false), true, GetPrimTypeBitSize(primType));
4682     Operand &opnd1 = LoadIntoRegister(rhsOpnd, primType);
4683     constexpr int64 immValue4 = 4; //integer as comment above
4684     SelectAArch64CCmp(opnd1, CreateImmOperand(0, primType, false), CreateImmOperand(immValue4, PTY_u8, false),
4685                       GetCondOperand(CC_NE), is64Bits);
4686     SelectAArch64CSet(resOpnd, GetCondOperand(CC_NE), is64Bits);
4687     return &resOpnd;
4688 }
4689 
SelectLor(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent,bool parentIsBr)4690 Operand *AArch64CGFunc::SelectLor(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent,
4691                                   bool parentIsBr)
4692 {
4693     PrimType primType = node.GetPrimType();
4694     DEBUG_ASSERT(IsPrimitiveInteger(primType), "Lior should be integer type");
4695     bool is64Bits = (GetPrimTypeBitSize(primType) == k64BitSize);
4696     RegOperand &resOpnd = GetOrCreateResOperand(parent, is64Bits ? PTY_u64 : PTY_u32);
4697     /*
4698      * OP0 band Op1
4699      * cmp  OP0, 0     # compare X0 with 0, sets Z bit
4700      * ccmp OP1, 0, 0 //==0100b, eq     # if(OP0==0,eq) cmp Op1 and 0, else NZCV <- 0000 makes OP0!=0
4701      * cset RES, ne     # if Z==1(i.e., OP0==0&&OP1==0) RES<-0, RES<-1
4702      */
4703     if (parentIsBr && !is64Bits && opnd0.IsRegister() && (static_cast<RegOperand *>(&opnd0)->GetValidBitsNum() == 1) &&
4704         opnd1.IsRegister() && (static_cast<RegOperand *>(&opnd1)->GetValidBitsNum() == 1)) {
4705         uint32 mOp = MOP_wiorrrr;
4706         static_cast<RegOperand &>(resOpnd).SetValidBitsNum(1);
4707         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, opnd1));
4708     } else {
4709         SelectBior(resOpnd, opnd0, opnd1, primType);
4710         SelectAArch64Cmp(resOpnd, CreateImmOperand(0, primType, false), true, GetPrimTypeBitSize(primType));
4711         SelectAArch64CSet(resOpnd, GetCondOperand(CC_NE), is64Bits);
4712     }
4713     return &resOpnd;
4714 }
4715 
SelectCmpOp(Operand & resOpnd,Operand & lhsOpnd,Operand & rhsOpnd,Opcode opcode,PrimType primType,const BaseNode & parent)4716 void AArch64CGFunc::SelectCmpOp(Operand &resOpnd, Operand &lhsOpnd, Operand &rhsOpnd, Opcode opcode, PrimType primType,
4717                                 const BaseNode &parent)
4718 {
4719     uint32 dsize = resOpnd.GetSize();
4720     bool isFloat = IsPrimitiveFloat(primType);
4721     Operand &opnd0 = LoadIntoRegister(lhsOpnd, primType);
4722 
4723     /*
4724      * most of FP constants are passed as MemOperand
4725      * except 0.0 which is passed as kOpdFPImmediate
4726      */
4727     Operand::OperandType opnd1Type = rhsOpnd.GetKind();
4728     Operand *opnd1 = &rhsOpnd;
4729     if ((opnd1Type != Operand::kOpdImmediate) && (opnd1Type != Operand::kOpdFPImmediate) &&
4730         (opnd1Type != Operand::kOpdOffset)) {
4731         opnd1 = &LoadIntoRegister(rhsOpnd, primType);
4732     }
4733 
4734     bool unsignedIntegerComparison = !isFloat && !IsSignedInteger(primType);
4735     /*
4736      * OP_cmp, OP_cmpl, OP_cmpg
4737      * <cmp> OP0, OP1  ; fcmp for OP_cmpl/OP_cmpg, cmp/fcmpe for OP_cmp
4738      * CSINV RES, WZR, WZR, GE
4739      * CSINC RES, RES, WZR, LE
4740      * if OP_cmpl, CSINV RES, RES, WZR, VC (no overflow)
4741      * if OP_cmpg, CSINC RES, RES, WZR, VC (no overflow)
4742      */
4743     RegOperand &xzr = GetZeroOpnd(dsize);
4744     if ((opcode == OP_cmpl) || (opcode == OP_cmpg)) {
4745         DEBUG_ASSERT(isFloat, "incorrect operand types");
4746         SelectTargetFPCmpQuiet(opnd0, *opnd1, GetPrimTypeBitSize(primType));
4747         SelectAArch64CSINV(resOpnd, xzr, xzr, GetCondOperand(CC_GE), (dsize == k64BitSize));
4748         SelectAArch64CSINC(resOpnd, resOpnd, xzr, GetCondOperand(CC_LE), (dsize == k64BitSize));
4749         if (opcode == OP_cmpl) {
4750             SelectAArch64CSINV(resOpnd, resOpnd, xzr, GetCondOperand(CC_VC), (dsize == k64BitSize));
4751         } else {
4752             SelectAArch64CSINC(resOpnd, resOpnd, xzr, GetCondOperand(CC_VC), (dsize == k64BitSize));
4753         }
4754         return;
4755     }
4756 
4757     if (opcode == OP_cmp) {
4758         SelectAArch64Cmp(opnd0, *opnd1, !isFloat, GetPrimTypeBitSize(primType));
4759         if (unsignedIntegerComparison) {
4760             SelectAArch64CSINV(resOpnd, xzr, xzr, GetCondOperand(CC_HS), (dsize == k64BitSize));
4761             SelectAArch64CSINC(resOpnd, resOpnd, xzr, GetCondOperand(CC_LS), (dsize == k64BitSize));
4762         } else {
4763             SelectAArch64CSINV(resOpnd, xzr, xzr, GetCondOperand(CC_GE), (dsize == k64BitSize));
4764             SelectAArch64CSINC(resOpnd, resOpnd, xzr, GetCondOperand(CC_LE), (dsize == k64BitSize));
4765         }
4766         return;
4767     }
4768 
4769     // lt u8 i32 ( xxx, 0 ) => get sign bit
4770     if ((opcode == OP_lt) && opnd0.IsRegister() && opnd1->IsImmediate() &&
4771         (static_cast<ImmOperand *>(opnd1)->GetValue() == 0) && parent.GetOpCode() != OP_select && !isFloat) {
4772         bool is64Bits = (opnd0.GetSize() == k64BitSize);
4773         if (!unsignedIntegerComparison) {
4774             int32 bitLen = is64Bits ? kBitLenOfShift64Bits : kBitLenOfShift32Bits;
4775             ImmOperand &shiftNum = CreateImmOperand(is64Bits ? kHighestBitOf64Bits : kHighestBitOf32Bits,
4776                                                     static_cast<uint32>(bitLen), false);
4777             MOperator mOpCode = is64Bits ? MOP_xlsrrri6 : MOP_wlsrrri5;
4778             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, opnd0, shiftNum));
4779             return;
4780         }
4781         ImmOperand &constNum = CreateImmOperand(0, is64Bits ? k64BitSize : k32BitSize, false);
4782         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(is64Bits ? MOP_xmovri64 : MOP_wmovri32, resOpnd, constNum));
4783         return;
4784     }
4785     SelectAArch64Cmp(opnd0, *opnd1, !isFloat, GetPrimTypeBitSize(primType));
4786 
4787     ConditionCode cc = CC_EQ;
4788     // need to handle unordered situation here.
4789     switch (opcode) {
4790         case OP_eq:
4791             cc = CC_EQ;
4792             break;
4793         case OP_ne:
4794             cc = isFloat ? CC_MI : CC_NE;
4795             break;
4796         case OP_le:
4797             cc = isFloat ? CC_LS : unsignedIntegerComparison ? CC_LS : CC_LE;
4798             break;
4799         case OP_ge:
4800             cc = unsignedIntegerComparison ? CC_HS : CC_GE;
4801             break;
4802         case OP_gt:
4803             cc = unsignedIntegerComparison ? CC_HI : CC_GT;
4804             break;
4805         case OP_lt:
4806             cc = isFloat ? CC_MI : unsignedIntegerComparison ? CC_LO : CC_LT;
4807             break;
4808         default:
4809             CHECK_FATAL(false, "illegal logical operator");
4810     }
4811     SelectAArch64CSet(resOpnd, GetCondOperand(cc), (dsize == k64BitSize));
4812     if (isFloat && opcode == OP_ne) {
4813         SelectAArch64CSINC(resOpnd, resOpnd, xzr, GetCondOperand(CC_LE), (dsize == k64BitSize));
4814     }
4815 }
4816 
SelectCmpOp(CompareNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4817 Operand *AArch64CGFunc::SelectCmpOp(CompareNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4818 {
4819     RegOperand *resOpnd = nullptr;
4820     if (!IsPrimitiveVector(node.GetPrimType())) {
4821         resOpnd = &GetOrCreateResOperand(parent, node.GetPrimType());
4822         SelectCmpOp(*resOpnd, opnd0, opnd1, node.GetOpCode(), node.GetOpndType(), parent);
4823     } else {
4824         resOpnd = SelectVectorCompare(&opnd0, node.Opnd(0)->GetPrimType(), &opnd1, node.Opnd(1)->GetPrimType(),
4825                                       node.GetOpCode());
4826     }
4827     return resOpnd;
4828 }
4829 
SelectTargetFPCmpQuiet(Operand & o0,Operand & o1,uint32 dsize)4830 void AArch64CGFunc::SelectTargetFPCmpQuiet(Operand &o0, Operand &o1, uint32 dsize)
4831 {
4832     MOperator mOpCode = 0;
4833     if (o1.GetKind() == Operand::kOpdFPImmediate) {
4834         CHECK_FATAL(static_cast<ImmOperand &>(o0).GetValue() == 0, "NIY");
4835         mOpCode = (dsize == k64BitSize) ? MOP_dcmpqri : (dsize == k32BitSize) ? MOP_scmpqri : MOP_hcmpqri;
4836     } else if (o1.GetKind() == Operand::kOpdRegister) {
4837         mOpCode = (dsize == k64BitSize) ? MOP_dcmpqrr : (dsize == k32BitSize) ? MOP_scmpqrr : MOP_hcmpqrr;
4838     } else {
4839         CHECK_FATAL(false, "unsupported operand type");
4840     }
4841     Operand &rflag = GetOrCreateRflag();
4842     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, rflag, o0, o1));
4843 }
4844 
SelectAArch64Cmp(Operand & o0,Operand & o1,bool isIntType,uint32 dsize)4845 void AArch64CGFunc::SelectAArch64Cmp(Operand &o0, Operand &o1, bool isIntType, uint32 dsize)
4846 {
4847     MOperator mOpCode = 0;
4848     Operand *newO1 = &o1;
4849     if (isIntType) {
4850         if ((o1.GetKind() == Operand::kOpdImmediate) || (o1.GetKind() == Operand::kOpdOffset)) {
4851             ImmOperand *immOpnd = static_cast<ImmOperand *>(&o1);
4852             /*
4853              * imm : 0 ~ 4095, shift: none, LSL #0, or LSL #12
4854              * aarch64 assembly takes up to 24-bits, if the lower 12 bits is all 0
4855              */
4856             if (immOpnd->IsInBitSize(kMaxImmVal12Bits, 0) || immOpnd->IsInBitSize(kMaxImmVal12Bits, kMaxImmVal12Bits)) {
4857                 mOpCode = (dsize == k64BitSize) ? MOP_xcmpri : MOP_wcmpri;
4858             } else {
4859                 /* load into register */
4860                 PrimType ptype = (dsize == k64BitSize) ? PTY_i64 : PTY_i32;
4861                 newO1 = &SelectCopy(o1, ptype, ptype);
4862                 mOpCode = (dsize == k64BitSize) ? MOP_xcmprr : MOP_wcmprr;
4863             }
4864         } else if (o1.GetKind() == Operand::kOpdRegister) {
4865             mOpCode = (dsize == k64BitSize) ? MOP_xcmprr : MOP_wcmprr;
4866         } else {
4867             CHECK_FATAL(false, "unsupported operand type");
4868         }
4869     } else { /* float */
4870         if (o1.GetKind() == Operand::kOpdFPImmediate) {
4871             CHECK_FATAL(static_cast<ImmOperand &>(o1).GetValue() == 0, "NIY");
4872             mOpCode = (dsize == k64BitSize) ? MOP_dcmperi : ((dsize == k32BitSize) ? MOP_scmperi : MOP_hcmperi);
4873         } else if (o1.GetKind() == Operand::kOpdRegister) {
4874             mOpCode = (dsize == k64BitSize) ? MOP_dcmperr : ((dsize == k32BitSize) ? MOP_scmperr : MOP_hcmperr);
4875         } else {
4876             CHECK_FATAL(false, "unsupported operand type");
4877         }
4878     }
4879     DEBUG_ASSERT(mOpCode != 0, "mOpCode undefined");
4880     Operand &rflag = GetOrCreateRflag();
4881     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, rflag, o0, *newO1));
4882 }
4883 
SelectAArch64CCmp(Operand & o,Operand & i,Operand & nzcv,CondOperand & cond,bool is64Bits)4884 void AArch64CGFunc::SelectAArch64CCmp(Operand &o, Operand &i, Operand &nzcv, CondOperand &cond, bool is64Bits)
4885 {
4886     uint32 mOpCode = is64Bits ? MOP_xccmpriic : MOP_wccmpriic;
4887     Operand &rflag = GetOrCreateRflag();
4888     std::vector<Operand *> opndVec;
4889     opndVec.push_back(&rflag);
4890     opndVec.push_back(&o);
4891     opndVec.push_back(&i);
4892     opndVec.push_back(&nzcv);
4893     opndVec.push_back(&cond);
4894     opndVec.push_back(&rflag);
4895     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, opndVec));
4896 }
4897 
SelectAArch64CSet(Operand & r,CondOperand & cond,bool is64Bits)4898 void AArch64CGFunc::SelectAArch64CSet(Operand &r, CondOperand &cond, bool is64Bits)
4899 {
4900     MOperator mOpCode = is64Bits ? MOP_xcsetrc : MOP_wcsetrc;
4901     Operand &rflag = GetOrCreateRflag();
4902     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, r, cond, rflag));
4903 }
4904 
SelectAArch64CSINV(Operand & res,Operand & o0,Operand & o1,CondOperand & cond,bool is64Bits)4905 void AArch64CGFunc::SelectAArch64CSINV(Operand &res, Operand &o0, Operand &o1, CondOperand &cond, bool is64Bits)
4906 {
4907     MOperator mOpCode = is64Bits ? MOP_xcsinvrrrc : MOP_wcsinvrrrc;
4908     Operand &rflag = GetOrCreateRflag();
4909     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, res, o0, o1, cond, rflag));
4910 }
4911 
SelectAArch64CSINC(Operand & res,Operand & o0,Operand & o1,CondOperand & cond,bool is64Bits)4912 void AArch64CGFunc::SelectAArch64CSINC(Operand &res, Operand &o0, Operand &o1, CondOperand &cond, bool is64Bits)
4913 {
4914     MOperator mOpCode = is64Bits ? MOP_xcsincrrrc : MOP_wcsincrrrc;
4915     Operand &rflag = GetOrCreateRflag();
4916     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, res, o0, o1, cond, rflag));
4917 }
4918 
SelectBand(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4919 Operand *AArch64CGFunc::SelectBand(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4920 {
4921     return SelectRelationOperator(kAND, node, opnd0, opnd1, parent);
4922 }
4923 
SelectBand(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)4924 void AArch64CGFunc::SelectBand(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
4925 {
4926     SelectRelationOperator(kAND, resOpnd, opnd0, opnd1, primType);
4927 }
4928 
SelectRelationOperator(RelationOperator operatorCode,const BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4929 Operand *AArch64CGFunc::SelectRelationOperator(RelationOperator operatorCode, const BinaryNode &node, Operand &opnd0,
4930                                                Operand &opnd1, const BaseNode &parent)
4931 {
4932     PrimType dtype = node.GetPrimType();
4933     bool isSigned = IsSignedInteger(dtype);
4934     uint32 dsize = GetPrimTypeBitSize(dtype);
4935     bool is64Bits = (dsize == k64BitSize);
4936     RegOperand *resOpnd = nullptr;
4937     if (!IsPrimitiveVector(dtype)) {
4938         PrimType primType =
4939             is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32); /* promoted type */
4940         resOpnd = &GetOrCreateResOperand(parent, primType);
4941         SelectRelationOperator(operatorCode, *resOpnd, opnd0, opnd1, primType);
4942     } else {
4943         /* vector operations */
4944         resOpnd = SelectVectorBitwiseOp(dtype, &opnd0, node.Opnd(0)->GetPrimType(), &opnd1, node.Opnd(1)->GetPrimType(),
4945                                         (operatorCode == kAND) ? OP_band : (operatorCode == kIOR ? OP_bior : OP_bxor));
4946     }
4947     return resOpnd;
4948 }
4949 
SelectRelationMop(RelationOperator operatorCode,RelationOperatorOpndPattern opndPattern,bool is64Bits,bool isBitmaskImmediate,bool isBitNumLessThan16) const4950 MOperator AArch64CGFunc::SelectRelationMop(RelationOperator operatorCode, RelationOperatorOpndPattern opndPattern,
4951                                            bool is64Bits, bool isBitmaskImmediate, bool isBitNumLessThan16) const
4952 {
4953     MOperator mOp = MOP_undef;
4954     if (opndPattern == kRegReg) {
4955         switch (operatorCode) {
4956             case kAND:
4957                 mOp = is64Bits ? MOP_xandrrr : MOP_wandrrr;
4958                 break;
4959             case kIOR:
4960                 mOp = is64Bits ? MOP_xiorrrr : MOP_wiorrrr;
4961                 break;
4962             case kEOR:
4963                 mOp = is64Bits ? MOP_xeorrrr : MOP_weorrrr;
4964                 break;
4965             default:
4966                 break;
4967         }
4968         return mOp;
4969     }
4970     /* opndPattern == KRegImm */
4971     if (isBitmaskImmediate) {
4972         switch (operatorCode) {
4973             case kAND:
4974                 mOp = is64Bits ? MOP_xandrri13 : MOP_wandrri12;
4975                 break;
4976             case kIOR:
4977                 mOp = is64Bits ? MOP_xiorrri13 : MOP_wiorrri12;
4978                 break;
4979             case kEOR:
4980                 mOp = is64Bits ? MOP_xeorrri13 : MOP_weorrri12;
4981                 break;
4982             default:
4983                 break;
4984         }
4985         return mOp;
4986     }
4987     /* normal imm value */
4988     if (isBitNumLessThan16) {
4989         switch (operatorCode) {
4990             case kAND:
4991                 mOp = is64Bits ? MOP_xandrrrs : MOP_wandrrrs;
4992                 break;
4993             case kIOR:
4994                 mOp = is64Bits ? MOP_xiorrrrs : MOP_wiorrrrs;
4995                 break;
4996             case kEOR:
4997                 mOp = is64Bits ? MOP_xeorrrrs : MOP_weorrrrs;
4998                 break;
4999             default:
5000                 break;
5001         }
5002         return mOp;
5003     }
5004     return mOp;
5005 }
5006 
SelectRelationOperator(RelationOperator operatorCode,Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)5007 void AArch64CGFunc::SelectRelationOperator(RelationOperator operatorCode, Operand &resOpnd, Operand &opnd0,
5008                                            Operand &opnd1, PrimType primType)
5009 {
5010     Operand::OperandType opnd0Type = opnd0.GetKind();
5011     Operand::OperandType opnd1Type = opnd1.GetKind();
5012     uint32 dsize = GetPrimTypeBitSize(primType);
5013     bool is64Bits = (dsize == k64BitSize);
5014     /* op #imm. #imm */
5015     if ((opnd0Type != Operand::kOpdRegister) && (opnd1Type != Operand::kOpdRegister)) {
5016         SelectRelationOperator(operatorCode, resOpnd, SelectCopy(opnd0, primType, primType), opnd1, primType);
5017         return;
5018     }
5019     /* op #imm, reg -> op reg, #imm */
5020     if ((opnd0Type != Operand::kOpdRegister) && (opnd1Type == Operand::kOpdRegister)) {
5021         SelectRelationOperator(operatorCode, resOpnd, opnd1, opnd0, primType);
5022         return;
5023     }
5024     /* op reg, reg */
5025     if ((opnd0Type == Operand::kOpdRegister) && (opnd1Type == Operand::kOpdRegister)) {
5026         DEBUG_ASSERT(IsPrimitiveInteger(primType), "NYI band");
5027         MOperator mOp = SelectRelationMop(operatorCode, kRegReg, is64Bits, false, false);
5028         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, opnd1));
5029         return;
5030     }
5031     /* op reg, #imm */
5032     if ((opnd0Type == Operand::kOpdRegister) && (opnd1Type != Operand::kOpdRegister)) {
5033         if (!((opnd1Type == Operand::kOpdImmediate) || (opnd1Type == Operand::kOpdOffset))) {
5034             SelectRelationOperator(operatorCode, resOpnd, opnd0, SelectCopy(opnd1, primType, primType), primType);
5035             return;
5036         }
5037 
5038         ImmOperand *immOpnd = static_cast<ImmOperand *>(&opnd1);
5039         if (immOpnd->IsZero()) {
5040             if (operatorCode == kAND) {
5041                 uint32 mopMv = is64Bits ? MOP_xmovrr : MOP_wmovrr;
5042                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopMv, resOpnd, GetZeroOpnd(dsize)));
5043             } else if ((operatorCode == kIOR) || (operatorCode == kEOR)) {
5044                 SelectCopy(resOpnd, primType, opnd0, primType);
5045             }
5046         } else if ((immOpnd->IsAllOnes()) || (!is64Bits && immOpnd->IsAllOnes32bit())) {
5047             if (operatorCode == kAND) {
5048                 SelectCopy(resOpnd, primType, opnd0, primType);
5049             } else if (operatorCode == kIOR) {
5050                 uint32 mopMovn = is64Bits ? MOP_xmovnri16 : MOP_wmovnri16;
5051                 ImmOperand &src16 = CreateImmOperand(0, k16BitSize, false);
5052                 BitShiftOperand *lslOpnd = GetLogicalShiftLeftOperand(0, is64Bits);
5053                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopMovn, resOpnd, src16, *lslOpnd));
5054             } else if (operatorCode == kEOR) {
5055                 SelectMvn(resOpnd, opnd0, primType);
5056             }
5057         } else if (immOpnd->IsBitmaskImmediate()) {
5058             MOperator mOp = SelectRelationMop(operatorCode, kRegImm, is64Bits, true, false);
5059             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, opnd1));
5060         } else {
5061             int64 immVal = immOpnd->GetValue();
5062             int32 tail0BitNum = GetTail0BitNum(immVal);
5063             int32 head0BitNum = GetHead0BitNum(immVal);
5064             const int32 bitNum = (k64BitSizeInt - head0BitNum) - tail0BitNum;
5065             RegOperand &regOpnd = CreateRegisterOperandOfType(primType);
5066 
5067             if (bitNum <= k16ValidBit) {
5068                 int64 newImm = (static_cast<uint64>(immVal) >> static_cast<uint32>(tail0BitNum)) & 0xFFFF;
5069                 ImmOperand &immOpnd1 = CreateImmOperand(newImm, k16BitSize, false);
5070                 SelectCopyImm(regOpnd, immOpnd1, primType);
5071                 MOperator mOp = SelectRelationMop(operatorCode, kRegImm, is64Bits, false, true);
5072                 int32 bitLen = is64Bits ? kBitLenOfShift64Bits : kBitLenOfShift32Bits;
5073                 BitShiftOperand &shiftOpnd =
5074                     CreateBitShiftOperand(BitShiftOperand::kLSL, static_cast<uint32>(tail0BitNum), bitLen);
5075                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, regOpnd, shiftOpnd));
5076             } else {
5077                 SelectCopyImm(regOpnd, *immOpnd, primType);
5078                 MOperator mOp = SelectRelationMop(operatorCode, kRegReg, is64Bits, false, false);
5079                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, regOpnd));
5080             }
5081         }
5082     }
5083 }
5084 
SelectBior(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5085 Operand *AArch64CGFunc::SelectBior(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5086 {
5087     return SelectRelationOperator(kIOR, node, opnd0, opnd1, parent);
5088 }
5089 
SelectBior(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)5090 void AArch64CGFunc::SelectBior(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
5091 {
5092     SelectRelationOperator(kIOR, resOpnd, opnd0, opnd1, primType);
5093 }
5094 
SelectMinOrMax(bool isMin,const BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5095 Operand *AArch64CGFunc::SelectMinOrMax(bool isMin, const BinaryNode &node, Operand &opnd0, Operand &opnd1,
5096                                        const BaseNode &parent)
5097 {
5098     PrimType dtype = node.GetPrimType();
5099     bool isSigned = IsSignedInteger(dtype);
5100     uint32 dsize = GetPrimTypeBitSize(dtype);
5101     bool is64Bits = (dsize == k64BitSize);
5102     bool isFloat = IsPrimitiveFloat(dtype);
5103     /* promoted type */
5104     PrimType primType = isFloat ? dtype : (is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32));
5105     RegOperand &resOpnd = GetOrCreateResOperand(parent, primType);
5106     SelectMinOrMax(isMin, resOpnd, opnd0, opnd1, primType);
5107     return &resOpnd;
5108 }
5109 
SelectMinOrMax(bool isMin,Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)5110 void AArch64CGFunc::SelectMinOrMax(bool isMin, Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
5111 {
5112     uint32 dsize = GetPrimTypeBitSize(primType);
5113     bool is64Bits = (dsize == k64BitSize);
5114     if (IsPrimitiveInteger(primType)) {
5115         RegOperand &regOpnd0 = LoadIntoRegister(opnd0, primType);
5116         Operand &regOpnd1 = LoadIntoRegister(opnd1, primType);
5117         SelectAArch64Cmp(regOpnd0, regOpnd1, true, dsize);
5118         Operand &newResOpnd = LoadIntoRegister(resOpnd, primType);
5119         if (isMin) {
5120             CondOperand &cc = IsSignedInteger(primType) ? GetCondOperand(CC_LT) : GetCondOperand(CC_LO);
5121             SelectAArch64Select(newResOpnd, regOpnd0, regOpnd1, cc, true, dsize);
5122         } else {
5123             CondOperand &cc = IsSignedInteger(primType) ? GetCondOperand(CC_GT) : GetCondOperand(CC_HI);
5124             SelectAArch64Select(newResOpnd, regOpnd0, regOpnd1, cc, true, dsize);
5125         }
5126     } else if (IsPrimitiveFloat(primType)) {
5127         RegOperand &regOpnd0 = LoadIntoRegister(opnd0, primType);
5128         RegOperand &regOpnd1 = LoadIntoRegister(opnd1, primType);
5129         SelectFMinFMax(resOpnd, regOpnd0, regOpnd1, is64Bits, isMin);
5130     } else {
5131         CHECK_FATAL(false, "NIY type max or min");
5132     }
5133 }
5134 
SelectMin(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5135 Operand *AArch64CGFunc::SelectMin(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5136 {
5137     return SelectMinOrMax(true, node, opnd0, opnd1, parent);
5138 }
5139 
SelectMin(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)5140 void AArch64CGFunc::SelectMin(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
5141 {
5142     SelectMinOrMax(true, resOpnd, opnd0, opnd1, primType);
5143 }
5144 
SelectMax(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5145 Operand *AArch64CGFunc::SelectMax(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5146 {
5147     return SelectMinOrMax(false, node, opnd0, opnd1, parent);
5148 }
5149 
SelectMax(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)5150 void AArch64CGFunc::SelectMax(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
5151 {
5152     SelectMinOrMax(false, resOpnd, opnd0, opnd1, primType);
5153 }
5154 
SelectFMinFMax(Operand & resOpnd,Operand & opnd0,Operand & opnd1,bool is64Bits,bool isMin)5155 void AArch64CGFunc::SelectFMinFMax(Operand &resOpnd, Operand &opnd0, Operand &opnd1, bool is64Bits, bool isMin)
5156 {
5157     uint32 mOpCode = isMin ? (is64Bits ? MOP_xfminrrr : MOP_wfminrrr) : (is64Bits ? MOP_xfmaxrrr : MOP_wfmaxrrr);
5158     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, opnd0, opnd1));
5159 }
5160 
SelectBxor(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5161 Operand *AArch64CGFunc::SelectBxor(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5162 {
5163     return SelectRelationOperator(kEOR, node, opnd0, opnd1, parent);
5164 }
5165 
SelectBxor(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)5166 void AArch64CGFunc::SelectBxor(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
5167 {
5168     SelectRelationOperator(kEOR, resOpnd, opnd0, opnd1, primType);
5169 }
5170 
SelectShift(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5171 Operand *AArch64CGFunc::SelectShift(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5172 {
5173     PrimType dtype = node.GetPrimType();
5174     bool isSigned = IsSignedInteger(dtype);
5175     uint32 dsize = GetPrimTypeBitSize(dtype);
5176     bool is64Bits = (dsize == k64BitSize);
5177     bool isFloat = IsPrimitiveFloat(dtype);
5178     RegOperand *resOpnd = nullptr;
5179     Opcode opcode = node.GetOpCode();
5180 
5181     bool isOneElemVector = false;
5182     BaseNode *expr = node.Opnd(0);
5183     if (expr->GetOpCode() == OP_dread) {
5184         MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(static_cast<DreadNode *>(expr)->GetStIdx());
5185         isOneElemVector = symbol->GetAttr(ATTR_oneelem_simd);
5186     }
5187 
5188     Operand *opd0 = &opnd0;
5189     PrimType otyp0 = expr->GetPrimType();
5190     if (IsPrimitiveVector(dtype) && opnd0.IsConstImmediate()) {
5191         opd0 = SelectVectorFromScalar(dtype, opd0, node.Opnd(0)->GetPrimType());
5192         otyp0 = dtype;
5193     }
5194 
5195     if (IsPrimitiveVector(dtype) && opnd1.IsConstImmediate()) {
5196         int64 sConst = static_cast<ImmOperand &>(opnd1).GetValue();
5197         resOpnd = SelectVectorShiftImm(dtype, opd0, &opnd1, static_cast<int32>(sConst), opcode);
5198     } else if ((IsPrimitiveVector(dtype) || isOneElemVector) && !opnd1.IsConstImmediate()) {
5199         resOpnd = SelectVectorShift(dtype, opd0, otyp0, &opnd1, node.Opnd(1)->GetPrimType(), opcode);
5200     } else {
5201         PrimType primType =
5202             isFloat ? dtype : (is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32));
5203         resOpnd = &GetOrCreateResOperand(parent, primType);
5204         ShiftDirection direct = (opcode == OP_lshr) ? kShiftLright : ((opcode == OP_ashr) ? kShiftAright : kShiftLeft);
5205         SelectShift(*resOpnd, opnd0, opnd1, direct, primType);
5206     }
5207 
5208     if (dtype == PTY_i16) {
5209         MOperator exOp = is64Bits ? MOP_xsxth64 : MOP_xsxth32;
5210         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(exOp, *resOpnd, *resOpnd));
5211     } else if (dtype == PTY_i8) {
5212         MOperator exOp = is64Bits ? MOP_xsxtb64 : MOP_xsxtb32;
5213         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(exOp, *resOpnd, *resOpnd));
5214     }
5215     return resOpnd;
5216 }
5217 
SelectRor(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5218 Operand *AArch64CGFunc::SelectRor(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5219 {
5220     PrimType dtype = node.GetPrimType();
5221     uint32 dsize = GetPrimTypeBitSize(dtype);
5222     PrimType primType = (dsize == k64BitSize) ? PTY_u64 : PTY_u32;
5223     RegOperand *resOpnd = &GetOrCreateResOperand(parent, primType);
5224     Operand *firstOpnd = &LoadIntoRegister(opnd0, primType);
5225     MOperator mopRor = (dsize == k64BitSize) ? MOP_xrorrrr : MOP_wrorrrr;
5226     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopRor, *resOpnd, *firstOpnd, opnd1));
5227     return resOpnd;
5228 }
5229 
SelectBxorShift(Operand & resOpnd,Operand * opnd0,Operand * opnd1,Operand & opnd2,PrimType primType)5230 void AArch64CGFunc::SelectBxorShift(Operand &resOpnd, Operand *opnd0, Operand *opnd1, Operand &opnd2, PrimType primType)
5231 {
5232     opnd0 = &LoadIntoRegister(*opnd0, primType);
5233     opnd1 = &LoadIntoRegister(*opnd1, primType);
5234     uint32 dsize = GetPrimTypeBitSize(primType);
5235     bool is64Bits = (dsize == k64BitSize);
5236     MOperator mopBxor = is64Bits ? MOP_xeorrrrs : MOP_weorrrrs;
5237     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBxor, resOpnd, *opnd0, *opnd1, opnd2));
5238 }
5239 
SelectShift(Operand & resOpnd,Operand & opnd0,Operand & opnd1,ShiftDirection direct,PrimType primType)5240 void AArch64CGFunc::SelectShift(Operand &resOpnd, Operand &opnd0, Operand &opnd1, ShiftDirection direct,
5241                                 PrimType primType)
5242 {
5243     Operand::OperandType opnd1Type = opnd1.GetKind();
5244     uint32 dsize = GetPrimTypeBitSize(primType);
5245     bool is64Bits = (dsize == k64BitSize);
5246     Operand *firstOpnd = &LoadIntoRegister(opnd0, primType);
5247 
5248     MOperator mopShift;
5249     if ((opnd1Type == Operand::kOpdImmediate) || (opnd1Type == Operand::kOpdOffset)) {
5250         ImmOperand *immOpnd1 = static_cast<ImmOperand *>(&opnd1);
5251         const int64 kVal = immOpnd1->GetValue();
5252         const uint32 kShiftamt = is64Bits ? kHighestBitOf64Bits : kHighestBitOf32Bits;
5253         if (kVal == 0) {
5254             SelectCopy(resOpnd, primType, *firstOpnd, primType);
5255             return;
5256         }
5257         /* e.g. a >> -1 */
5258         if ((kVal < 0) || (kVal > kShiftamt)) {
5259             SelectShift(resOpnd, *firstOpnd, SelectCopy(opnd1, primType, primType), direct, primType);
5260             return;
5261         }
5262         switch (direct) {
5263             case kShiftLeft:
5264                 mopShift = is64Bits ? MOP_xlslrri6 : MOP_wlslrri5;
5265                 break;
5266             case kShiftAright:
5267                 mopShift = is64Bits ? MOP_xasrrri6 : MOP_wasrrri5;
5268                 break;
5269             case kShiftLright:
5270                 mopShift = is64Bits ? MOP_xlsrrri6 : MOP_wlsrrri5;
5271                 break;
5272         }
5273     } else if (opnd1Type != Operand::kOpdRegister) {
5274         SelectShift(resOpnd, *firstOpnd, SelectCopy(opnd1, primType, primType), direct, primType);
5275         return;
5276     } else {
5277         switch (direct) {
5278             case kShiftLeft:
5279                 mopShift = is64Bits ? MOP_xlslrrr : MOP_wlslrrr;
5280                 break;
5281             case kShiftAright:
5282                 mopShift = is64Bits ? MOP_xasrrrr : MOP_wasrrrr;
5283                 break;
5284             case kShiftLright:
5285                 mopShift = is64Bits ? MOP_xlsrrrr : MOP_wlsrrrr;
5286                 break;
5287         }
5288     }
5289 
5290     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopShift, resOpnd, *firstOpnd, opnd1));
5291 }
5292 
SelectAbsSub(Insn & lastInsn,const UnaryNode & node,Operand & newOpnd0)5293 Operand *AArch64CGFunc::SelectAbsSub(Insn &lastInsn, const UnaryNode &node, Operand &newOpnd0)
5294 {
5295     PrimType dtyp = node.GetPrimType();
5296     bool is64Bits = (GetPrimTypeBitSize(dtyp) == k64BitSize);
5297     /* promoted type */
5298     PrimType primType = is64Bits ? (PTY_i64) : (PTY_i32);
5299     RegOperand &resOpnd = CreateRegisterOperandOfType(primType);
5300     uint32 mopCsneg = is64Bits ? MOP_xcnegrrrc : MOP_wcnegrrrc;
5301     /* ABS requires the operand be interpreted as a signed integer */
5302     CondOperand &condOpnd = GetCondOperand(CC_MI);
5303     MOperator newMop = lastInsn.GetMachineOpcode() + 1;
5304     Operand &rflag = GetOrCreateRflag();
5305     std::vector<Operand *> opndVec;
5306     opndVec.push_back(&rflag);
5307     for (uint32 i = 0; i < lastInsn.GetOperandSize(); i++) {
5308         opndVec.push_back(&lastInsn.GetOperand(i));
5309     }
5310     Insn *subsInsn = &GetInsnBuilder()->BuildInsn(newMop, opndVec);
5311     GetCurBB()->ReplaceInsn(lastInsn, *subsInsn);
5312     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopCsneg, resOpnd, newOpnd0, condOpnd, rflag));
5313     return &resOpnd;
5314 }
5315 
SelectAbs(UnaryNode & node,Operand & opnd0)5316 Operand *AArch64CGFunc::SelectAbs(UnaryNode &node, Operand &opnd0)
5317 {
5318     PrimType dtyp = node.GetPrimType();
5319     if (IsPrimitiveVector(dtyp)) {
5320         return SelectVectorAbs(dtyp, &opnd0);
5321     } else if (IsPrimitiveFloat(dtyp)) {
5322         CHECK_FATAL(GetPrimTypeBitSize(dtyp) >= k32BitSize, "We don't support hanf-word FP operands yet");
5323         bool is64Bits = (GetPrimTypeBitSize(dtyp) == k64BitSize);
5324         Operand &newOpnd0 = LoadIntoRegister(opnd0, dtyp);
5325         RegOperand &resOpnd = CreateRegisterOperandOfType(dtyp);
5326         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(is64Bits ? MOP_dabsrr : MOP_sabsrr, resOpnd, newOpnd0));
5327         return &resOpnd;
5328     } else {
5329         bool is64Bits = (GetPrimTypeBitSize(dtyp) == k64BitSize);
5330         /* promoted type */
5331         PrimType primType = is64Bits ? (PTY_i64) : (PTY_i32);
5332         Operand &newOpnd0 = LoadIntoRegister(opnd0, primType);
5333         Insn *lastInsn = GetCurBB()->GetLastInsn();
5334         if (lastInsn != nullptr && lastInsn->GetMachineOpcode() >= MOP_xsubrrr &&
5335             lastInsn->GetMachineOpcode() <= MOP_wsubrri12) {
5336             return SelectAbsSub(*lastInsn, node, newOpnd0);
5337         }
5338         RegOperand &resOpnd = CreateRegisterOperandOfType(primType);
5339         SelectAArch64Cmp(newOpnd0, CreateImmOperand(0, is64Bits ? PTY_u64 : PTY_u32, false), true,
5340                          GetPrimTypeBitSize(dtyp));
5341         uint32 mopCsneg = is64Bits ? MOP_xcsnegrrrc : MOP_wcsnegrrrc;
5342         /* ABS requires the operand be interpreted as a signed integer */
5343         CondOperand &condOpnd = GetCondOperand(CC_GE);
5344         Operand &rflag = GetOrCreateRflag();
5345         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopCsneg, resOpnd, newOpnd0, newOpnd0, condOpnd, rflag));
5346         return &resOpnd;
5347     }
5348 }
5349 
SelectBnot(UnaryNode & node,Operand & opnd0,const BaseNode & parent)5350 Operand *AArch64CGFunc::SelectBnot(UnaryNode &node, Operand &opnd0, const BaseNode &parent)
5351 {
5352     PrimType dtype = node.GetPrimType();
5353     DEBUG_ASSERT(IsPrimitiveInteger(dtype) || IsPrimitiveVectorInteger(dtype), "bnot expect integer or NYI");
5354     uint32 bitSize = GetPrimTypeBitSize(dtype);
5355     bool is64Bits = (bitSize == k64BitSize);
5356     bool isSigned = IsSignedInteger(dtype);
5357     RegOperand *resOpnd = nullptr;
5358     if (!IsPrimitiveVector(dtype)) {
5359         /* promoted type */
5360         PrimType primType = is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32);
5361         resOpnd = &GetOrCreateResOperand(parent, primType);
5362 
5363         Operand &newOpnd0 = LoadIntoRegister(opnd0, primType);
5364 
5365         uint32 mopBnot = is64Bits ? MOP_xnotrr : MOP_wnotrr;
5366         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBnot, *resOpnd, newOpnd0));
5367         /* generate and resOpnd, resOpnd, 0x1/0xFF/0xFFFF for PTY_u1/PTY_u8/PTY_u16 */
5368         int64 immValue = 0;
5369         if (bitSize == k1BitSize) {
5370             immValue = 1;
5371         } else if (bitSize == k8BitSize) {
5372             immValue = 0xFF;
5373         } else if (bitSize == k16BitSize) {
5374             immValue = 0xFFFF;
5375         }
5376         if (immValue != 0) {
5377             ImmOperand &imm = CreateImmOperand(PTY_u32, immValue);
5378             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wandrri12, *resOpnd, *resOpnd, imm));
5379         }
5380     } else {
5381         /* vector operand */
5382         resOpnd = SelectVectorNot(dtype, &opnd0);
5383     }
5384     return resOpnd;
5385 }
5386 
SelectBswap(IntrinsicopNode & node,Operand & opnd0,const BaseNode & parent)5387 Operand *AArch64CGFunc::SelectBswap(IntrinsicopNode &node, Operand &opnd0, const BaseNode &parent)
5388 {
5389     PrimType dtype = node.GetPrimType();
5390     auto bitWidth = (GetPrimTypeBitSize(dtype));
5391     RegOperand *resOpnd = nullptr;
5392     resOpnd = &GetOrCreateResOperand(parent, dtype);
5393     Operand &newOpnd0 = LoadIntoRegister(opnd0, dtype);
5394     uint32 mopBswap = bitWidth == 64 ? MOP_xrevrr : (bitWidth == 32 ? MOP_wrevrr : MOP_wrevrr16);
5395     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBswap, *resOpnd, newOpnd0));
5396     return resOpnd;
5397 }
5398 
SelectRegularBitFieldLoad(ExtractbitsNode & node,const BaseNode & parent)5399 Operand *AArch64CGFunc::SelectRegularBitFieldLoad(ExtractbitsNode &node, const BaseNode &parent)
5400 {
5401     PrimType dtype = node.GetPrimType();
5402     bool isSigned = IsSignedInteger(dtype);
5403     uint8 bitOffset = node.GetBitsOffset();
5404     uint8 bitSize = node.GetBitsSize();
5405     bool is64Bits = (GetPrimTypeBitSize(dtype) == k64BitSize);
5406     CHECK_FATAL(!is64Bits, "dest opnd should not be 64bit");
5407     PrimType destType = GetIntegerPrimTypeBySizeAndSign(bitSize, isSigned);
5408     Operand *result =
5409         SelectIread(parent, *static_cast<IreadNode *>(node.Opnd(0)), static_cast<int>(bitOffset / k8BitSize), destType);
5410     return result;
5411 }
5412 
SelectExtractbits(ExtractbitsNode & node,Operand & srcOpnd,const BaseNode & parent)5413 Operand *AArch64CGFunc::SelectExtractbits(ExtractbitsNode &node, Operand &srcOpnd, const BaseNode &parent)
5414 {
5415     uint8 bitOffset = node.GetBitsOffset();
5416     uint8 bitSize = node.GetBitsSize();
5417     RegOperand *srcVecRegOperand = static_cast<RegOperand *>(&srcOpnd);
5418     if (srcVecRegOperand && srcVecRegOperand->IsRegister() && (srcVecRegOperand->GetSize() == k128BitSize)) {
5419         if ((bitSize == k8BitSize || bitSize == k16BitSize || bitSize == k32BitSize || bitSize == k64BitSize) &&
5420             (bitOffset % bitSize) == k0BitSize) {
5421             uint32 lane = bitOffset / bitSize;
5422             PrimType srcVecPtype;
5423             if (bitSize == k64BitSize) {
5424                 srcVecPtype = PTY_v2u64;
5425             } else if (bitSize == k32BitSize) {
5426                 srcVecPtype = PTY_v4u32;
5427             } else if (bitSize == k16BitSize) {
5428                 srcVecPtype = PTY_v8u16;
5429             } else {
5430                 srcVecPtype = PTY_v16u8;
5431             }
5432             RegOperand *resRegOperand =
5433                 SelectVectorGetElement(node.GetPrimType(), &srcOpnd, srcVecPtype, static_cast<int32>(lane));
5434             return resRegOperand;
5435         } else {
5436             CHECK_FATAL(false, "NYI");
5437         }
5438     }
5439     PrimType dtype = node.GetPrimType();
5440     RegOperand &resOpnd = GetOrCreateResOperand(parent, dtype);
5441     bool isSigned =
5442         (node.GetOpCode() == OP_sext) ? true : (node.GetOpCode() == OP_zext) ? false : IsSignedInteger(dtype);
5443     bool is64Bits = (GetPrimTypeBitSize(dtype) == k64BitSize);
5444     uint32 immWidth = is64Bits ? kMaxImmVal13Bits : kMaxImmVal12Bits;
5445     Operand &opnd0 = LoadIntoRegister(srcOpnd, dtype);
5446     if (bitOffset == 0) {
5447         if (!isSigned && (bitSize < immWidth)) {
5448             SelectBand(resOpnd, opnd0,
5449                        CreateImmOperand(static_cast<int64>((static_cast<uint64>(1) << bitSize) - 1), immWidth, false),
5450                        dtype);
5451             return &resOpnd;
5452         } else {
5453             MOperator mOp = MOP_undef;
5454             if (bitSize == k8BitSize) {
5455                 mOp = is64Bits ? (isSigned ? MOP_xsxtb64 : MOP_undef)
5456                                : (isSigned ? MOP_xsxtb32 : (opnd0.GetSize() == k32BitSize ? MOP_xuxtb32 : MOP_undef));
5457             } else if (bitSize == k16BitSize) {
5458                 mOp = is64Bits ? (isSigned ? MOP_xsxth64 : MOP_undef)
5459                                : (isSigned ? MOP_xsxth32 : (opnd0.GetSize() == k32BitSize ? MOP_xuxth32 : MOP_undef));
5460             } else if (bitSize == k32BitSize) {
5461                 mOp = is64Bits ? (isSigned ? MOP_xsxtw64 : MOP_xuxtw64) : MOP_wmovrr;
5462             }
5463             if (mOp != MOP_undef) {
5464                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0));
5465                 return &resOpnd;
5466             }
5467         }
5468     }
5469     uint32 mopBfx =
5470         is64Bits ? (isSigned ? MOP_xsbfxrri6i6 : MOP_xubfxrri6i6) : (isSigned ? MOP_wsbfxrri5i5 : MOP_wubfxrri5i5);
5471     ImmOperand &immOpnd1 = CreateImmOperand(bitOffset, k8BitSize, false);
5472     ImmOperand &immOpnd2 = CreateImmOperand(bitSize, k8BitSize, false);
5473     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBfx, resOpnd, opnd0, immOpnd1, immOpnd2));
5474     return &resOpnd;
5475 }
5476 
5477 /*
5478  *  operand fits in MOVK if
5479  *     is64Bits && boffst == 0, 16, 32, 48 && bSize == 16, so boffset / 16 == 0, 1, 2, 3; (boffset / 16 ) & (~3) == 0
5480  *  or is32Bits && boffset == 0, 16 && bSize == 16, so boffset / 16 == 0, 1; (boffset / 16) & (~1) == 0
5481  *  imm range of aarch64-movk [0 - 65536] imm16
5482  */
IsMoveWideKeepable(int64 offsetVal,uint32 bitOffset,uint32 bitSize,bool is64Bits)5483 inline bool IsMoveWideKeepable(int64 offsetVal, uint32 bitOffset, uint32 bitSize, bool is64Bits)
5484 {
5485     DEBUG_ASSERT(is64Bits || (bitOffset < k32BitSize), "");
5486     bool isOutOfRange = offsetVal < 0;
5487     if (!isOutOfRange) {
5488         isOutOfRange = (static_cast<unsigned long int>(offsetVal) >> k16BitSize) > 0;
5489     }
5490     return (!isOutOfRange) && bitSize == k16BitSize &&
5491            ((bitOffset >> k16BitShift) & ~static_cast<uint32>(is64Bits ? 0x3 : 0x1)) == 0;
5492 }
5493 
5494 /* we use the fact that A ^ B ^ A == B, A ^ 0 = A */
SelectDepositBits(DepositbitsNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5495 Operand *AArch64CGFunc::SelectDepositBits(DepositbitsNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5496 {
5497     uint32 bitOffset = node.GetBitsOffset();
5498     uint32 bitSize = node.GetBitsSize();
5499     PrimType regType = node.GetPrimType();
5500     bool is64Bits = GetPrimTypeBitSize(regType) == k64BitSize;
5501     /*
5502      * if operand 1 is immediate and fits in MOVK, use it
5503      * MOVK Wd, #imm{, LSL #shift} ; 32-bit general registers
5504      * MOVK Xd, #imm{, LSL #shift} ; 64-bit general registers
5505      */
5506     if (opnd1.IsIntImmediate() &&
5507         IsMoveWideKeepable(static_cast<ImmOperand &>(opnd1).GetValue(), bitOffset, bitSize, is64Bits)) {
5508         RegOperand &resOpnd = GetOrCreateResOperand(parent, regType);
5509         SelectCopy(resOpnd, regType, opnd0, regType);
5510         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn((is64Bits ? MOP_xmovkri16 : MOP_wmovkri16), resOpnd, opnd1,
5511                                                            *GetLogicalShiftLeftOperand(bitOffset, is64Bits)));
5512         return &resOpnd;
5513     } else {
5514         Operand &movOpnd = LoadIntoRegister(opnd1, regType);
5515         uint32 mopBfi = is64Bits ? MOP_xbfirri6i6 : MOP_wbfirri5i5;
5516         ImmOperand &immOpnd1 = CreateImmOperand(bitOffset, k8BitSize, false);
5517         ImmOperand &immOpnd2 = CreateImmOperand(bitSize, k8BitSize, false);
5518         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBfi, opnd0, movOpnd, immOpnd1, immOpnd2));
5519         return &opnd0;
5520     }
5521 }
5522 
SelectLnot(UnaryNode & node,Operand & srcOpnd,const BaseNode & parent)5523 Operand *AArch64CGFunc::SelectLnot(UnaryNode &node, Operand &srcOpnd, const BaseNode &parent)
5524 {
5525     PrimType dtype = node.GetPrimType();
5526     RegOperand &resOpnd = GetOrCreateResOperand(parent, dtype);
5527     bool is64Bits = (GetPrimTypeBitSize(dtype) == k64BitSize);
5528     Operand &opnd0 = LoadIntoRegister(srcOpnd, dtype);
5529     SelectAArch64Cmp(opnd0, CreateImmOperand(0, is64Bits ? PTY_u64 : PTY_u32, false), true, GetPrimTypeBitSize(dtype));
5530     SelectAArch64CSet(resOpnd, GetCondOperand(CC_EQ), is64Bits);
5531     return &resOpnd;
5532 }
5533 
SelectNeg(UnaryNode & node,Operand & opnd0,const BaseNode & parent)5534 Operand *AArch64CGFunc::SelectNeg(UnaryNode &node, Operand &opnd0, const BaseNode &parent)
5535 {
5536     PrimType dtype = node.GetPrimType();
5537     bool is64Bits = (GetPrimTypeBitSize(dtype) == k64BitSize);
5538     RegOperand *resOpnd = nullptr;
5539     if (!IsPrimitiveVector(dtype)) {
5540         PrimType primType;
5541         if (IsPrimitiveFloat(dtype)) {
5542             primType = dtype;
5543         } else {
5544             primType = is64Bits ? (PTY_i64) : (PTY_i32); /* promoted type */
5545         }
5546         resOpnd = &GetOrCreateResOperand(parent, primType);
5547         SelectNeg(*resOpnd, opnd0, primType);
5548     } else {
5549         /* vector operand */
5550         resOpnd = SelectVectorNeg(dtype, &opnd0);
5551     }
5552     return resOpnd;
5553 }
5554 
SelectNeg(Operand & dest,Operand & srcOpnd,PrimType primType)5555 void AArch64CGFunc::SelectNeg(Operand &dest, Operand &srcOpnd, PrimType primType)
5556 {
5557     Operand &opnd0 = LoadIntoRegister(srcOpnd, primType);
5558     bool is64Bits = (GetPrimTypeBitSize(primType) == k64BitSize);
5559     MOperator mOp;
5560     if (IsPrimitiveFloat(primType)) {
5561         mOp = is64Bits ? MOP_xfnegrr : MOP_wfnegrr;
5562     } else {
5563         mOp = is64Bits ? MOP_xinegrr : MOP_winegrr;
5564     }
5565     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, dest, opnd0));
5566 }
5567 
SelectMvn(Operand & dest,Operand & src,PrimType primType)5568 void AArch64CGFunc::SelectMvn(Operand &dest, Operand &src, PrimType primType)
5569 {
5570     Operand &opnd0 = LoadIntoRegister(src, primType);
5571     bool is64Bits = (GetPrimTypeBitSize(primType) == k64BitSize);
5572     MOperator mOp;
5573     DEBUG_ASSERT(!IsPrimitiveFloat(primType), "Instruction 'mvn' do not have float version.");
5574     mOp = is64Bits ? MOP_xnotrr : MOP_wnotrr;
5575     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, dest, opnd0));
5576 }
5577 
SelectRecip(UnaryNode & node,Operand & src,const BaseNode & parent)5578 Operand *AArch64CGFunc::SelectRecip(UnaryNode &node, Operand &src, const BaseNode &parent)
5579 {
5580     /*
5581      * fconsts s15, #112
5582      * fdivs s0, s15, s0
5583      */
5584     PrimType dtype = node.GetPrimType();
5585     if (!IsPrimitiveFloat(dtype)) {
5586         DEBUG_ASSERT(false, "should be float type");
5587         return nullptr;
5588     }
5589     Operand &opnd0 = LoadIntoRegister(src, dtype);
5590     RegOperand &resOpnd = GetOrCreateResOperand(parent, dtype);
5591     Operand *one = nullptr;
5592     if (GetPrimTypeBitSize(dtype) == k64BitSize) {
5593         MIRDoubleConst *c = memPool->New<MIRDoubleConst>(1.0, *GlobalTables::GetTypeTable().GetTypeTable().at(PTY_f64));
5594         one = SelectDoubleConst(*c, node);
5595     } else if (GetPrimTypeBitSize(dtype) == k32BitSize) {
5596         MIRFloatConst *c = memPool->New<MIRFloatConst>(1.0f, *GlobalTables::GetTypeTable().GetTypeTable().at(PTY_f32));
5597         one = SelectFloatConst(*c, node);
5598     } else {
5599         CHECK_FATAL(false, "we don't support half-precision fp operations yet");
5600     }
5601     SelectDiv(resOpnd, *one, opnd0, dtype);
5602     return &resOpnd;
5603 }
5604 
SelectSqrt(UnaryNode & node,Operand & src,const BaseNode & parent)5605 Operand *AArch64CGFunc::SelectSqrt(UnaryNode &node, Operand &src, const BaseNode &parent)
5606 {
5607     /*
5608      * gcc generates code like below for better accurate
5609      * fsqrts  s15, s0
5610      * fcmps s15, s15
5611      * fmstat
5612      * beq .L4
5613      * push  {r3, lr}
5614      * bl  sqrtf
5615      * pop {r3, pc}
5616      * .L4:
5617      * fcpys s0, s15
5618      * bx  lr
5619      */
5620     PrimType dtype = node.GetPrimType();
5621     if (!IsPrimitiveFloat(dtype)) {
5622         DEBUG_ASSERT(false, "should be float type");
5623         return nullptr;
5624     }
5625     bool is64Bits = (GetPrimTypeBitSize(dtype) == k64BitSize);
5626     Operand &opnd0 = LoadIntoRegister(src, dtype);
5627     RegOperand &resOpnd = GetOrCreateResOperand(parent, dtype);
5628     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(is64Bits ? MOP_vsqrtd : MOP_vsqrts, resOpnd, opnd0));
5629     return &resOpnd;
5630 }
5631 
SelectCvtFloat2Int(Operand & resOpnd,Operand & srcOpnd,PrimType itype,PrimType ftype)5632 void AArch64CGFunc::SelectCvtFloat2Int(Operand &resOpnd, Operand &srcOpnd, PrimType itype, PrimType ftype)
5633 {
5634     bool is64BitsFloat = (ftype == PTY_f64);
5635     MOperator mOp = 0;
5636 
5637     DEBUG_ASSERT(((ftype == PTY_f64) || (ftype == PTY_f32)), "wrong from type");
5638     Operand &opnd0 = LoadIntoRegister(srcOpnd, ftype);
5639     switch (itype) {
5640         case PTY_i32:
5641             mOp = !is64BitsFloat ? MOP_vcvtrf : MOP_vcvtrd;
5642             break;
5643         case PTY_u32:
5644         case PTY_a32:
5645             mOp = !is64BitsFloat ? MOP_vcvturf : MOP_vcvturd;
5646             break;
5647         case PTY_i64:
5648             mOp = !is64BitsFloat ? MOP_xvcvtrf : MOP_xvcvtrd;
5649             break;
5650         case PTY_u64:
5651         case PTY_a64:
5652             mOp = !is64BitsFloat ? MOP_xvcvturf : MOP_xvcvturd;
5653             break;
5654         default:
5655             CHECK_FATAL(false, "unexpected type");
5656     }
5657     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0));
5658 }
5659 
SelectCvtInt2Float(Operand & resOpnd,Operand & origOpnd0,PrimType toType,PrimType fromType)5660 void AArch64CGFunc::SelectCvtInt2Float(Operand &resOpnd, Operand &origOpnd0, PrimType toType, PrimType fromType)
5661 {
5662     DEBUG_ASSERT((toType == PTY_f32) || (toType == PTY_f64), "unexpected type");
5663     bool is64BitsFloat = (toType == PTY_f64);
5664     MOperator mOp = 0;
5665     uint32 fsize = GetPrimTypeBitSize(fromType);
5666 
5667     PrimType itype = (GetPrimTypeBitSize(fromType) == k64BitSize) ? (IsSignedInteger(fromType) ? PTY_i64 : PTY_u64)
5668                                                                   : (IsSignedInteger(fromType) ? PTY_i32 : PTY_u32);
5669 
5670     Operand *opnd0 = &LoadIntoRegister(origOpnd0, itype);
5671 
5672     /* need extension before cvt */
5673     DEBUG_ASSERT(opnd0->IsRegister(), "opnd should be a register operand");
5674     Operand *srcOpnd = opnd0;
5675     if (IsSignedInteger(fromType) && (fsize < k32BitSize)) {
5676         srcOpnd = &CreateRegisterOperandOfType(itype);
5677         mOp = (fsize == k8BitSize) ? MOP_xsxtb32 : MOP_xsxth32;
5678         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, *srcOpnd, *opnd0));
5679     }
5680 
5681     switch (itype) {
5682         case PTY_i32:
5683             mOp = !is64BitsFloat ? MOP_vcvtfr : MOP_vcvtdr;
5684             break;
5685         case PTY_u32:
5686             mOp = !is64BitsFloat ? MOP_vcvtufr : MOP_vcvtudr;
5687             break;
5688         case PTY_i64:
5689             mOp = !is64BitsFloat ? MOP_xvcvtfr : MOP_xvcvtdr;
5690             break;
5691         case PTY_u64:
5692             mOp = !is64BitsFloat ? MOP_xvcvtufr : MOP_xvcvtudr;
5693             break;
5694         default:
5695             CHECK_FATAL(false, "unexpected type");
5696     }
5697     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, *srcOpnd));
5698 }
5699 
SelectIntrinsicOpWithOneParam(IntrinsicopNode & intrnNode,std::string name)5700 Operand *AArch64CGFunc::SelectIntrinsicOpWithOneParam(IntrinsicopNode &intrnNode, std::string name)
5701 {
5702     BaseNode *argexpr = intrnNode.Opnd(0);
5703     PrimType ptype = argexpr->GetPrimType();
5704     Operand *opnd = HandleExpr(intrnNode, *argexpr);
5705     if (intrnNode.GetIntrinsic() == INTRN_C_ffs) {
5706         DEBUG_ASSERT(intrnNode.GetPrimType() == PTY_i32, "Unexpect Size");
5707         return SelectAArch64ffs(*opnd, ptype);
5708     }
5709     if (opnd->IsMemoryAccessOperand()) {
5710         RegOperand &ldDest = CreateRegisterOperandOfType(ptype);
5711         Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype), ptype), ldDest, *opnd);
5712         GetCurBB()->AppendInsn(insn);
5713         opnd = &ldDest;
5714     }
5715     std::vector<Operand *> opndVec;
5716     RegOperand *dst = &CreateRegisterOperandOfType(ptype);
5717     opndVec.push_back(dst);  /* result */
5718     opndVec.push_back(opnd); /* param 0 */
5719     SelectLibCall(name, opndVec, ptype, ptype);
5720 
5721     return dst;
5722 }
5723 
SelectIntrinsicOpWithNParams(IntrinsicopNode & intrnNode,PrimType retType,const std::string & name)5724 Operand *AArch64CGFunc::SelectIntrinsicOpWithNParams(IntrinsicopNode &intrnNode, PrimType retType,
5725                                                      const std::string &name)
5726 {
5727     MapleVector<BaseNode *> argNodes = intrnNode.GetNopnd();
5728     std::vector<Operand *> opndVec;
5729     std::vector<PrimType> opndTypes;
5730     RegOperand *retOpnd = &CreateRegisterOperandOfType(retType);
5731     opndVec.push_back(retOpnd);
5732     opndTypes.push_back(retType);
5733 
5734     for (BaseNode *argexpr : argNodes) {
5735         PrimType ptype = argexpr->GetPrimType();
5736         Operand *opnd = HandleExpr(intrnNode, *argexpr);
5737         if (opnd->IsMemoryAccessOperand()) {
5738             RegOperand &ldDest = CreateRegisterOperandOfType(ptype);
5739             Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype), ptype), ldDest, *opnd);
5740             GetCurBB()->AppendInsn(insn);
5741             opnd = &ldDest;
5742         }
5743         opndVec.push_back(opnd);
5744         opndTypes.push_back(ptype);
5745     }
5746     SelectLibCallNArg(name, opndVec, opndTypes, retType, false);
5747 
5748     return retOpnd;
5749 }
5750 
5751 /* According to  gcc.target/aarch64/ffs.c */
SelectAArch64ffs(Operand & argOpnd,PrimType argType)5752 Operand *AArch64CGFunc::SelectAArch64ffs(Operand &argOpnd, PrimType argType)
5753 {
5754     RegOperand &destOpnd = LoadIntoRegister(argOpnd, argType);
5755     uint32 argSize = GetPrimTypeBitSize(argType);
5756     DEBUG_ASSERT((argSize == k64BitSize || argSize == k32BitSize), "Unexpect arg type");
5757     /* cmp */
5758     ImmOperand &zeroOpnd = CreateImmOperand(0, argSize, false);
5759     Operand &rflag = GetOrCreateRflag();
5760     GetCurBB()->AppendInsn(
5761         GetInsnBuilder()->BuildInsn(argSize == k64BitSize ? MOP_xcmpri : MOP_wcmpri, rflag, destOpnd, zeroOpnd));
5762     /* rbit */
5763     RegOperand *tempResReg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, GetPrimTypeSize(argType)));
5764     GetCurBB()->AppendInsn(
5765         GetInsnBuilder()->BuildInsn(argSize == k64BitSize ? MOP_xrbit : MOP_wrbit, *tempResReg, destOpnd));
5766     /* clz */
5767     GetCurBB()->AppendInsn(
5768         GetInsnBuilder()->BuildInsn(argSize == k64BitSize ? MOP_xclz : MOP_wclz, *tempResReg, *tempResReg));
5769     /* csincc */
5770     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(argSize == k64BitSize ? MOP_xcsincrrrc : MOP_wcsincrrrc,
5771                                                        *tempResReg, GetZeroOpnd(k32BitSize), *tempResReg,
5772                                                        GetCondOperand(CC_EQ), rflag));
5773     return tempResReg;
5774 }
5775 
SelectRoundLibCall(RoundType roundType,const TypeCvtNode & node,Operand & opnd0)5776 Operand *AArch64CGFunc::SelectRoundLibCall(RoundType roundType, const TypeCvtNode &node, Operand &opnd0)
5777 {
5778     PrimType ftype = node.FromType();
5779     PrimType rtype = node.GetPrimType();
5780     bool is64Bits = (ftype == PTY_f64);
5781     std::vector<Operand *> opndVec;
5782     RegOperand *resOpnd;
5783     if (is64Bits) {
5784         resOpnd = &GetOrCreatePhysicalRegisterOperand(D0, k64BitSize, kRegTyFloat);
5785     } else {
5786         resOpnd = &GetOrCreatePhysicalRegisterOperand(S0, k32BitSize, kRegTyFloat);
5787     }
5788     opndVec.push_back(resOpnd);
5789     RegOperand &regOpnd0 = LoadIntoRegister(opnd0, ftype);
5790     opndVec.push_back(&regOpnd0);
5791     std::string libName;
5792     if (roundType == kCeil) {
5793         libName.assign(is64Bits ? "ceil" : "ceilf");
5794     } else if (roundType == kFloor) {
5795         libName.assign(is64Bits ? "floor" : "floorf");
5796     } else {
5797         libName.assign(is64Bits ? "round" : "roundf");
5798     }
5799     SelectLibCall(libName, opndVec, ftype, rtype);
5800 
5801     return resOpnd;
5802 }
5803 
SelectRoundOperator(RoundType roundType,const TypeCvtNode & node,Operand & opnd0,const BaseNode & parent)5804 Operand *AArch64CGFunc::SelectRoundOperator(RoundType roundType, const TypeCvtNode &node, Operand &opnd0,
5805                                             const BaseNode &parent)
5806 {
5807     PrimType itype = node.GetPrimType();
5808     if ((mirModule.GetSrcLang() == kSrcLangC) && ((itype == PTY_f64) || (itype == PTY_f32))) {
5809         SelectRoundLibCall(roundType, node, opnd0);
5810     }
5811     PrimType ftype = node.FromType();
5812     DEBUG_ASSERT(((ftype == PTY_f64) || (ftype == PTY_f32)), "wrong float type");
5813     bool is64Bits = (ftype == PTY_f64);
5814     RegOperand &resOpnd = GetOrCreateResOperand(parent, itype);
5815     RegOperand &regOpnd0 = LoadIntoRegister(opnd0, ftype);
5816     MOperator mop = MOP_undef;
5817     if (roundType == kCeil) {
5818         mop = is64Bits ? MOP_xvcvtps : MOP_vcvtps;
5819     } else if (roundType == kFloor) {
5820         mop = is64Bits ? MOP_xvcvtms : MOP_vcvtms;
5821     } else {
5822         mop = is64Bits ? MOP_xvcvtas : MOP_vcvtas;
5823     }
5824     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mop, resOpnd, regOpnd0));
5825     return &resOpnd;
5826 }
5827 
SelectCeil(TypeCvtNode & node,Operand & opnd0,const BaseNode & parent)5828 Operand *AArch64CGFunc::SelectCeil(TypeCvtNode &node, Operand &opnd0, const BaseNode &parent)
5829 {
5830     return SelectRoundOperator(kCeil, node, opnd0, parent);
5831 }
5832 
5833 /* float to int floor */
SelectFloor(TypeCvtNode & node,Operand & opnd0,const BaseNode & parent)5834 Operand *AArch64CGFunc::SelectFloor(TypeCvtNode &node, Operand &opnd0, const BaseNode &parent)
5835 {
5836     return SelectRoundOperator(kFloor, node, opnd0, parent);
5837 }
5838 
SelectRound(TypeCvtNode & node,Operand & opnd0,const BaseNode & parent)5839 Operand *AArch64CGFunc::SelectRound(TypeCvtNode &node, Operand &opnd0, const BaseNode &parent)
5840 {
5841     return SelectRoundOperator(kRound, node, opnd0, parent);
5842 }
5843 
LIsPrimitivePointer(PrimType ptype)5844 static bool LIsPrimitivePointer(PrimType ptype)
5845 {
5846     return ((ptype >= PTY_ptr) && (ptype <= PTY_a64));
5847 }
5848 
SelectRetype(TypeCvtNode & node,Operand & opnd0)5849 Operand *AArch64CGFunc::SelectRetype(TypeCvtNode &node, Operand &opnd0)
5850 {
5851     PrimType fromType = node.Opnd(0)->GetPrimType();
5852     PrimType toType = node.GetPrimType();
5853     DEBUG_ASSERT(GetPrimTypeSize(fromType) == GetPrimTypeSize(toType), "retype bit widith doesn' match");
5854     if (LIsPrimitivePointer(fromType) && LIsPrimitivePointer(toType)) {
5855         return &LoadIntoRegister(opnd0, toType);
5856     }
5857     if (IsPrimitiveVector(fromType) || IsPrimitiveVector(toType)) {
5858         return &LoadIntoRegister(opnd0, toType);
5859     }
5860     Operand::OperandType opnd0Type = opnd0.GetKind();
5861     RegOperand *resOpnd = &CreateRegisterOperandOfType(toType);
5862     if (IsPrimitiveInteger(fromType) || IsPrimitiveFloat(fromType)) {
5863         bool isFromInt = IsPrimitiveInteger(fromType);
5864         bool is64Bits = GetPrimTypeBitSize(fromType) == k64BitSize;
5865         PrimType itype =
5866             isFromInt ? ((GetPrimTypeBitSize(fromType) == k64BitSize) ? (IsSignedInteger(fromType) ? PTY_i64 : PTY_u64)
5867                                                                       : (IsSignedInteger(fromType) ? PTY_i32 : PTY_u32))
5868                       : (is64Bits ? PTY_f64 : PTY_f32);
5869 
5870         /*
5871          * if source operand is in memory,
5872          * simply read it as a value of 'toType 'into the dest operand
5873          * and return
5874          */
5875         if (opnd0Type == Operand::kOpdMem) {
5876             resOpnd = &SelectCopy(opnd0, toType, toType);
5877             return resOpnd;
5878         }
5879         /* according to aarch64 encoding format, convert int to float expression */
5880         bool isImm = false;
5881         ImmOperand *imm = static_cast<ImmOperand *>(&opnd0);
5882         uint64 val = static_cast<uint64>(imm->GetValue());
5883         uint64 canRepreset = is64Bits ? (val & 0xffffffffffff) : (val & 0x7ffff);
5884         uint32 val1 = is64Bits ? (val >> 61) & 0x3 : (val >> 29) & 0x3;
5885         uint32 val2 = is64Bits ? (val >> 54) & 0xff : (val >> 25) & 0x1f;
5886         bool isSame = is64Bits ? ((val2 == 0) || (val2 == 0xff)) : ((val2 == 0) || (val2 == 0x1f));
5887         canRepreset = (canRepreset == 0) && ((val1 & 0x1) ^ ((val1 & 0x2) >> 1)) && isSame;
5888         Operand *newOpnd0 = &opnd0;
5889         if (IsPrimitiveInteger(fromType) && IsPrimitiveFloat(toType) && canRepreset) {
5890             uint64 temp1 = is64Bits ? (val >> 63) << 7 : (val >> 31) << 7;
5891             uint64 temp2 = is64Bits ? val >> 48 : val >> 19;
5892             int64 imm8 = (temp2 & 0x7f) | temp1;
5893             newOpnd0 = &CreateImmOperand(imm8, k8BitSize, false, kNotVary, true);
5894             isImm = true;
5895         } else {
5896             newOpnd0 = &LoadIntoRegister(opnd0, itype);
5897         }
5898         if ((IsPrimitiveFloat(fromType) && IsPrimitiveInteger(toType)) ||
5899             (IsPrimitiveFloat(toType) && IsPrimitiveInteger(fromType))) {
5900             MOperator mopFmov = (isImm ? (is64Bits ? MOP_xdfmovri : MOP_wsfmovri) : isFromInt)
5901                                     ? (is64Bits ? MOP_xvmovdr : MOP_xvmovsr)
5902                                     : (is64Bits ? MOP_xvmovrd : MOP_xvmovrs);
5903             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopFmov, *resOpnd, *newOpnd0));
5904             return resOpnd;
5905         } else {
5906             return newOpnd0;
5907         }
5908     } else {
5909         CHECK_FATAL(false, "NYI retype");
5910     }
5911     return nullptr;
5912 }
5913 
SelectCvtFloat2Float(Operand & resOpnd,Operand & srcOpnd,PrimType fromType,PrimType toType)5914 void AArch64CGFunc::SelectCvtFloat2Float(Operand &resOpnd, Operand &srcOpnd, PrimType fromType, PrimType toType)
5915 {
5916     Operand &opnd0 = LoadIntoRegister(srcOpnd, fromType);
5917     MOperator mOp = 0;
5918     switch (toType) {
5919         case PTY_f32: {
5920             CHECK_FATAL(fromType == PTY_f64, "unexpected cvt from type");
5921             mOp = MOP_xvcvtfd;
5922             break;
5923         }
5924         case PTY_f64: {
5925             CHECK_FATAL(fromType == PTY_f32, "unexpected cvt from type");
5926             mOp = MOP_xvcvtdf;
5927             break;
5928         }
5929         default:
5930             CHECK_FATAL(false, "unexpected cvt to type");
5931     }
5932 
5933     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0));
5934 }
5935 
5936 /*
5937  * This should be regarded only as a reference.
5938  *
5939  * C11 specification.
5940  * 6.3.1.3 Signed and unsigned integers
5941  * 1 When a value with integer type is converted to another integer
5942  *  type other than _Bool, if the value can be represented by the
5943  *  new type, it is unchanged.
5944  * 2 Otherwise, if the new type is unsigned, the value is converted
5945  *  by repeatedly adding or subtracting one more than the maximum
5946  *  value that can be represented in the new type until the value
5947  *  is in the range of the new type.60)
5948  * 3 Otherwise, the new type is signed and the value cannot be
5949  *  represented in it; either the result is implementation-defined
5950  *  or an implementation-defined signal is raised.
5951  */
SelectCvtInt2Int(const BaseNode * parent,Operand * & resOpnd,Operand * opnd0,PrimType fromType,PrimType toType)5952 void AArch64CGFunc::SelectCvtInt2Int(const BaseNode *parent, Operand *&resOpnd, Operand *opnd0, PrimType fromType,
5953                                      PrimType toType)
5954 {
5955     uint32 fsize = GetPrimTypeBitSize(fromType);
5956     if (fromType == PTY_i128 || fromType == PTY_u128) {
5957         fsize = k64BitSize;
5958     }
5959     uint32 tsize = GetPrimTypeBitSize(toType);
5960     if (toType == PTY_i128 || toType == PTY_u128) {
5961         tsize = k64BitSize;
5962     }
5963     bool isExpand = tsize > fsize;
5964     bool is64Bit = (tsize == k64BitSize);
5965     if ((parent != nullptr) && opnd0->IsIntImmediate() &&
5966         ((parent->GetOpCode() == OP_band) || (parent->GetOpCode() == OP_bior) || (parent->GetOpCode() == OP_bxor) ||
5967          (parent->GetOpCode() == OP_ashr) || (parent->GetOpCode() == OP_lshr) || (parent->GetOpCode() == OP_shl))) {
5968         ImmOperand *simm = static_cast<ImmOperand *>(opnd0);
5969         DEBUG_ASSERT(simm != nullptr, "simm is nullptr in AArch64CGFunc::SelectCvtInt2Int");
5970         bool isSign = false;
5971         int64 origValue = simm->GetValue();
5972         int64 newValue = origValue;
5973         int64 signValue = 0;
5974         if (!isExpand) {
5975             /* 64--->32 */
5976             if (fsize > tsize) {
5977                 if (IsSignedInteger(toType)) {
5978                     if (origValue < 0) {
5979                         signValue = static_cast<int64>(0xFFFFFFFFFFFFFFFFLL & (1ULL << static_cast<uint32>(tsize)));
5980                     }
5981                     newValue = static_cast<int64>(
5982                         (static_cast<uint64>(origValue) & ((1ULL << static_cast<uint32>(tsize)) - 1u)) |
5983                         static_cast<uint64>(signValue));
5984                 } else {
5985                     newValue = static_cast<uint64>(origValue) & ((1ULL << static_cast<uint32>(tsize)) - 1u);
5986                 }
5987             }
5988         }
5989         if (IsSignedInteger(toType)) {
5990             isSign = true;
5991         }
5992         resOpnd = &static_cast<Operand &>(CreateImmOperand(newValue, GetPrimTypeSize(toType) * kBitsPerByte, isSign));
5993         return;
5994     }
5995     if (isExpand) { /* Expansion */
5996         /* if cvt expr's parent is add,and,xor and some other,we can use the imm version */
5997         PrimType primType = ((fsize == k64BitSize) ? (IsSignedInteger(fromType) ? PTY_i64 : PTY_u64)
5998                                                    : (IsSignedInteger(fromType) ? PTY_i32 : PTY_u32));
5999         opnd0 = &LoadIntoRegister(*opnd0, primType);
6000 
6001         if (IsSignedInteger(fromType)) {
6002             DEBUG_ASSERT((is64Bit || (fsize == k8BitSize || fsize == k16BitSize)), "incorrect from size");
6003 
6004             MOperator mOp =
6005                 (is64Bit ? ((fsize == k8BitSize) ? MOP_xsxtb64 : ((fsize == k16BitSize) ? MOP_xsxth64 : MOP_xsxtw64))
6006                          : ((fsize == k8BitSize) ? MOP_xsxtb32 : MOP_xsxth32));
6007             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, *resOpnd, *opnd0));
6008         } else {
6009             /* Unsigned */
6010             if (is64Bit) {
6011                 if (fsize == k8BitSize) {
6012                     ImmOperand &immOpnd = CreateImmOperand(0xff, k64BitSize, false);
6013                     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xandrri13, *resOpnd, *opnd0, immOpnd));
6014                 } else if (fsize == k16BitSize) {
6015                     ImmOperand &immOpnd = CreateImmOperand(0xffff, k64BitSize, false);
6016                     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xandrri13, *resOpnd, *opnd0, immOpnd));
6017                 } else {
6018                     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xuxtw64, *resOpnd, *opnd0));
6019                 }
6020             } else {
6021                 DEBUG_ASSERT(((fsize == k8BitSize) || (fsize == k16BitSize)), "incorrect from size");
6022                 if (fsize == k8BitSize) {
6023                     static_cast<RegOperand *>(opnd0)->SetValidBitsNum(k8BitSize);
6024                     static_cast<RegOperand *>(resOpnd)->SetValidBitsNum(k8BitSize);
6025                 }
6026                 if (fromType == PTY_u1) {
6027                     static_cast<RegOperand *>(opnd0)->SetValidBitsNum(1);
6028                     static_cast<RegOperand *>(resOpnd)->SetValidBitsNum(1);
6029                 }
6030                 GetCurBB()->AppendInsn(
6031                     GetInsnBuilder()->BuildInsn((fsize == k8BitSize) ? MOP_xuxtb32 : MOP_xuxth32, *resOpnd, *opnd0));
6032             }
6033         }
6034     } else { /* Same size or truncate */
6035 #ifdef CNV_OPTIMIZE
6036         /*
6037          * No code needed for aarch64 with same reg.
6038          * Just update regno.
6039          */
6040         RegOperand *reg = static_cast<RegOperand *>(resOpnd);
6041         reg->regNo = static_cast<RegOperand *>(opnd0)->regNo;
6042 #else
6043         /*
6044          *  This is not really needed if opnd0 is result from a load.
6045          * Hopefully the FE will get rid of the redundant conversions for loads.
6046          */
6047         PrimType primType = ((fsize == k64BitSize) ? (IsSignedInteger(fromType) ? PTY_i64 : PTY_u64)
6048                                                    : (IsSignedInteger(fromType) ? PTY_i32 : PTY_u32));
6049         opnd0 = &LoadIntoRegister(*opnd0, primType);
6050 
6051         if (fsize > tsize) {
6052             if (tsize == k8BitSize) {
6053                 MOperator mOp = IsSignedInteger(toType) ? MOP_xsxtb32 : MOP_xuxtb32;
6054                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, *resOpnd, *opnd0));
6055             } else if (tsize == k16BitSize) {
6056                 MOperator mOp = IsSignedInteger(toType) ? MOP_xsxth32 : MOP_xuxth32;
6057                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, *resOpnd, *opnd0));
6058             } else {
6059                 MOperator mOp = IsSignedInteger(toType) ? MOP_xsbfxrri6i6 : MOP_xubfxrri6i6;
6060                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, *resOpnd, *opnd0,
6061                                                                    CreateImmOperand(0, k8BitSize, false),
6062                                                                    CreateImmOperand(tsize, k8BitSize, false)));
6063             }
6064         } else {
6065             /* same size, so resOpnd can be set */
6066             if ((mirModule.IsJavaModule()) || (IsSignedInteger(fromType) == IsSignedInteger(toType)) ||
6067                 (GetPrimTypeSize(toType) >= k4BitSize)) {
6068                 resOpnd = opnd0;
6069             } else if (IsUnsignedInteger(toType)) {
6070                 MOperator mop;
6071                 switch (toType) {
6072                     case PTY_u8:
6073                         mop = MOP_xuxtb32;
6074                         break;
6075                     case PTY_u16:
6076                         mop = MOP_xuxth32;
6077                         break;
6078                     default:
6079                         CHECK_FATAL(0, "Unhandled unsigned convert");
6080                 }
6081                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mop, *resOpnd, *opnd0));
6082             } else {
6083                 /* signed target */
6084                 uint32 size = GetPrimTypeSize(toType);
6085                 MOperator mop;
6086                 switch (toType) {
6087                     case PTY_i8:
6088                         mop = (size > k4BitSize) ? MOP_xsxtb64 : MOP_xsxtb32;
6089                         break;
6090                     case PTY_i16:
6091                         mop = (size > k4BitSize) ? MOP_xsxth64 : MOP_xsxth32;
6092                         break;
6093                     default:
6094                         CHECK_FATAL(0, "Unhandled unsigned convert");
6095                 }
6096                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mop, *resOpnd, *opnd0));
6097             }
6098         }
6099 #endif
6100     }
6101 }
6102 
SelectCvt(const BaseNode & parent,TypeCvtNode & node,Operand & opnd0)6103 Operand *AArch64CGFunc::SelectCvt(const BaseNode &parent, TypeCvtNode &node, Operand &opnd0)
6104 {
6105     PrimType fromType = node.FromType();
6106     PrimType toType = node.GetPrimType();
6107     if (fromType == toType) {
6108         return &opnd0; /* noop */
6109     }
6110     Operand *resOpnd = &GetOrCreateResOperand(parent, toType);
6111     if (IsPrimitiveFloat(toType) && IsPrimitiveInteger(fromType)) {
6112         SelectCvtInt2Float(*resOpnd, opnd0, toType, fromType);
6113     } else if (IsPrimitiveFloat(fromType) && IsPrimitiveInteger(toType)) {
6114         SelectCvtFloat2Int(*resOpnd, opnd0, toType, fromType);
6115     } else if (IsPrimitiveInteger(fromType) && IsPrimitiveInteger(toType)) {
6116         SelectCvtInt2Int(&parent, resOpnd, &opnd0, fromType, toType);
6117     } else if (IsPrimitiveVector(toType) || IsPrimitiveVector(fromType)) {
6118         CHECK_FATAL(IsPrimitiveVector(toType) && IsPrimitiveVector(fromType), "Invalid vector cvt operands");
6119         SelectVectorCvt(resOpnd, toType, &opnd0, fromType);
6120     } else { /* both are float type */
6121         SelectCvtFloat2Float(*resOpnd, opnd0, fromType, toType);
6122     }
6123     return resOpnd;
6124 }
6125 
SelectTrunc(TypeCvtNode & node,Operand & opnd0,const BaseNode & parent)6126 Operand *AArch64CGFunc::SelectTrunc(TypeCvtNode &node, Operand &opnd0, const BaseNode &parent)
6127 {
6128     PrimType ftype = node.FromType();
6129     bool is64Bits = (GetPrimTypeBitSize(node.GetPrimType()) == k64BitSize);
6130     PrimType itype = (is64Bits) ? (IsSignedInteger(node.GetPrimType()) ? PTY_i64 : PTY_u64)
6131                                 : (IsSignedInteger(node.GetPrimType()) ? PTY_i32 : PTY_u32); /* promoted type */
6132     RegOperand &resOpnd = GetOrCreateResOperand(parent, itype);
6133     SelectCvtFloat2Int(resOpnd, opnd0, itype, ftype);
6134     return &resOpnd;
6135 }
6136 
SelectSelect(Operand & resOpnd,Operand & condOpnd,Operand & trueOpnd,Operand & falseOpnd,PrimType dtype,PrimType ctype,bool hasCompare,ConditionCode cc)6137 void AArch64CGFunc::SelectSelect(Operand &resOpnd, Operand &condOpnd, Operand &trueOpnd, Operand &falseOpnd,
6138                                  PrimType dtype, PrimType ctype, bool hasCompare, ConditionCode cc)
6139 {
6140     DEBUG_ASSERT(&resOpnd != &condOpnd, "resOpnd cannot be the same as condOpnd");
6141     bool isIntType = IsPrimitiveInteger(dtype);
6142     DEBUG_ASSERT((IsPrimitiveInteger(dtype) || IsPrimitiveFloat(dtype)), "unknown type for select");
6143     // making condOpnd and cmpInsn closer will provide more opportunity for opt
6144     Operand &newTrueOpnd = LoadIntoRegister(trueOpnd, dtype);
6145     Operand &newFalseOpnd = LoadIntoRegister(falseOpnd, dtype);
6146     Operand &newCondOpnd = LoadIntoRegister(condOpnd, ctype);
6147     if (hasCompare) {
6148         SelectAArch64Cmp(newCondOpnd, CreateImmOperand(0, ctype, false), true, GetPrimTypeBitSize(ctype));
6149         cc = CC_NE;
6150     }
6151     Operand &newResOpnd = LoadIntoRegister(resOpnd, dtype);
6152     SelectAArch64Select(newResOpnd, newTrueOpnd, newFalseOpnd, GetCondOperand(cc), isIntType,
6153                         GetPrimTypeBitSize(dtype));
6154 }
6155 
SelectSelect(TernaryNode & expr,Operand & cond,Operand & trueOpnd,Operand & falseOpnd,const BaseNode & parent,bool hasCompare)6156 Operand *AArch64CGFunc::SelectSelect(TernaryNode &expr, Operand &cond, Operand &trueOpnd, Operand &falseOpnd,
6157                                      const BaseNode &parent, bool hasCompare)
6158 {
6159     PrimType dtype = expr.GetPrimType();
6160     PrimType ctype = expr.Opnd(0)->GetPrimType();
6161 
6162     ConditionCode cc = CC_NE;
6163     Opcode opcode = expr.Opnd(0)->GetOpCode();
6164     PrimType cmpType = static_cast<CompareNode *>(expr.Opnd(0))->GetOpndType();
6165     bool isFloat = false;
6166     bool unsignedIntegerComparison = false;
6167     if (!IsPrimitiveVector(cmpType)) {
6168         isFloat = IsPrimitiveFloat(cmpType);
6169         unsignedIntegerComparison = !isFloat && !IsSignedInteger(cmpType);
6170     } else {
6171         isFloat = IsPrimitiveVectorFloat(cmpType);
6172         unsignedIntegerComparison = !isFloat && IsPrimitiveUnSignedVector(cmpType);
6173     }
6174     switch (opcode) {
6175         case OP_eq:
6176             cc = CC_EQ;
6177             break;
6178         case OP_ne:
6179             cc = CC_NE;
6180             break;
6181         case OP_le:
6182             cc = unsignedIntegerComparison ? CC_LS : CC_LE;
6183             break;
6184         case OP_ge:
6185             cc = unsignedIntegerComparison ? CC_HS : CC_GE;
6186             break;
6187         case OP_gt:
6188             cc = unsignedIntegerComparison ? CC_HI : CC_GT;
6189             break;
6190         case OP_lt:
6191             cc = unsignedIntegerComparison ? CC_LO : CC_LT;
6192             break;
6193         default:
6194             hasCompare = true;
6195             break;
6196     }
6197     if (!IsPrimitiveVector(dtype)) {
6198         RegOperand &resOpnd = GetOrCreateResOperand(parent, dtype);
6199         SelectSelect(resOpnd, cond, trueOpnd, falseOpnd, dtype, ctype, hasCompare, cc);
6200         return &resOpnd;
6201     } else {
6202         return SelectVectorSelect(cond, dtype, trueOpnd, falseOpnd);
6203     }
6204 }
6205 
6206 /*
6207  * syntax: select <prim-type> (<opnd0>, <opnd1>, <opnd2>)
6208  * <opnd0> must be of integer type.
6209  * <opnd1> and <opnd2> must be of the type given by <prim-type>.
6210  * If <opnd0> is not 0, return <opnd1>.  Otherwise, return <opnd2>.
6211  */
SelectAArch64Select(Operand & dest,Operand & o0,Operand & o1,CondOperand & cond,bool isIntType,uint32 dsize)6212 void AArch64CGFunc::SelectAArch64Select(Operand &dest, Operand &o0, Operand &o1, CondOperand &cond, bool isIntType,
6213                                         uint32 dsize)
6214 {
6215     uint32 mOpCode =
6216         isIntType ? ((dsize == k64BitSize) ? MOP_xcselrrrc : MOP_wcselrrrc)
6217                   : ((dsize == k64BitSize) ? MOP_dcselrrrc : ((dsize == k32BitSize) ? MOP_scselrrrc : MOP_hcselrrrc));
6218     Operand &rflag = GetOrCreateRflag();
6219     if (o1.IsImmediate()) {
6220         uint32 movOp = (dsize == k64BitSize ? MOP_xmovri64 : MOP_wmovri32);
6221         RegOperand &movDest =
6222             CreateVirtualRegisterOperand(NewVReg(kRegTyInt, (dsize == k64BitSize) ? k8ByteSize : k4ByteSize));
6223         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(movOp, movDest, o1));
6224         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, dest, o0, movDest, cond, rflag));
6225         return;
6226     }
6227     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, dest, o0, o1, cond, rflag));
6228 }
6229 
SelectRangeGoto(RangeGotoNode & rangeGotoNode,Operand & srcOpnd)6230 void AArch64CGFunc::SelectRangeGoto(RangeGotoNode &rangeGotoNode, Operand &srcOpnd)
6231 {
6232     const SmallCaseVector &switchTable = rangeGotoNode.GetRangeGotoTable();
6233     MIRType *etype = GlobalTables::GetTypeTable().GetTypeFromTyIdx(static_cast<TyIdx>(PTY_a64));
6234     /*
6235      * we store 8-byte displacement ( jump_label - offset_table_address )
6236      * in the table. Refer to AArch64Emit::Emit() in aarch64emit.cpp
6237      */
6238     std::vector<uint32> sizeArray;
6239     sizeArray.emplace_back(switchTable.size());
6240     MIRArrayType *arrayType = memPool->New<MIRArrayType>(etype->GetTypeIndex(), sizeArray);
6241     MIRAggConst *arrayConst = memPool->New<MIRAggConst>(mirModule, *arrayType);
6242     for (const auto &itPair : switchTable) {
6243         LabelIdx labelIdx = itPair.second;
6244         GetCurBB()->PushBackRangeGotoLabel(labelIdx);
6245         MIRConst *mirConst = memPool->New<MIRLblConst>(labelIdx, GetFunction().GetPuidx(), *etype);
6246         arrayConst->AddItem(mirConst, 0);
6247     }
6248 
6249     MIRSymbol *lblSt = GetFunction().GetSymTab()->CreateSymbol(kScopeLocal);
6250     lblSt->SetStorageClass(kScFstatic);
6251     lblSt->SetSKind(kStConst);
6252     lblSt->SetTyIdx(arrayType->GetTypeIndex());
6253     lblSt->SetKonst(arrayConst);
6254     std::string lblStr(".LB_");
6255     MIRSymbol *funcSt = GlobalTables::GetGsymTable().GetSymbolFromStidx(GetFunction().GetStIdx().Idx());
6256     uint32 labelIdxTmp = GetLabelIdx();
6257     lblStr += funcSt->GetName();
6258     lblStr += std::to_string(labelIdxTmp++);
6259     SetLabelIdx(labelIdxTmp);
6260     lblSt->SetNameStrIdx(lblStr);
6261     AddEmitSt(GetCurBB()->GetId(), *lblSt);
6262 
6263     PrimType itype = rangeGotoNode.Opnd(0)->GetPrimType();
6264     Operand &opnd0 = LoadIntoRegister(srcOpnd, itype);
6265 
6266     regno_t vRegNO = NewVReg(kRegTyInt, 8u);
6267     RegOperand *addOpnd = &CreateVirtualRegisterOperand(vRegNO);
6268 
6269     int32 minIdx = switchTable[0].first;
6270     SelectAdd(*addOpnd, opnd0,
6271               CreateImmOperand(-static_cast<int64>(minIdx) - static_cast<int64>(rangeGotoNode.GetTagOffset()),
6272                                GetPrimTypeBitSize(itype), true),
6273               itype);
6274 
6275     /* contains the index */
6276     if (addOpnd->GetSize() != GetPrimTypeBitSize(PTY_u64)) {
6277         addOpnd = static_cast<RegOperand *>(&SelectCopy(*addOpnd, PTY_u64, PTY_u64));
6278     }
6279 
6280     RegOperand &baseOpnd = CreateRegisterOperandOfType(PTY_u64);
6281     StImmOperand &stOpnd = CreateStImmOperand(*lblSt, 0, 0);
6282 
6283     /* load the address of the switch table */
6284     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrp, baseOpnd, stOpnd));
6285     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrpl12, baseOpnd, baseOpnd, stOpnd));
6286 
6287     /* load the displacement into a register by accessing memory at base + index*8 */
6288     Operand *disp = CreateMemOperand(MemOperand::kAddrModeBOrX, k64BitSize, baseOpnd, *addOpnd, k8BitShift);
6289     RegOperand &tgt = CreateRegisterOperandOfType(PTY_a64);
6290     SelectAdd(tgt, baseOpnd, *disp, PTY_u64);
6291     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xbr, tgt));
6292 }
6293 
SelectLazyLoad(Operand & opnd0,PrimType primType)6294 Operand *AArch64CGFunc::SelectLazyLoad(Operand &opnd0, PrimType primType)
6295 {
6296     DEBUG_ASSERT(opnd0.IsRegister(), "wrong type.");
6297     RegOperand &resOpnd = CreateRegisterOperandOfType(primType);
6298     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_lazy_ldr, resOpnd, opnd0));
6299     return &resOpnd;
6300 }
6301 
SelectLazyLoadStatic(MIRSymbol & st,int64 offset,PrimType primType)6302 Operand *AArch64CGFunc::SelectLazyLoadStatic(MIRSymbol &st, int64 offset, PrimType primType)
6303 {
6304     StImmOperand &srcOpnd = CreateStImmOperand(st, offset, 0);
6305     RegOperand &resOpnd = CreateRegisterOperandOfType(primType);
6306     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_lazy_ldr_static, resOpnd, srcOpnd));
6307     return &resOpnd;
6308 }
6309 
SelectLoadArrayClassCache(MIRSymbol & st,int64 offset,PrimType primType)6310 Operand *AArch64CGFunc::SelectLoadArrayClassCache(MIRSymbol &st, int64 offset, PrimType primType)
6311 {
6312     StImmOperand &srcOpnd = CreateStImmOperand(st, offset, 0);
6313     RegOperand &resOpnd = CreateRegisterOperandOfType(primType);
6314     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_arrayclass_cache_ldr, resOpnd, srcOpnd));
6315     return &resOpnd;
6316 }
6317 
SelectAlloca(UnaryNode & node,Operand & opnd0)6318 Operand *AArch64CGFunc::SelectAlloca(UnaryNode &node, Operand &opnd0)
6319 {
6320     if (!CGOptions::IsArm64ilp32()) {
6321         DEBUG_ASSERT((node.GetPrimType() == PTY_a64), "wrong type");
6322     }
6323     if (GetCG()->IsLmbc()) {
6324         SetHasVLAOrAlloca(true);
6325     }
6326     PrimType stype = node.Opnd(0)->GetPrimType();
6327     Operand *resOpnd = &opnd0;
6328     if (GetPrimTypeBitSize(stype) < GetPrimTypeBitSize(PTY_u64)) {
6329         resOpnd = &CreateRegisterOperandOfType(PTY_u64);
6330         SelectCvtInt2Int(nullptr, resOpnd, &opnd0, stype, PTY_u64);
6331     }
6332 
6333     RegOperand &aliOp = CreateRegisterOperandOfType(PTY_u64);
6334 
6335     SelectAdd(aliOp, *resOpnd, CreateImmOperand(kAarch64StackPtrAlignment - 1, k64BitSize, true), PTY_u64);
6336     Operand &shifOpnd = CreateImmOperand(__builtin_ctz(kAarch64StackPtrAlignment), k64BitSize, true);
6337     SelectShift(aliOp, aliOp, shifOpnd, kShiftLright, PTY_u64);
6338     SelectShift(aliOp, aliOp, shifOpnd, kShiftLeft, PTY_u64);
6339     Operand &spOpnd = GetOrCreatePhysicalRegisterOperand(RSP, k64BitSize, kRegTyInt);
6340     SelectSub(spOpnd, spOpnd, aliOp, PTY_u64);
6341     int64 allocaOffset = GetMemlayout()->SizeOfArgsToStackPass();
6342     if (GetCG()->IsLmbc()) {
6343         allocaOffset -= kDivide2 * k8ByteSize;
6344     }
6345     if (allocaOffset > 0) {
6346         RegOperand &resallo = CreateRegisterOperandOfType(PTY_u64);
6347         SelectAdd(resallo, spOpnd, CreateImmOperand(allocaOffset, k64BitSize, true), PTY_u64);
6348         return &resallo;
6349     } else {
6350         return &SelectCopy(spOpnd, PTY_u64, PTY_u64);
6351     }
6352 }
6353 
SelectMalloc(UnaryNode & node,Operand & opnd0)6354 Operand *AArch64CGFunc::SelectMalloc(UnaryNode &node, Operand &opnd0)
6355 {
6356     PrimType retType = node.GetPrimType();
6357     DEBUG_ASSERT((retType == PTY_a64), "wrong type");
6358 
6359     std::vector<Operand *> opndVec;
6360     RegOperand &resOpnd = CreateRegisterOperandOfType(retType);
6361     opndVec.emplace_back(&resOpnd);
6362     opndVec.emplace_back(&opnd0);
6363     /* Use calloc to make sure allocated memory is zero-initialized */
6364     const std::string &funcName = "calloc";
6365     PrimType srcPty = PTY_u64;
6366     if (opnd0.GetSize() <= k32BitSize) {
6367         srcPty = PTY_u32;
6368     }
6369     Operand &opnd1 = CreateImmOperand(1, srcPty, false);
6370     opndVec.emplace_back(&opnd1);
6371     SelectLibCall(funcName, opndVec, srcPty, retType);
6372     return &resOpnd;
6373 }
6374 
SelectGCMalloc(GCMallocNode & node)6375 Operand *AArch64CGFunc::SelectGCMalloc(GCMallocNode &node)
6376 {
6377     PrimType retType = node.GetPrimType();
6378     DEBUG_ASSERT((retType == PTY_a64), "wrong type");
6379 
6380     /* Get the size and alignment of the type. */
6381     TyIdx tyIdx = node.GetTyIdx();
6382     uint64 size = GetBecommon().GetTypeSize(tyIdx);
6383     uint8 align = RTSupport::GetRTSupportInstance().GetObjectAlignment();
6384 
6385     /* Generate the call to MCC_NewObj */
6386     Operand &opndSize = CreateImmOperand(static_cast<int64>(size), k64BitSize, false);
6387     Operand &opndAlign = CreateImmOperand(align, k64BitSize, false);
6388 
6389     RegOperand &resOpnd = CreateRegisterOperandOfType(retType);
6390 
6391     std::vector<Operand *> opndVec {&resOpnd, &opndSize, &opndAlign};
6392 
6393     const std::string &funcName = "MCC_NewObj";
6394     SelectLibCall(funcName, opndVec, PTY_u64, retType);
6395 
6396     return &resOpnd;
6397 }
6398 
SelectJarrayMalloc(JarrayMallocNode & node,Operand & opnd0)6399 Operand *AArch64CGFunc::SelectJarrayMalloc(JarrayMallocNode &node, Operand &opnd0)
6400 {
6401     PrimType retType = node.GetPrimType();
6402     DEBUG_ASSERT((retType == PTY_a64), "wrong type");
6403 
6404     /* Extract jarray type */
6405     TyIdx tyIdx = node.GetTyIdx();
6406     MIRType *type = GlobalTables::GetTypeTable().GetTypeFromTyIdx(tyIdx);
6407     DEBUG_ASSERT(type != nullptr, "nullptr check");
6408     CHECK_FATAL(type->GetKind() == kTypeJArray, "expect MIRJarrayType");
6409     auto jaryType = static_cast<MIRJarrayType *>(type);
6410     uint64 fixedSize = RTSupport::GetRTSupportInstance().GetArrayContentOffset();
6411     uint8 align = RTSupport::GetRTSupportInstance().GetObjectAlignment();
6412 
6413     MIRType *elemType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(jaryType->GetElemTyIdx());
6414     PrimType elemPrimType = elemType->GetPrimType();
6415     uint64 elemSize = GetPrimTypeSize(elemPrimType);
6416 
6417     /* Generate the cal to MCC_NewObj_flexible */
6418     Operand &opndFixedSize = CreateImmOperand(PTY_u64, static_cast<int64>(fixedSize));
6419     Operand &opndElemSize = CreateImmOperand(PTY_u64, static_cast<int64>(elemSize));
6420 
6421     Operand *opndNElems = &opnd0;
6422 
6423     Operand *opndNElems64 = &static_cast<Operand &>(CreateRegisterOperandOfType(PTY_u64));
6424     SelectCvtInt2Int(nullptr, opndNElems64, opndNElems, PTY_u32, PTY_u64);
6425 
6426     Operand &opndAlign = CreateImmOperand(PTY_u64, align);
6427 
6428     RegOperand &resOpnd = CreateRegisterOperandOfType(retType);
6429 
6430     std::vector<Operand *> opndVec {&resOpnd, &opndFixedSize, &opndElemSize, opndNElems64, &opndAlign};
6431 
6432     const std::string &funcName = "MCC_NewObj_flexible";
6433     SelectLibCall(funcName, opndVec, PTY_u64, retType);
6434 
6435     /* Generate the store of the object length field */
6436     MemOperand &opndArrayLengthField =
6437         CreateMemOpnd(resOpnd, static_cast<int64>(RTSupport::GetRTSupportInstance().GetArrayLengthOffset()), k4BitSize);
6438     RegOperand *regOpndNElems = &SelectCopy(*opndNElems, PTY_u32, PTY_u32);
6439     DEBUG_ASSERT(regOpndNElems != nullptr, "null ptr check!");
6440     SelectCopy(opndArrayLengthField, PTY_u32, *regOpndNElems, PTY_u32);
6441 
6442     return &resOpnd;
6443 }
6444 
IsRegRematCand(const RegOperand & reg) const6445 bool AArch64CGFunc::IsRegRematCand(const RegOperand &reg) const
6446 {
6447     MIRPreg *preg = GetPseudoRegFromVirtualRegNO(reg.GetRegisterNumber(), CGOptions::DoCGSSA());
6448     if (preg != nullptr && preg->GetOp() != OP_undef) {
6449         if (preg->GetOp() == OP_constval && cg->GetRematLevel() >= kRematConst) {
6450             return true;
6451         } else if (preg->GetOp() == OP_addrof && cg->GetRematLevel() >= kRematAddr) {
6452             return true;
6453         } else if (preg->GetOp() == OP_iread && cg->GetRematLevel() >= kRematDreadGlobal) {
6454             return true;
6455         } else {
6456             return false;
6457         }
6458     } else {
6459         return false;
6460     }
6461 }
6462 
ClearRegRematInfo(const RegOperand & reg) const6463 void AArch64CGFunc::ClearRegRematInfo(const RegOperand &reg) const
6464 {
6465     MIRPreg *preg = GetPseudoRegFromVirtualRegNO(reg.GetRegisterNumber(), CGOptions::DoCGSSA());
6466     if (preg != nullptr && preg->GetOp() != OP_undef) {
6467         preg->SetOp(OP_undef);
6468     }
6469 }
6470 
IsRegSameRematInfo(const RegOperand & regDest,const RegOperand & regSrc) const6471 bool AArch64CGFunc::IsRegSameRematInfo(const RegOperand &regDest, const RegOperand &regSrc) const
6472 {
6473     MIRPreg *pregDest = GetPseudoRegFromVirtualRegNO(regDest.GetRegisterNumber(), CGOptions::DoCGSSA());
6474     MIRPreg *pregSrc = GetPseudoRegFromVirtualRegNO(regSrc.GetRegisterNumber(), CGOptions::DoCGSSA());
6475     if (pregDest != nullptr && pregDest == pregSrc) {
6476         if (pregDest->GetOp() == OP_constval && cg->GetRematLevel() >= kRematConst) {
6477             return true;
6478         } else if (pregDest->GetOp() == OP_addrof && cg->GetRematLevel() >= kRematAddr) {
6479             return true;
6480         } else if (pregDest->GetOp() == OP_iread && cg->GetRematLevel() >= kRematDreadGlobal) {
6481             return true;
6482         } else {
6483             return false;
6484         }
6485     } else {
6486         return false;
6487     }
6488 }
6489 
ReplaceOpndInInsn(RegOperand & regDest,RegOperand & regSrc,Insn & insn,regno_t destNO)6490 void AArch64CGFunc::ReplaceOpndInInsn(RegOperand &regDest, RegOperand &regSrc, Insn &insn, regno_t destNO)
6491 {
6492     auto opndNum = static_cast<int32>(insn.GetOperandSize());
6493     for (int i = opndNum - 1; i >= 0; --i) {
6494         Operand &opnd = insn.GetOperand(static_cast<uint32>(i));
6495         if (opnd.IsList()) {
6496             std::list<RegOperand *> tempRegStore;
6497             auto &opndList = static_cast<ListOperand &>(opnd).GetOperands();
6498             bool needReplace = false;
6499             for (auto it = opndList.cbegin(), end = opndList.cend(); it != end; ++it) {
6500                 auto *regOpnd = *it;
6501                 if (regOpnd->GetRegisterNumber() == destNO) {
6502                     needReplace = true;
6503                     tempRegStore.push_back(&regSrc);
6504                 } else {
6505                     tempRegStore.push_back(regOpnd);
6506                 }
6507             }
6508             if (needReplace) {
6509                 opndList.clear();
6510                 for (auto &newOpnd : std::as_const(tempRegStore)) {
6511                     static_cast<ListOperand &>(opnd).PushOpnd(*newOpnd);
6512                 }
6513             }
6514         } else if (opnd.IsMemoryAccessOperand()) {
6515             auto &memOpnd = static_cast<MemOperand &>(opnd);
6516             RegOperand *baseRegOpnd = memOpnd.GetBaseRegister();
6517             RegOperand *indexRegOpnd = memOpnd.GetIndexRegister();
6518             MemOperand *newMem = static_cast<MemOperand *>(memOpnd.Clone(*GetMemoryPool()));
6519             if ((baseRegOpnd != nullptr && baseRegOpnd->GetRegisterNumber() == destNO) ||
6520                 (indexRegOpnd != nullptr && indexRegOpnd->GetRegisterNumber() == destNO)) {
6521                 if (baseRegOpnd != nullptr && baseRegOpnd->GetRegisterNumber() == destNO) {
6522                     newMem->SetBaseRegister(regSrc);
6523                 }
6524                 if (indexRegOpnd != nullptr && indexRegOpnd->GetRegisterNumber() == destNO) {
6525                     auto *newRegSrc = static_cast<RegOperand*>(regSrc.Clone(*GetMemoryPool()));
6526                     newRegSrc->SetSize(indexRegOpnd->GetSize()); // retain the original size
6527                     newMem->SetIndexRegister(*newRegSrc);
6528                 }
6529                 insn.SetMemOpnd(newMem);
6530             }
6531         } else if (opnd.IsRegister()) {
6532             auto &regOpnd = static_cast<RegOperand &>(opnd);
6533             if (regOpnd.GetRegisterNumber() == destNO) {
6534                 DEBUG_ASSERT(regOpnd.GetRegisterNumber() != kRFLAG, "both condi and reg");
6535                 if (regOpnd.GetBaseRefOpnd() != nullptr) {
6536                     regSrc.SetBaseRefOpnd(*regOpnd.GetBaseRefOpnd());
6537                     ReplaceRegReference(regOpnd.GetRegisterNumber(), regSrc.GetRegisterNumber());
6538                 }
6539                 insn.SetOperand(static_cast<uint32>(i), regSrc);
6540             }
6541             if (regOpnd.GetBaseRefOpnd() != nullptr && regOpnd.GetBaseRefOpnd()->GetRegisterNumber() == destNO) {
6542                 regOpnd.SetBaseRefOpnd(regSrc);
6543                 ReplaceRegReference(regOpnd.GetBaseRefOpnd()->GetRegisterNumber(), regSrc.GetRegisterNumber());
6544             }
6545         }
6546     }
6547     if (insn.GetStackMap() != nullptr) {
6548         for (auto [deoptVreg, opnd] : insn.GetStackMap()->GetDeoptInfo().GetDeoptBundleInfo()) {
6549             if (!opnd->IsRegister()) {
6550                 continue;
6551             }
6552             auto &regOpnd = static_cast<RegOperand&>(*opnd);
6553             if (regOpnd.GetRegisterNumber() == destNO) {
6554                 insn.GetStackMap()->GetDeoptInfo().ReplaceDeoptBundleInfo(deoptVreg, regSrc);
6555                 ReplaceRegReference(regOpnd.GetRegisterNumber(), regSrc.GetRegisterNumber());
6556             }
6557         }
6558     }
6559 }
6560 
CleanupDeadMov(bool dumpInfo)6561 void AArch64CGFunc::CleanupDeadMov(bool dumpInfo)
6562 {
6563     /* clean dead mov. */
6564     FOR_ALL_BB(bb, this) {
6565         FOR_BB_INSNS_SAFE(insn, bb, ninsn) {
6566             if (!insn->IsMachineInstruction()) {
6567                 continue;
6568             }
6569             if (insn->GetMachineOpcode() == MOP_xmovrr || insn->GetMachineOpcode() == MOP_wmovrr ||
6570                 insn->GetMachineOpcode() == MOP_xvmovs || insn->GetMachineOpcode() == MOP_xvmovd) {
6571                 RegOperand &regDest = static_cast<RegOperand &>(insn->GetOperand(kInsnFirstOpnd));
6572                 RegOperand &regSrc = static_cast<RegOperand &>(insn->GetOperand(kInsnSecondOpnd));
6573                 if (!regSrc.IsVirtualRegister() || !regDest.IsVirtualRegister()) {
6574                     continue;
6575                 }
6576 
6577                 if (regSrc.GetRegisterNumber() == regDest.GetRegisterNumber()) {
6578                     bb->RemoveInsn(*insn);
6579                 } else if (insn->IsPhiMovInsn() && dumpInfo) {
6580                     LogInfo::MapleLogger() << "fail to remove mov: " << regDest.GetRegisterNumber() << " <- "
6581                                            << regSrc.GetRegisterNumber() << std::endl;
6582                 }
6583             }
6584         }
6585     }
6586 }
6587 
GetRealCallerSaveRegs(const Insn & insn,std::set<regno_t> & realSaveRegs)6588 void AArch64CGFunc::GetRealCallerSaveRegs(const Insn &insn, std::set<regno_t> &realSaveRegs)
6589 {
6590     auto *targetOpnd = insn.GetCallTargetOperand();
6591     CHECK_FATAL(targetOpnd != nullptr, "target is null in AArch64Insn::IsCallToFunctionThatNeverReturns");
6592     if (CGOptions::DoIPARA() && targetOpnd->IsFuncNameOpnd()) {
6593         FuncNameOperand *target = static_cast<FuncNameOperand *>(targetOpnd);
6594         const MIRSymbol *funcSt = target->GetFunctionSymbol();
6595         DEBUG_ASSERT(funcSt->GetSKind() == kStFunc, "funcst must be a function name symbol");
6596         MIRFunction *func = funcSt->GetFunction();
6597         if (func != nullptr && func->IsReferedRegsValid()) {
6598             for (auto preg : func->GetReferedRegs()) {
6599                 if (AArch64Abi::IsCallerSaveReg(static_cast<AArch64reg>(preg))) {
6600                     realSaveRegs.insert(preg);
6601                 }
6602             }
6603             return;
6604         }
6605     }
6606     for (uint32 i = R0; i <= kMaxRegNum; ++i) {
6607         if (AArch64Abi::IsCallerSaveReg(static_cast<AArch64reg>(i))) {
6608             realSaveRegs.insert(i);
6609         }
6610     }
6611 }
6612 
GetZeroOpnd(uint32 bitLen)6613 RegOperand &AArch64CGFunc::GetZeroOpnd(uint32 bitLen)
6614 {
6615     /*
6616      * It is possible to have a bitLen < 32, eg stb.
6617      * Set it to 32 if it is less than 32.
6618      */
6619     if (bitLen < k32BitSize) {
6620         bitLen = k32BitSize;
6621     }
6622     DEBUG_ASSERT((bitLen == k32BitSize || bitLen == k64BitSize), "illegal bit length = %d", bitLen);
6623     return (bitLen == k32BitSize) ? GetOrCreatePhysicalRegisterOperand(RZR, k32BitSize, kRegTyInt)
6624                                   : GetOrCreatePhysicalRegisterOperand(RZR, k64BitSize, kRegTyInt);
6625 }
6626 
IsFrameReg(const RegOperand & opnd) const6627 bool AArch64CGFunc::IsFrameReg(const RegOperand &opnd) const
6628 {
6629     if (opnd.GetRegisterNumber() == RFP) {
6630         return true;
6631     } else {
6632         return false;
6633     }
6634 }
6635 
IsSaveReg(const RegOperand & reg,MIRType & mirType,BECommon & cgBeCommon)6636 bool AArch64CGFunc::IsSaveReg(const RegOperand &reg, MIRType &mirType, BECommon &cgBeCommon)
6637 {
6638     CCImpl &retLocator = *GetOrCreateLocator(GetCurCallConvKind());
6639     CCLocInfo retMechanism;
6640     retLocator.InitReturnInfo(mirType, retMechanism);
6641     if (retMechanism.GetRegCount() > 0) {
6642         return reg.GetRegisterNumber() == retMechanism.GetReg0() || reg.GetRegisterNumber() == retMechanism.GetReg1() ||
6643                reg.GetRegisterNumber() == retMechanism.GetReg2() || reg.GetRegisterNumber() == retMechanism.GetReg3();
6644     }
6645     return false;
6646 }
6647 
IsSPOrFP(const RegOperand & opnd) const6648 bool AArch64CGFunc::IsSPOrFP(const RegOperand &opnd) const
6649 {
6650     const RegOperand &regOpnd = static_cast<const RegOperand &>(opnd);
6651     regno_t regNO = opnd.GetRegisterNumber();
6652     return (regOpnd.IsPhysicalRegister() &&
6653             (regNO == RSP || regNO == RFP || (regNO == R29 && CGOptions::UseFramePointer())));
6654 }
6655 
IsReturnReg(const RegOperand & opnd) const6656 bool AArch64CGFunc::IsReturnReg(const RegOperand &opnd) const
6657 {
6658     regno_t regNO = opnd.GetRegisterNumber();
6659     return (regNO == R0) || (regNO == V0);
6660 }
6661 
6662 /*
6663  * This function returns true to indicate that the clean up code needs to be generated,
6664  * otherwise it does not need. In GCOnly mode, it always returns false.
6665  */
NeedCleanup()6666 bool AArch64CGFunc::NeedCleanup()
6667 {
6668     if (CGOptions::IsGCOnly()) {
6669         return false;
6670     }
6671     AArch64MemLayout *layout = static_cast<AArch64MemLayout *>(GetMemlayout());
6672     if (layout->GetSizeOfRefLocals() > 0) {
6673         return true;
6674     }
6675     for (uint32 i = 0; i < GetFunction().GetFormalCount(); i++) {
6676         TypeAttrs ta = GetFunction().GetNthParamAttr(i);
6677         if (ta.GetAttr(ATTR_localrefvar)) {
6678             return true;
6679         }
6680     }
6681 
6682     return false;
6683 }
6684 
6685 /*
6686  * bb must be the cleanup bb.
6687  * this function must be invoked before register allocation.
6688  * extended epilogue is specific for fast exception handling and is made up of
6689  * clean up code and epilogue.
6690  * clean up code is generated here while epilogue is generated in GeneratePrologEpilog()
6691  */
GenerateCleanupCodeForExtEpilog(BB & bb)6692 void AArch64CGFunc::GenerateCleanupCodeForExtEpilog(BB &bb)
6693 {
6694     DEBUG_ASSERT(GetLastBB()->GetPrev()->GetFirstStmt() == GetCleanupLabel(), "must be");
6695 
6696     if (NeedCleanup()) {
6697         /* this is necessary for code insertion. */
6698         SetCurBB(bb);
6699 
6700         RegOperand &regOpnd0 =
6701             GetOrCreatePhysicalRegisterOperand(R0, GetPointerSize() * kBitsPerByte, GetRegTyFromPrimTy(PTY_a64));
6702         RegOperand &regOpnd1 =
6703             GetOrCreatePhysicalRegisterOperand(R1, GetPointerSize() * kBitsPerByte, GetRegTyFromPrimTy(PTY_a64));
6704         /* allocate 16 bytes to store reg0 and reg1 (each reg has 8 bytes) */
6705         MemOperand &frameAlloc = CreateCallFrameOperand(-16, GetPointerSize() * kBitsPerByte);
6706         Insn &allocInsn = GetInsnBuilder()->BuildInsn(MOP_xstp, regOpnd0, regOpnd1, frameAlloc);
6707         allocInsn.SetDoNotRemove(true);
6708         AppendInstructionTo(allocInsn, *this);
6709 
6710         /* invoke MCC_CleanupLocalStackRef(). */
6711         HandleRCCall(false);
6712         /* deallocate 16 bytes which used to store reg0 and reg1 */
6713         MemOperand &frameDealloc = CreateCallFrameOperand(16, GetPointerSize() * kBitsPerByte);
6714         GenRetCleanup(cleanEANode, true);
6715         Insn &deallocInsn = GetInsnBuilder()->BuildInsn(MOP_xldp, regOpnd0, regOpnd1, frameDealloc);
6716         deallocInsn.SetDoNotRemove(true);
6717         AppendInstructionTo(deallocInsn, *this);
6718         /* Update cleanupbb since bb may have been splitted */
6719         SetCleanupBB(*GetCurBB());
6720     }
6721 }
6722 
6723 /*
6724  * bb must be the cleanup bb.
6725  * this function must be invoked before register allocation.
6726  */
GenerateCleanupCode(BB & bb)6727 void AArch64CGFunc::GenerateCleanupCode(BB &bb)
6728 {
6729     DEBUG_ASSERT(GetLastBB()->GetPrev()->GetFirstStmt() == GetCleanupLabel(), "must be");
6730     if (!NeedCleanup()) {
6731         return;
6732     }
6733 
6734     /* this is necessary for code insertion. */
6735     SetCurBB(bb);
6736 
6737     /* R0 is lived-in for clean-up code, save R0 before invocation */
6738     RegOperand &livein = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
6739 
6740     if (!GetCG()->GenLocalRC()) {
6741         /* by pass local RC operations. */
6742     } else if (Globals::GetInstance()->GetOptimLevel() > CGOptions::kLevel0) {
6743         regno_t vreg = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
6744         RegOperand &backupRegOp = CreateVirtualRegisterOperand(vreg);
6745         backupRegOp.SetRegNotBBLocal();
6746         SelectCopy(backupRegOp, PTY_a64, livein, PTY_a64);
6747 
6748         /* invoke MCC_CleanupLocalStackRef(). */
6749         HandleRCCall(false);
6750         SelectCopy(livein, PTY_a64, backupRegOp, PTY_a64);
6751     } else {
6752         /*
6753          * Register Allocation for O0 can not handle this case, so use a callee saved register directly.
6754          * If yieldpoint is enabled, we use R20 instead R19.
6755          */
6756         AArch64reg backupRegNO = GetCG()->GenYieldPoint() ? R20 : R19;
6757         RegOperand &backupRegOp =
6758             GetOrCreatePhysicalRegisterOperand(backupRegNO, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
6759         SelectCopy(backupRegOp, PTY_a64, livein, PTY_a64);
6760         /* invoke MCC_CleanupLocalStackRef(). */
6761         HandleRCCall(false);
6762         SelectCopy(livein, PTY_a64, backupRegOp, PTY_a64);
6763     }
6764 
6765     /* invoke _Unwind_Resume */
6766     std::string funcName("_Unwind_Resume");
6767     MIRSymbol *sym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
6768     sym->SetNameStrIdx(funcName);
6769     sym->SetStorageClass(kScText);
6770     sym->SetSKind(kStFunc);
6771     ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
6772     srcOpnds->PushOpnd(livein);
6773     AppendCall(*sym, *srcOpnds);
6774     /*
6775      * this instruction is unreachable, but we need it as the return address of previous
6776      * "bl _Unwind_Resume" for stack unwinding.
6777      */
6778     Insn &nop = GetInsnBuilder()->BuildInsn(MOP_xblr, livein, *srcOpnds);
6779     GetCurBB()->AppendInsn(nop);
6780     GetCurBB()->SetHasCall();
6781 
6782     /* Update cleanupbb since bb may have been splitted */
6783     SetCleanupBB(*GetCurBB());
6784 }
6785 
FloatParamRegRequired(MIRStructType * structType,uint32 & fpSize)6786 uint32 AArch64CGFunc::FloatParamRegRequired(MIRStructType *structType, uint32 &fpSize)
6787 {
6788     CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
6789     AArch64CallConvImpl parmlocator(GetBecommon());
6790     return parmlocator.FloatParamRegRequired(*structType, fpSize);
6791 }
6792 
6793 /*
6794  * Map param registers to formals.  For small structs passed in param registers,
6795  * create a move to vreg since lmbc IR does not create a regassign for them.
6796  */
AssignLmbcFormalParams()6797 void AArch64CGFunc::AssignLmbcFormalParams()
6798 {
6799     PrimType primType;
6800     uint32 offset;
6801     regno_t intReg = R0;
6802     regno_t fpReg = V0;
6803     for (auto param : GetLmbcParamVec()) {
6804         primType = param->GetPrimType();
6805         offset = param->GetOffset();
6806         if (param->IsReturn()) {
6807             param->SetRegNO(R8);
6808         } else if (IsPrimitiveInteger(primType)) {
6809             if (intReg > R7) {
6810                 param->SetRegNO(0);
6811             } else {
6812                 param->SetRegNO(intReg);
6813                 if (!param->HasRegassign()) {
6814                     uint32 bytelen = GetPrimTypeSize(primType);
6815                     uint32 bitlen = bytelen * kBitsPerByte;
6816                     MemOperand *mOpnd = GenLmbcFpMemOperand(static_cast<int32>(offset), bytelen);
6817                     RegOperand &src = GetOrCreatePhysicalRegisterOperand(AArch64reg(intReg), bitlen, kRegTyInt);
6818                     MOperator mOp = PickStInsn(bitlen, primType);
6819                     Insn &store = GetInsnBuilder()->BuildInsn(mOp, src, *mOpnd);
6820                     GetCurBB()->AppendInsn(store);
6821                 }
6822                 intReg++;
6823             }
6824         } else if (IsPrimitiveFloat(primType)) {
6825             if (fpReg > V7) {
6826                 param->SetRegNO(0);
6827             } else {
6828                 param->SetRegNO(fpReg);
6829                 if (!param->HasRegassign()) {
6830                     uint32 bytelen = GetPrimTypeSize(primType);
6831                     uint32 bitlen = bytelen * kBitsPerByte;
6832                     MemOperand *mOpnd = GenLmbcFpMemOperand(static_cast<int32>(offset), bytelen);
6833                     RegOperand &src = GetOrCreatePhysicalRegisterOperand(AArch64reg(fpReg), bitlen, kRegTyFloat);
6834                     MOperator mOp = PickStInsn(bitlen, primType);
6835                     Insn &store = GetInsnBuilder()->BuildInsn(mOp, src, *mOpnd);
6836                     GetCurBB()->AppendInsn(store);
6837                 }
6838                 fpReg++;
6839             }
6840         } else if (primType == PTY_agg) {
6841             if (param->IsPureFloat()) {
6842                 uint32 numFpRegs = param->GetNumRegs();
6843                 if ((fpReg + numFpRegs - kOneRegister) > V7) {
6844                     param->SetRegNO(0);
6845                 } else {
6846                     param->SetRegNO(fpReg);
6847                     param->SetNumRegs(numFpRegs);
6848                     fpReg += numFpRegs;
6849                 }
6850             } else if (param->GetSize() > k16ByteSize) {
6851                 if (intReg > R7) {
6852                     param->SetRegNO(0);
6853                 } else {
6854                     param->SetRegNO(intReg);
6855                     param->SetIsOnStack();
6856                     param->SetOnStackOffset(((intReg - R0 + fpReg) - V0) * k8ByteSize);
6857                     uint32 bytelen = GetPrimTypeSize(PTY_a64);
6858                     uint32 bitlen = bytelen * kBitsPerByte;
6859                     MemOperand *mOpnd = GenLmbcFpMemOperand(static_cast<int32>(param->GetOnStackOffset()), bytelen);
6860                     RegOperand &src = GetOrCreatePhysicalRegisterOperand(AArch64reg(intReg), bitlen, kRegTyInt);
6861                     MOperator mOp = PickStInsn(bitlen, PTY_a64);
6862                     Insn &store = GetInsnBuilder()->BuildInsn(mOp, src, *mOpnd);
6863                     GetCurBB()->AppendInsn(store);
6864                     intReg++;
6865                 }
6866             } else if (param->GetSize() <= k8ByteSize) {
6867                 if (intReg > R7) {
6868                     param->SetRegNO(0);
6869                 } else {
6870                     param->SetRegNO(intReg);
6871                     param->SetNumRegs(kOneRegister);
6872                     intReg++;
6873                 }
6874             } else {
6875                 /* size > 8 && size <= 16 */
6876                 if ((intReg + kOneRegister) > R7) {
6877                     param->SetRegNO(0);
6878                 } else {
6879                     param->SetRegNO(intReg);
6880                     param->SetNumRegs(kTwoRegister);
6881                     intReg += kTwoRegister;
6882                 }
6883             }
6884             if (param->GetRegNO() != 0) {
6885                 for (uint32 i = 0; i < param->GetNumRegs(); ++i) {
6886                     PrimType pType = PTY_i64;
6887                     RegType rType = kRegTyInt;
6888                     uint32 rSize = k8ByteSize;
6889                     if (param->IsPureFloat()) {
6890                         rType = kRegTyFloat;
6891                         if (param->GetFpSize() <= k4ByteSize) {
6892                             pType = PTY_f32;
6893                             rSize = k4ByteSize;
6894                         } else {
6895                             pType = PTY_f64;
6896                         }
6897                     }
6898                     regno_t vreg = NewVReg(rType, rSize);
6899                     RegOperand &dest = GetOrCreateVirtualRegisterOperand(vreg);
6900                     RegOperand &src = GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(param->GetRegNO() + i),
6901                                                                          rSize * kBitsPerByte, rType);
6902                     SelectCopy(dest, pType, src, pType);
6903                     if (param->GetVregNO() == 0) {
6904                         param->SetVregNO(vreg);
6905                     }
6906                     Operand *memOpd = &CreateMemOpnd(RFP, offset + (i * rSize), rSize);
6907                     GetCurBB()->AppendInsn(
6908                         GetInsnBuilder()->BuildInsn(PickStInsn(rSize * kBitsPerByte, pType), dest, *memOpd));
6909                 }
6910             }
6911         } else {
6912             CHECK_FATAL(false, "lmbc formal primtype not handled");
6913         }
6914     }
6915 }
6916 
LmbcGenSaveSpForAlloca()6917 void AArch64CGFunc::LmbcGenSaveSpForAlloca()
6918 {
6919     if (GetMirModule().GetFlavor() != MIRFlavor::kFlavorLmbc || !HasVLAOrAlloca()) {
6920         return;
6921     }
6922     Operand &spOpnd = GetOrCreatePhysicalRegisterOperand(RSP, k64BitSize, kRegTyInt);
6923     RegOperand &spSaveOpnd = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, GetPointerSize()));
6924     Insn &save = GetInsnBuilder()->BuildInsn(MOP_xmovrr, spSaveOpnd, spOpnd);
6925     GetFirstBB()->AppendInsn(save);
6926     for (auto *retBB : GetExitBBsVec()) {
6927         Insn &restore = GetInsnBuilder()->BuildInsn(MOP_xmovrr, spOpnd, spSaveOpnd);
6928         retBB->AppendInsn(restore);
6929         restore.SetFrameDef(true);
6930     }
6931 }
6932 
6933 /* if offset < 0, allocation; otherwise, deallocation */
CreateCallFrameOperand(int32 offset,uint32 size)6934 MemOperand &AArch64CGFunc::CreateCallFrameOperand(int32 offset, uint32 size)
6935 {
6936     MemOperand *memOpnd = CreateStackMemOpnd(RSP, offset, size);
6937     memOpnd->SetIndexOpt((offset < 0) ? MemOperand::kPreIndex : MemOperand::kPostIndex);
6938     return *memOpnd;
6939 }
6940 
GetLogicalShiftLeftOperand(uint32 shiftAmount,bool is64bits) const6941 BitShiftOperand *AArch64CGFunc::GetLogicalShiftLeftOperand(uint32 shiftAmount, bool is64bits) const
6942 {
6943     /* num(0, 16, 32, 48) >> 4 is num1(0, 1, 2, 3), num1 & (~3) == 0 */
6944     DEBUG_ASSERT((!shiftAmount || ((shiftAmount >> 4) & ~static_cast<uint32>(3)) == 0),
6945                  "shift amount should be one of 0, 16, 32, 48");
6946     /* movkLslOperands[4]~movkLslOperands[7] is for 64 bits */
6947     return &movkLslOperands[(shiftAmount >> 4) + (is64bits ? 4 : 0)];
6948 }
6949 
6950 AArch64CGFunc::MovkLslOperandArray AArch64CGFunc::movkLslOperands = {
6951     BitShiftOperand(BitShiftOperand::kLSL, 0, 4),
6952     BitShiftOperand(BitShiftOperand::kLSL, 16, 4),
6953     BitShiftOperand(BitShiftOperand::kLSL, static_cast<uint32>(-1), 0), /* invalid entry */
6954     BitShiftOperand(BitShiftOperand::kLSL, static_cast<uint32>(-1), 0), /* invalid entry */
6955     BitShiftOperand(BitShiftOperand::kLSL, 0, 6),
6956     BitShiftOperand(BitShiftOperand::kLSL, 16, 6),
6957     BitShiftOperand(BitShiftOperand::kLSL, 32, 6),
6958     BitShiftOperand(BitShiftOperand::kLSL, 48, 6),
6959 };
6960 
CreateStkTopOpnd(uint32 offset,uint32 size)6961 MemOperand &AArch64CGFunc::CreateStkTopOpnd(uint32 offset, uint32 size)
6962 {
6963     AArch64reg reg;
6964     if (GetMirModule().GetFlavor() == MIRFlavor::kFlavorLmbc) {
6965         reg = RSP;
6966     } else {
6967         reg = RFP;
6968     }
6969     MemOperand *memOp = CreateStackMemOpnd(reg, static_cast<int32>(offset), size);
6970     return *memOp;
6971 }
6972 
CreateStackMemOpnd(regno_t preg,int32 offset,uint32 size)6973 MemOperand *AArch64CGFunc::CreateStackMemOpnd(regno_t preg, int32 offset, uint32 size)
6974 {
6975     auto *memOp =
6976         memPool->New<MemOperand>(memPool->New<RegOperand>(preg, k64BitSize, kRegTyInt),
6977                                  &CreateOfstOpnd(static_cast<uint64>(static_cast<int64>(offset)), k32BitSize), size);
6978     if (preg == RFP || preg == RSP) {
6979         memOp->SetStackMem(true);
6980     }
6981     return memOp;
6982 }
6983 
CreateMemOperand(MemOperand::AArch64AddressingMode mode,uint32 size,RegOperand & base,RegOperand * index,ImmOperand * offset,const MIRSymbol * symbol) const6984 MemOperand *AArch64CGFunc::CreateMemOperand(MemOperand::AArch64AddressingMode mode, uint32 size, RegOperand &base,
6985                                             RegOperand *index, ImmOperand *offset, const MIRSymbol *symbol) const
6986 {
6987     auto *memOp = memPool->New<MemOperand>(mode, size, base, index, offset, symbol);
6988     if (base.GetRegisterNumber() == RFP || base.GetRegisterNumber() == RSP) {
6989         memOp->SetStackMem(true);
6990     }
6991     return memOp;
6992 }
6993 
CreateMemOperand(MemOperand::AArch64AddressingMode mode,uint32 size,RegOperand & base,RegOperand & index,ImmOperand * offset,const MIRSymbol & symbol,bool noExtend)6994 MemOperand *AArch64CGFunc::CreateMemOperand(MemOperand::AArch64AddressingMode mode, uint32 size, RegOperand &base,
6995                                             RegOperand &index, ImmOperand *offset, const MIRSymbol &symbol,
6996                                             bool noExtend)
6997 {
6998     auto *memOp = memPool->New<MemOperand>(mode, size, base, index, offset, symbol, noExtend);
6999     if (base.GetRegisterNumber() == RFP || base.GetRegisterNumber() == RSP) {
7000         memOp->SetStackMem(true);
7001     }
7002     return memOp;
7003 }
7004 
CreateMemOperand(MemOperand::AArch64AddressingMode mode,uint32 dSize,RegOperand & base,RegOperand & indexOpnd,uint32 shift,bool isSigned) const7005 MemOperand *AArch64CGFunc::CreateMemOperand(MemOperand::AArch64AddressingMode mode, uint32 dSize, RegOperand &base,
7006                                             RegOperand &indexOpnd, uint32 shift, bool isSigned) const
7007 {
7008     auto *memOp = memPool->New<MemOperand>(mode, dSize, base, indexOpnd, shift, isSigned);
7009     if (base.GetRegisterNumber() == RFP || base.GetRegisterNumber() == RSP) {
7010         memOp->SetStackMem(true);
7011     }
7012     return memOp;
7013 }
7014 
CreateMemOperand(MemOperand::AArch64AddressingMode mode,uint32 dSize,const MIRSymbol & sym)7015 MemOperand *AArch64CGFunc::CreateMemOperand(MemOperand::AArch64AddressingMode mode, uint32 dSize, const MIRSymbol &sym)
7016 {
7017     auto *memOp = memPool->New<MemOperand>(mode, dSize, sym);
7018     return memOp;
7019 }
7020 
GenSaveMethodInfoCode(BB & bb)7021 void AArch64CGFunc::GenSaveMethodInfoCode(BB &bb)
7022 {
7023     if (GetCG()->UseFastUnwind()) {
7024         BB *formerCurBB = GetCurBB();
7025         GetDummyBB()->ClearInsns();
7026         SetCurBB(*GetDummyBB());
7027         /*
7028          * FUNCATTR_bridge for function: Ljava_2Flang_2FString_3B_7CcompareTo_7C_28Ljava_2Flang_2FObject_3B_29I, to
7029          * exclude this funciton this function is a bridge function generated for Java Genetic
7030          */
7031         if ((GetFunction().GetAttr(FUNCATTR_native) || GetFunction().GetAttr(FUNCATTR_fast_native)) &&
7032             !GetFunction().GetAttr(FUNCATTR_critical_native) && !GetFunction().GetAttr(FUNCATTR_bridge)) {
7033             RegOperand &fpReg = GetOrCreatePhysicalRegisterOperand(RFP, GetPointerSize() * kBitsPerByte, kRegTyInt);
7034 
7035             ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
7036             RegOperand &parmRegOpnd1 = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, kRegTyInt);
7037             srcOpnds->PushOpnd(parmRegOpnd1);
7038             Operand &immOpnd = CreateImmOperand(0, k64BitSize, false);
7039             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadri64, parmRegOpnd1, immOpnd));
7040             RegOperand &parmRegOpnd2 = GetOrCreatePhysicalRegisterOperand(R1, k64BitSize, kRegTyInt);
7041             srcOpnds->PushOpnd(parmRegOpnd2);
7042             SelectCopy(parmRegOpnd2, PTY_a64, fpReg, PTY_a64);
7043 
7044             MIRSymbol *sym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
7045             std::string funcName("MCC_SetRiskyUnwindContext");
7046             sym->SetNameStrIdx(funcName);
7047 
7048             sym->SetStorageClass(kScText);
7049             sym->SetSKind(kStFunc);
7050             AppendCall(*sym, *srcOpnds);
7051             bb.SetHasCall();
7052         }
7053 
7054         bb.InsertAtBeginning(*GetDummyBB());
7055         SetCurBB(*formerCurBB);
7056     }
7057 }
7058 
HasStackLoadStore()7059 bool AArch64CGFunc::HasStackLoadStore()
7060 {
7061     FOR_ALL_BB(bb, this) {
7062         FOR_BB_INSNS(insn, bb) {
7063             uint32 opndNum = insn->GetOperandSize();
7064             for (uint32 i = 0; i < opndNum; ++i) {
7065                 Operand &opnd = insn->GetOperand(i);
7066                 if (opnd.IsMemoryAccessOperand()) {
7067                     auto &memOpnd = static_cast<MemOperand &>(opnd);
7068                     Operand *base = memOpnd.GetBaseRegister();
7069 
7070                     if ((base != nullptr) && base->IsRegister()) {
7071                         RegOperand *regOpnd = static_cast<RegOperand *>(base);
7072                         RegType regType = regOpnd->GetRegisterType();
7073                         uint32 regNO = regOpnd->GetRegisterNumber();
7074                         if (((regType != kRegTyCc) && ((regNO == RFP) || (regNO == RSP))) || (regType == kRegTyVary)) {
7075                             return true;
7076                         }
7077                     }
7078                 }
7079             }
7080         }
7081     }
7082     return false;
7083 }
7084 
GenerateYieldpoint(BB & bb)7085 void AArch64CGFunc::GenerateYieldpoint(BB &bb)
7086 {
7087     /* ldr wzr, [RYP]  # RYP hold address of the polling page. */
7088     auto &wzr = GetZeroOpnd(k32BitSize);
7089     auto &pollingPage = CreateMemOpnd(RYP, 0, k32BitSize);
7090     auto &yieldPoint = GetInsnBuilder()->BuildInsn(MOP_wldr, wzr, pollingPage);
7091     if (GetCG()->GenerateVerboseCG()) {
7092         yieldPoint.SetComment("yieldpoint");
7093     }
7094     bb.AppendInsn(yieldPoint);
7095 }
7096 
ProcessReturnReg(PrimType primType,int32 sReg)7097 Operand &AArch64CGFunc::ProcessReturnReg(PrimType primType, int32 sReg)
7098 {
7099     return GetTargetRetOperand(primType, sReg);
7100 }
7101 
GetTargetRetOperand(PrimType primType,int32 sReg)7102 Operand &AArch64CGFunc::GetTargetRetOperand(PrimType primType, int32 sReg)
7103 {
7104     uint32 bitSize = GetPrimTypeBitSize(primType) < k32BitSize ? k32BitSize : GetPrimTypeBitSize(primType);
7105     AArch64reg pReg;
7106     if (sReg < 0) {
7107         return GetOrCreatePhysicalRegisterOperand(IsPrimitiveFloat(primType) || (IsPrimitiveVector(primType)) ? S0 : R0,
7108                                                   bitSize, GetRegTyFromPrimTy(primType));
7109     } else {
7110         switch (sReg) {
7111             case kSregRetval0:
7112                 pReg = IsPrimitiveFloat(primType) || (IsPrimitiveVector(primType)) ? S0 : R0;
7113                 break;
7114             case kSregRetval1:
7115                 pReg = R1;
7116                 break;
7117             default:
7118                 pReg = RLAST_INT_REG;
7119                 DEBUG_ASSERT(0, "GetTargetRetOperand: NYI");
7120         }
7121         return GetOrCreatePhysicalRegisterOperand(pReg, bitSize, GetRegTyFromPrimTy(primType));
7122     }
7123 }
7124 
CreateRegisterOperandOfType(PrimType primType)7125 RegOperand &AArch64CGFunc::CreateRegisterOperandOfType(PrimType primType)
7126 {
7127     RegType regType = GetRegTyFromPrimTy(primType);
7128     uint32 byteLength = GetPrimTypeSize(primType);
7129     return CreateRegisterOperandOfType(regType, byteLength);
7130 }
7131 
CreateRegisterOperandOfType(RegType regty,uint32 byteLen)7132 RegOperand &AArch64CGFunc::CreateRegisterOperandOfType(RegType regty, uint32 byteLen)
7133 {
7134     /* BUG: if half-precision floating point operations are supported? */
7135     /* AArch64 has 32-bit and 64-bit registers only */
7136     if (byteLen < k4ByteSize) {
7137         byteLen = k4ByteSize;
7138     }
7139     regno_t vRegNO = NewVReg(regty, byteLen);
7140     return CreateVirtualRegisterOperand(vRegNO);
7141 }
7142 
CreateRflagOperand()7143 RegOperand &AArch64CGFunc::CreateRflagOperand()
7144 {
7145     /* AArch64 has Status register that is 32-bit wide. */
7146     regno_t vRegNO = NewVRflag();
7147     return CreateVirtualRegisterOperand(vRegNO);
7148 }
7149 
MergeReturn()7150 void AArch64CGFunc::MergeReturn()
7151 {
7152     DEBUG_ASSERT(GetCurBB()->GetPrev()->GetFirstStmt() == GetCleanupLabel(), "must be");
7153 
7154     uint32 exitBBSize = GetExitBBsVec().size();
7155     if (exitBBSize == 0) {
7156         return;
7157     }
7158     if ((exitBBSize == 1) && GetExitBB(0) == GetCurBB()) {
7159         return;
7160     }
7161     if (exitBBSize == 1) {
7162         BB *onlyExitBB = GetExitBB(0);
7163         BB *onlyExitBBNext = onlyExitBB->GetNext();
7164         StmtNode *stmt = onlyExitBBNext->GetFirstStmt();
7165         /* only deal with the return_BB in the middle */
7166         if (stmt != GetCleanupLabel()) {
7167             LabelIdx labidx = CreateLabel();
7168             BB *retBB = CreateNewBB(labidx, onlyExitBB->IsUnreachable(), BB::kBBReturn, onlyExitBB->GetFrequency());
7169             onlyExitBB->AppendBB(*retBB);
7170             /* modify the original return BB. */
7171             DEBUG_ASSERT(onlyExitBB->GetKind() == BB::kBBReturn, "Error: suppose to merge multi return bb");
7172             onlyExitBB->SetKind(BB::kBBFallthru);
7173 
7174             GetExitBBsVec().pop_back();
7175             GetExitBBsVec().emplace_back(retBB);
7176             return;
7177         }
7178     }
7179 
7180     LabelIdx labidx = CreateLabel();
7181     LabelOperand &targetOpnd = GetOrCreateLabelOperand(labidx);
7182     uint32 freq = 0;
7183     for (auto *tmpBB : GetExitBBsVec()) {
7184         DEBUG_ASSERT(tmpBB->GetKind() == BB::kBBReturn, "Error: suppose to merge multi return bb");
7185         tmpBB->SetKind(BB::kBBGoto);
7186         tmpBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xuncond, targetOpnd));
7187         freq += tmpBB->GetFrequency();
7188     }
7189     BB *retBB = CreateNewBB(labidx, false, BB::kBBReturn, freq);
7190     GetCleanupBB()->PrependBB(*retBB);
7191 
7192     GetExitBBsVec().clear();
7193     GetExitBBsVec().emplace_back(retBB);
7194 }
7195 
HandleRetCleanup(NaryStmtNode & retNode)7196 void AArch64CGFunc::HandleRetCleanup(NaryStmtNode &retNode)
7197 {
7198     if (!GetCG()->GenLocalRC()) {
7199         /* handle local rc is disabled. */
7200         return;
7201     }
7202 
7203     constexpr uint8 opSize = 11;
7204     Opcode ops[opSize] = {OP_label, OP_goto,      OP_brfalse, OP_brtrue, OP_return, OP_call,
7205                           OP_icall, OP_rangegoto, OP_catch,   OP_try,    OP_endtry};
7206     std::set<Opcode> branchOp(ops, ops + opSize);
7207 
7208     /* get cleanup intrinsic */
7209     bool found = false;
7210     StmtNode *cleanupNode = retNode.GetPrev();
7211     cleanEANode = nullptr;
7212     while (cleanupNode != nullptr) {
7213         if (branchOp.find(cleanupNode->GetOpCode()) != branchOp.end()) {
7214             if (cleanupNode->GetOpCode() == OP_call) {
7215                 CallNode *callNode = static_cast<CallNode *>(cleanupNode);
7216                 MIRFunction *fn = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(callNode->GetPUIdx());
7217                 MIRSymbol *fsym = GetFunction().GetLocalOrGlobalSymbol(fn->GetStIdx(), false);
7218                 if ((fsym->GetName() == "MCC_DecRef_NaiveRCFast") || (fsym->GetName() == "MCC_IncRef_NaiveRCFast") ||
7219                     (fsym->GetName() == "MCC_IncDecRef_NaiveRCFast") || (fsym->GetName() == "MCC_LoadRefStatic") ||
7220                     (fsym->GetName() == "MCC_LoadRefField") || (fsym->GetName() == "MCC_LoadReferentField") ||
7221                     (fsym->GetName() == "MCC_LoadRefField_NaiveRCFast") ||
7222                     (fsym->GetName() == "MCC_LoadVolatileField") ||
7223                     (fsym->GetName() == "MCC_LoadVolatileStaticField") || (fsym->GetName() == "MCC_LoadWeakField") ||
7224                     (fsym->GetName() == "MCC_CheckObjMem")) {
7225                     cleanupNode = cleanupNode->GetPrev();
7226                     continue;
7227                 } else {
7228                     break;
7229                 }
7230             } else {
7231                 break;
7232             }
7233         }
7234 
7235         if (cleanupNode->GetOpCode() == OP_intrinsiccall) {
7236             IntrinsiccallNode *tempNode = static_cast<IntrinsiccallNode *>(cleanupNode);
7237             if ((tempNode->GetIntrinsic() == INTRN_MPL_CLEANUP_LOCALREFVARS) ||
7238                 (tempNode->GetIntrinsic() == INTRN_MPL_CLEANUP_LOCALREFVARS_SKIP)) {
7239                 GenRetCleanup(tempNode);
7240                 if (cleanEANode != nullptr) {
7241                     GenRetCleanup(cleanEANode, true);
7242                 }
7243                 found = true;
7244                 break;
7245             }
7246             if (tempNode->GetIntrinsic() == INTRN_MPL_CLEANUP_NORETESCOBJS) {
7247                 cleanEANode = tempNode;
7248             }
7249         }
7250         cleanupNode = cleanupNode->GetPrev();
7251     }
7252 
7253     if (!found) {
7254         MIRSymbol *retRef = nullptr;
7255         if (retNode.NumOpnds() != 0) {
7256             retRef = GetRetRefSymbol(*static_cast<NaryStmtNode &>(retNode).Opnd(0));
7257         }
7258         HandleRCCall(false, retRef);
7259     }
7260 }
7261 
GenRetCleanup(const IntrinsiccallNode * cleanupNode,bool forEA)7262 bool AArch64CGFunc::GenRetCleanup(const IntrinsiccallNode *cleanupNode, bool forEA)
7263 {
7264 #undef CC_DEBUG_INFO
7265 
7266 #ifdef CC_DEBUG_INFO
7267     LogInfo::MapleLogger() << "==============" << GetFunction().GetName() << "==============" << '\n';
7268 #endif
7269 
7270     if (cleanupNode == nullptr) {
7271         return false;
7272     }
7273 
7274     int32 minByteOffset = INT_MAX;
7275     int32 maxByteOffset = 0;
7276 
7277     int32 skipIndex = -1;
7278     MIRSymbol *skipSym = nullptr;
7279     size_t refSymNum = 0;
7280     if (cleanupNode->GetIntrinsic() == INTRN_MPL_CLEANUP_LOCALREFVARS) {
7281         refSymNum = cleanupNode->GetNopndSize();
7282         if (refSymNum < 1) {
7283             return true;
7284         }
7285     } else if (cleanupNode->GetIntrinsic() == INTRN_MPL_CLEANUP_LOCALREFVARS_SKIP) {
7286         refSymNum = cleanupNode->GetNopndSize();
7287         /* refSymNum == 0, no local refvars; refSymNum == 1 and cleanup skip, so nothing to do */
7288         if (refSymNum <= 1) {
7289             return true;
7290         }
7291         BaseNode *skipExpr = cleanupNode->Opnd(refSymNum - 1);
7292 
7293         CHECK_FATAL(skipExpr->GetOpCode() == OP_dread, "should be dread");
7294         DreadNode *refNode = static_cast<DreadNode *>(skipExpr);
7295         skipSym = GetFunction().GetLocalOrGlobalSymbol(refNode->GetStIdx());
7296 
7297         refSymNum -= 1;
7298     } else if (cleanupNode->GetIntrinsic() == INTRN_MPL_CLEANUP_NORETESCOBJS) {
7299         refSymNum = cleanupNode->GetNopndSize();
7300         /* the number of operands of intrinsic call INTRN_MPL_CLEANUP_NORETESCOBJS must be more than 1 */
7301         if (refSymNum <= 1) {
7302             return true;
7303         }
7304         BaseNode *skipexpr = cleanupNode->Opnd(0);
7305         CHECK_FATAL(skipexpr->GetOpCode() == OP_dread, "should be dread");
7306         DreadNode *refnode = static_cast<DreadNode *>(skipexpr);
7307         skipSym = GetFunction().GetLocalOrGlobalSymbol(refnode->GetStIdx());
7308     }
7309 
7310     /* now compute the offset range */
7311     std::vector<int32> offsets;
7312     AArch64MemLayout *memLayout = static_cast<AArch64MemLayout *>(this->GetMemlayout());
7313     for (size_t i = 0; i < refSymNum; ++i) {
7314         BaseNode *argExpr = cleanupNode->Opnd(i);
7315         CHECK_FATAL(argExpr->GetOpCode() == OP_dread, "should be dread");
7316         DreadNode *refNode = static_cast<DreadNode *>(argExpr);
7317         MIRSymbol *refSymbol = GetFunction().GetLocalOrGlobalSymbol(refNode->GetStIdx());
7318         if (memLayout->GetSymAllocTable().size() <= refSymbol->GetStIndex()) {
7319             ERR(kLncErr, "access memLayout->GetSymAllocTable() failed");
7320             return false;
7321         }
7322         AArch64SymbolAlloc *symLoc =
7323             static_cast<AArch64SymbolAlloc *>(memLayout->GetSymAllocInfo(refSymbol->GetStIndex()));
7324         int32 tempOffset = GetBaseOffset(*symLoc);
7325         offsets.emplace_back(tempOffset);
7326 #ifdef CC_DEBUG_INFO
7327         LogInfo::MapleLogger() << "refsym " << refSymbol->GetName() << " offset " << tempOffset << '\n';
7328 #endif
7329         minByteOffset = (minByteOffset > tempOffset) ? tempOffset : minByteOffset;
7330         maxByteOffset = (maxByteOffset < tempOffset) ? tempOffset : maxByteOffset;
7331     }
7332 
7333     /* get the skip offset */
7334     int32 skipOffset = -1;
7335     if (skipSym != nullptr) {
7336         AArch64SymbolAlloc *symLoc =
7337             static_cast<AArch64SymbolAlloc *>(memLayout->GetSymAllocInfo(skipSym->GetStIndex()));
7338         CHECK_FATAL(GetBaseOffset(*symLoc) < std::numeric_limits<int32>::max(), "out of range");
7339         skipOffset = GetBaseOffset(*symLoc);
7340         offsets.emplace_back(skipOffset);
7341 
7342 #ifdef CC_DEBUG_INFO
7343         LogInfo::MapleLogger() << "skip " << skipSym->GetName() << " offset " << skipOffset << '\n';
7344 #endif
7345 
7346         skipIndex = symLoc->GetOffset() / kOffsetAlign;
7347     }
7348 
7349     /* call runtime cleanup */
7350     if (minByteOffset < INT_MAX) {
7351         int32 refLocBase = memLayout->GetRefLocBaseLoc();
7352         uint32 refNum = memLayout->GetSizeOfRefLocals() / kOffsetAlign;
7353         CHECK_FATAL((refLocBase + (refNum - 1) * kIntregBytelen) < std::numeric_limits<int32>::max(), "out of range");
7354         int32 refLocEnd = refLocBase + (refNum - 1) * kIntregBytelen;
7355         int32 realMin = minByteOffset < refLocBase ? refLocBase : minByteOffset;
7356         int32 realMax = maxByteOffset > refLocEnd ? refLocEnd : maxByteOffset;
7357         if (forEA) {
7358             std::sort(offsets.begin(), offsets.end());
7359             int32 prev = offsets[0];
7360             for (size_t i = 1; i < offsets.size(); i++) {
7361                 CHECK_FATAL((offsets[i] == prev) || ((offsets[i] - prev) == kIntregBytelen), "must be");
7362                 prev = offsets[i];
7363             }
7364             CHECK_FATAL((refLocBase - prev) == kIntregBytelen, "must be");
7365             realMin = minByteOffset;
7366             realMax = maxByteOffset;
7367         }
7368 #ifdef CC_DEBUG_INFO
7369         LogInfo::MapleLogger() << " realMin " << realMin << " realMax " << realMax << '\n';
7370 #endif
7371         if (realMax < realMin) {
7372             /* maybe there is a cleanup intrinsic bug, use CHECK_FATAL instead? */
7373             CHECK_FATAL(false, "must be");
7374         }
7375 
7376         /* optimization for little slot cleanup */
7377         if (realMax == realMin && !forEA) {
7378             RegOperand &phyOpnd = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7379             Operand &stackLoc = CreateStkTopOpnd(static_cast<uint32>(realMin), GetPointerSize() * kBitsPerByte);
7380             Insn &ldrInsn = GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_a64), phyOpnd, stackLoc);
7381             GetCurBB()->AppendInsn(ldrInsn);
7382 
7383             ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
7384             srcOpnds->PushOpnd(phyOpnd);
7385             MIRSymbol *callSym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
7386             std::string funcName("MCC_DecRef_NaiveRCFast");
7387             callSym->SetNameStrIdx(funcName);
7388             callSym->SetStorageClass(kScText);
7389             callSym->SetSKind(kStFunc);
7390             Insn &callInsn = AppendCall(*callSym, *srcOpnds);
7391             callInsn.SetRefSkipIdx(skipIndex);
7392             GetCurBB()->SetHasCall();
7393             /* because of return stmt is often the last stmt */
7394             GetCurBB()->SetFrequency(frequency);
7395 
7396             return true;
7397         }
7398         ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
7399 
7400         ImmOperand &beginOpnd = CreateImmOperand(realMin, k64BitSize, true);
7401         regno_t vRegNO0 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
7402         RegOperand &vReg0 = CreateVirtualRegisterOperand(vRegNO0);
7403         RegOperand &fpOpnd = GetOrCreateStackBaseRegOperand();
7404         SelectAdd(vReg0, fpOpnd, beginOpnd, PTY_i64);
7405 
7406         RegOperand &parmRegOpnd1 = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7407         srcOpnds->PushOpnd(parmRegOpnd1);
7408         SelectCopy(parmRegOpnd1, PTY_a64, vReg0, PTY_a64);
7409 
7410         uint32 realRefNum = (realMax - realMin) / kOffsetAlign + 1;
7411 
7412         ImmOperand &countOpnd = CreateImmOperand(realRefNum, k64BitSize, true);
7413 
7414         RegOperand &parmRegOpnd2 = GetOrCreatePhysicalRegisterOperand(R1, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7415         srcOpnds->PushOpnd(parmRegOpnd2);
7416         SelectCopyImm(parmRegOpnd2, countOpnd, PTY_i64);
7417 
7418         MIRSymbol *funcSym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
7419         if ((skipSym != nullptr) && (skipOffset >= realMin) && (skipOffset <= realMax)) {
7420             /* call cleanupskip */
7421             uint32 stOffset = (skipOffset - realMin) / kOffsetAlign;
7422             ImmOperand &retLoc = CreateImmOperand(stOffset, k64BitSize, true);
7423 
7424             RegOperand &parmRegOpnd3 = GetOrCreatePhysicalRegisterOperand(R2, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7425             srcOpnds->PushOpnd(parmRegOpnd3);
7426             SelectCopyImm(parmRegOpnd3, retLoc, PTY_i64);
7427 
7428             std::string funcName;
7429             if (forEA) {
7430                 funcName = "MCC_CleanupNonRetEscObj";
7431             } else {
7432                 funcName = "MCC_CleanupLocalStackRefSkip_NaiveRCFast";
7433             }
7434             funcSym->SetNameStrIdx(funcName);
7435 #ifdef CC_DEBUG_INFO
7436             LogInfo::MapleLogger() << "num " << real_ref_num << " skip loc " << stOffset << '\n';
7437 #endif
7438         } else {
7439             /* call cleanup */
7440             CHECK_FATAL(!forEA, "must be");
7441             std::string funcName("MCC_CleanupLocalStackRef_NaiveRCFast");
7442             funcSym->SetNameStrIdx(funcName);
7443 #ifdef CC_DEBUG_INFO
7444             LogInfo::MapleLogger() << "num " << real_ref_num << '\n';
7445 #endif
7446         }
7447 
7448         funcSym->SetStorageClass(kScText);
7449         funcSym->SetSKind(kStFunc);
7450         Insn &callInsn = AppendCall(*funcSym, *srcOpnds);
7451         callInsn.SetRefSkipIdx(skipIndex);
7452         GetCurBB()->SetHasCall();
7453         GetCurBB()->SetFrequency(frequency);
7454     }
7455     return true;
7456 }
7457 
CreateVirtualRegisterOperand(regno_t vRegNO,uint32 size,RegType kind,uint32 flg) const7458 RegOperand *AArch64CGFunc::CreateVirtualRegisterOperand(regno_t vRegNO, uint32 size, RegType kind, uint32 flg) const
7459 {
7460     RegOperand *res = memPool->New<RegOperand>(vRegNO, size, kind, flg);
7461     return res;
7462 }
7463 
CreateVirtualRegisterOperand(regno_t vRegNO)7464 RegOperand &AArch64CGFunc::CreateVirtualRegisterOperand(regno_t vRegNO)
7465 {
7466     DEBUG_ASSERT((vRegOperandTable.find(vRegNO) == vRegOperandTable.end()), "already exist");
7467     DEBUG_ASSERT(vRegNO < vRegTable.size(), "index out of range");
7468     uint8 bitSize = static_cast<uint8>((static_cast<uint32>(vRegTable[vRegNO].GetSize())) * kBitsPerByte);
7469     RegOperand *res = CreateVirtualRegisterOperand(vRegNO, bitSize, vRegTable.at(vRegNO).GetType());
7470     vRegOperandTable[vRegNO] = res;
7471     return *res;
7472 }
7473 
GetOrCreateVirtualRegisterOperand(regno_t vRegNO)7474 RegOperand &AArch64CGFunc::GetOrCreateVirtualRegisterOperand(regno_t vRegNO)
7475 {
7476     auto it = vRegOperandTable.find(vRegNO);
7477     return (it != vRegOperandTable.end()) ? *(it->second) : CreateVirtualRegisterOperand(vRegNO);
7478 }
7479 
GetOrCreateVirtualRegisterOperand(RegOperand & regOpnd)7480 RegOperand &AArch64CGFunc::GetOrCreateVirtualRegisterOperand(RegOperand &regOpnd)
7481 {
7482     regno_t regNO = regOpnd.GetRegisterNumber();
7483     auto it = vRegOperandTable.find(regNO);
7484     if (it != vRegOperandTable.end()) {
7485         it->second->SetSize(regOpnd.GetSize());
7486         it->second->SetRegisterNumber(regNO);
7487         it->second->SetRegisterType(regOpnd.GetRegisterType());
7488         it->second->SetValidBitsNum(regOpnd.GetValidBitsNum());
7489         return *it->second;
7490     } else {
7491         auto *newRegOpnd = static_cast<RegOperand *>(regOpnd.Clone(*memPool));
7492         regno_t newRegNO = newRegOpnd->GetRegisterNumber();
7493         if (newRegNO >= maxRegCount) {
7494             maxRegCount = newRegNO + kRegIncrStepLen;
7495             vRegTable.resize(maxRegCount);
7496         }
7497         vRegOperandTable[newRegNO] = newRegOpnd;
7498         VirtualRegNode *vregNode = memPool->New<VirtualRegNode>(newRegOpnd->GetRegisterType(), newRegOpnd->GetSize());
7499         vRegTable[newRegNO] = *vregNode;
7500         vRegCount = maxRegCount;
7501         return *newRegOpnd;
7502     }
7503 }
7504 
7505 /*
7506  * Traverse all call insn to determine return type of it
7507  * If the following insn is mov/str/blr and use R0/V0, it means the call insn have reture value
7508  */
DetermineReturnTypeofCall()7509 void AArch64CGFunc::DetermineReturnTypeofCall()
7510 {
7511     FOR_ALL_BB(bb, this) {
7512         if (bb->IsUnreachable() || !bb->HasCall()) {
7513             continue;
7514         }
7515         FOR_BB_INSNS(insn, bb) {
7516             if (!insn->IsTargetInsn()) {
7517                 continue;
7518             }
7519             if (!insn->IsCall() || insn->GetMachineOpcode() == MOP_asm) {
7520                 continue;
7521             }
7522             Insn *nextInsn = insn->GetNextMachineInsn();
7523             if (nextInsn == nullptr) {
7524                 continue;
7525             }
7526             if ((nextInsn->GetMachineOpcode() != MOP_asm) &&
7527                 ((nextInsn->IsMove() && nextInsn->GetOperand(kInsnSecondOpnd).IsRegister()) || nextInsn->IsStore() ||
7528                  (nextInsn->IsCall() && nextInsn->GetOperand(kInsnFirstOpnd).IsRegister()))) {
7529                 auto *srcOpnd = static_cast<RegOperand *>(&nextInsn->GetOperand(kInsnFirstOpnd));
7530                 CHECK_FATAL(srcOpnd != nullptr, "nullptr");
7531                 if (!srcOpnd->IsPhysicalRegister()) {
7532                     continue;
7533                 }
7534                 if (srcOpnd->GetRegisterNumber() == R0) {
7535                     insn->SetRetType(Insn::kRegInt);
7536                     continue;
7537                 }
7538                 if (srcOpnd->GetRegisterNumber() == V0) {
7539                     insn->SetRetType(Insn::kRegFloat);
7540                 }
7541             }
7542         }
7543     }
7544 }
7545 
HandleRCCall(bool begin,const MIRSymbol * retRef)7546 void AArch64CGFunc::HandleRCCall(bool begin, const MIRSymbol *retRef)
7547 {
7548     if (!GetCG()->GenLocalRC() && !begin) {
7549         /* handle local rc is disabled. */
7550         return;
7551     }
7552 
7553     AArch64MemLayout *memLayout = static_cast<AArch64MemLayout *>(this->GetMemlayout());
7554     int32 refNum = static_cast<int32>(memLayout->GetSizeOfRefLocals() / kOffsetAlign);
7555     if (!refNum) {
7556         if (begin) {
7557             GenerateYieldpoint(*GetCurBB());
7558             yieldPointInsn = GetCurBB()->GetLastInsn();
7559         }
7560         return;
7561     }
7562 
7563     /* no MCC_CleanupLocalStackRefSkip when ret_ref is the only ref symbol */
7564     if ((refNum == 1) && (retRef != nullptr)) {
7565         if (begin) {
7566             GenerateYieldpoint(*GetCurBB());
7567             yieldPointInsn = GetCurBB()->GetLastInsn();
7568         }
7569         return;
7570     }
7571     CHECK_FATAL(refNum < 0xFFFF, "not enough room for size.");
7572     int32 refLocBase = memLayout->GetRefLocBaseLoc();
7573     CHECK_FATAL((refLocBase >= 0) && (refLocBase < 0xFFFF), "not enough room for offset.");
7574     int32 formalRef = 0;
7575     /* avoid store zero to formal localrefvars. */
7576     if (begin) {
7577         for (uint32 i = 0; i < GetFunction().GetFormalCount(); ++i) {
7578             if (GetFunction().GetNthParamAttr(i).GetAttr(ATTR_localrefvar)) {
7579                 refNum--;
7580                 formalRef++;
7581             }
7582         }
7583     }
7584     /*
7585      * if the number of local refvar is less than 12, use stp or str to init local refvar
7586      * else call function MCC_InitializeLocalStackRef to init.
7587      */
7588     if (begin && (refNum <= kRefNum12) && ((refLocBase + kIntregBytelen * (refNum - 1)) < kStpLdpImm64UpperBound)) {
7589         int32 pairNum = refNum / kDivide2;
7590         int32 singleNum = refNum % kDivide2;
7591         const int32 pairRefBytes = 16; /* the size of each pair of ref is 16 bytes */
7592         int32 ind = 0;
7593         while (ind < pairNum) {
7594             int32 offset = memLayout->GetRefLocBaseLoc() + kIntregBytelen * formalRef + pairRefBytes * ind;
7595             Operand &zeroOp = GetZeroOpnd(k64BitSize);
7596             Operand &stackLoc = CreateStkTopOpnd(static_cast<uint32>(offset), GetPointerSize() * kBitsPerByte);
7597             Insn &setInc = GetInsnBuilder()->BuildInsn(MOP_xstp, zeroOp, zeroOp, stackLoc);
7598             GetCurBB()->AppendInsn(setInc);
7599             ind++;
7600         }
7601         if (singleNum > 0) {
7602             int32 offset = memLayout->GetRefLocBaseLoc() + kIntregBytelen * formalRef + kIntregBytelen * (refNum - 1);
7603             Operand &zeroOp = GetZeroOpnd(k64BitSize);
7604             Operand &stackLoc = CreateStkTopOpnd(static_cast<uint32>(offset), GetPointerSize() * kBitsPerByte);
7605             Insn &setInc = GetInsnBuilder()->BuildInsn(MOP_xstr, zeroOp, stackLoc);
7606             GetCurBB()->AppendInsn(setInc);
7607         }
7608         /* Insert Yield Point just after localrefvar are initialized. */
7609         GenerateYieldpoint(*GetCurBB());
7610         yieldPointInsn = GetCurBB()->GetLastInsn();
7611         return;
7612     }
7613 
7614     /* refNum is 1 and refvar is not returned, this refvar need to call MCC_DecRef_NaiveRCFast. */
7615     if ((refNum == 1) && !begin && (retRef == nullptr)) {
7616         RegOperand &phyOpnd = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7617         Operand &stackLoc =
7618             CreateStkTopOpnd(static_cast<uint32>(memLayout->GetRefLocBaseLoc()), GetPointerSize() * kBitsPerByte);
7619         Insn &ldrInsn = GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_a64), phyOpnd, stackLoc);
7620         GetCurBB()->AppendInsn(ldrInsn);
7621 
7622         ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
7623         srcOpnds->PushOpnd(phyOpnd);
7624         MIRSymbol *callSym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
7625         std::string funcName("MCC_DecRef_NaiveRCFast");
7626         callSym->SetNameStrIdx(funcName);
7627         callSym->SetStorageClass(kScText);
7628         callSym->SetSKind(kStFunc);
7629 
7630         AppendCall(*callSym, *srcOpnds);
7631         GetCurBB()->SetHasCall();
7632         if (frequency != 0) {
7633             GetCurBB()->SetFrequency(frequency);
7634         }
7635         return;
7636     }
7637 
7638     /* refNum is 2 and one of refvar is returned, only another one is needed to call MCC_DecRef_NaiveRCFast. */
7639     if ((refNum == 2) && !begin && retRef != nullptr) {
7640         AArch64SymbolAlloc *symLoc =
7641             static_cast<AArch64SymbolAlloc *>(memLayout->GetSymAllocInfo(retRef->GetStIndex()));
7642         int32 stOffset = symLoc->GetOffset() / kOffsetAlign;
7643         RegOperand &phyOpnd = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7644         Operand *stackLoc = nullptr;
7645         if (stOffset == 0) {
7646             /* just have to Dec the next one. */
7647             stackLoc = &CreateStkTopOpnd(static_cast<uint32>(memLayout->GetRefLocBaseLoc()) + kIntregBytelen,
7648                                          GetPointerSize() * kBitsPerByte);
7649         } else {
7650             /* just have to Dec the current one. */
7651             stackLoc =
7652                 &CreateStkTopOpnd(static_cast<uint32>(memLayout->GetRefLocBaseLoc()), GetPointerSize() * kBitsPerByte);
7653         }
7654         Insn &ldrInsn = GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_a64), phyOpnd, *stackLoc);
7655         GetCurBB()->AppendInsn(ldrInsn);
7656 
7657         ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
7658         srcOpnds->PushOpnd(phyOpnd);
7659         MIRSymbol *callSym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
7660         std::string funcName("MCC_DecRef_NaiveRCFast");
7661         callSym->SetNameStrIdx(funcName);
7662         callSym->SetStorageClass(kScText);
7663         callSym->SetSKind(kStFunc);
7664         Insn &callInsn = AppendCall(*callSym, *srcOpnds);
7665         callInsn.SetRefSkipIdx(stOffset);
7666         GetCurBB()->SetHasCall();
7667         if (frequency != 0) {
7668             GetCurBB()->SetFrequency(frequency);
7669         }
7670         return;
7671     }
7672 
7673     bool needSkip = false;
7674     ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
7675 
7676     ImmOperand *beginOpnd =
7677         &CreateImmOperand(memLayout->GetRefLocBaseLoc() + kIntregBytelen * formalRef, k64BitSize, true);
7678     ImmOperand *countOpnd = &CreateImmOperand(refNum, k64BitSize, true);
7679     int32 refSkipIndex = -1;
7680     if (!begin && retRef != nullptr) {
7681         AArch64SymbolAlloc *symLoc =
7682             static_cast<AArch64SymbolAlloc *>(memLayout->GetSymAllocInfo(retRef->GetStIndex()));
7683         int32 stOffset = symLoc->GetOffset() / kOffsetAlign;
7684         refSkipIndex = stOffset;
7685         if (stOffset == 0) {
7686             /* ret_ref at begin. */
7687             beginOpnd = &CreateImmOperand(memLayout->GetRefLocBaseLoc() + kIntregBytelen, k64BitSize, true);
7688             countOpnd = &CreateImmOperand(refNum - 1, k64BitSize, true);
7689         } else if (stOffset == (refNum - 1)) {
7690             /* ret_ref at end. */
7691             countOpnd = &CreateImmOperand(refNum - 1, k64BitSize, true);
7692         } else {
7693             needSkip = true;
7694         }
7695     }
7696 
7697     regno_t vRegNO0 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
7698     RegOperand &vReg0 = CreateVirtualRegisterOperand(vRegNO0);
7699     RegOperand &fpOpnd = GetOrCreateStackBaseRegOperand();
7700     SelectAdd(vReg0, fpOpnd, *beginOpnd, PTY_i64);
7701 
7702     RegOperand &parmRegOpnd1 = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7703     srcOpnds->PushOpnd(parmRegOpnd1);
7704     SelectCopy(parmRegOpnd1, PTY_a64, vReg0, PTY_a64);
7705 
7706     regno_t vRegNO1 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
7707     RegOperand &vReg1 = CreateVirtualRegisterOperand(vRegNO1);
7708     SelectCopyImm(vReg1, *countOpnd, PTY_i64);
7709 
7710     RegOperand &parmRegOpnd2 = GetOrCreatePhysicalRegisterOperand(R1, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7711     srcOpnds->PushOpnd(parmRegOpnd2);
7712     SelectCopy(parmRegOpnd2, PTY_a64, vReg1, PTY_a64);
7713 
7714     MIRSymbol *sym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
7715     if (begin) {
7716         std::string funcName("MCC_InitializeLocalStackRef");
7717         sym->SetNameStrIdx(funcName);
7718         CHECK_FATAL(countOpnd->GetValue() > 0, "refCount should be greater than 0.");
7719         refCount = static_cast<uint32>(countOpnd->GetValue());
7720         beginOffset = beginOpnd->GetValue();
7721     } else if (!needSkip) {
7722         std::string funcName("MCC_CleanupLocalStackRef_NaiveRCFast");
7723         sym->SetNameStrIdx(funcName);
7724     } else {
7725         CHECK_NULL_FATAL(retRef);
7726         if (retRef->GetStIndex() >= memLayout->GetSymAllocTable().size()) {
7727             CHECK_FATAL(false, "index out of range in AArch64CGFunc::HandleRCCall");
7728         }
7729         AArch64SymbolAlloc *symLoc =
7730             static_cast<AArch64SymbolAlloc *>(memLayout->GetSymAllocInfo(retRef->GetStIndex()));
7731         int32 stOffset = symLoc->GetOffset() / kOffsetAlign;
7732         ImmOperand &retLoc = CreateImmOperand(stOffset, k64BitSize, true);
7733 
7734         regno_t vRegNO2 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
7735         RegOperand &vReg2 = CreateVirtualRegisterOperand(vRegNO2);
7736         SelectCopyImm(vReg2, retLoc, PTY_i64);
7737 
7738         RegOperand &parmRegOpnd3 = GetOrCreatePhysicalRegisterOperand(R2, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7739         srcOpnds->PushOpnd(parmRegOpnd3);
7740         SelectCopy(parmRegOpnd3, PTY_a64, vReg2, PTY_a64);
7741 
7742         std::string funcName("MCC_CleanupLocalStackRefSkip_NaiveRCFast");
7743         sym->SetNameStrIdx(funcName);
7744     }
7745     sym->SetStorageClass(kScText);
7746     sym->SetSKind(kStFunc);
7747 
7748     Insn &callInsn = AppendCall(*sym, *srcOpnds);
7749     callInsn.SetRefSkipIdx(refSkipIndex);
7750     if (frequency != 0) {
7751         GetCurBB()->SetFrequency(frequency);
7752     }
7753     GetCurBB()->SetHasCall();
7754     if (begin) {
7755         /* Insert Yield Point just after localrefvar are initialized. */
7756         GenerateYieldpoint(*GetCurBB());
7757         yieldPointInsn = GetCurBB()->GetLastInsn();
7758     }
7759 }
7760 
SelectParmListDreadSmallAggregate(const MIRSymbol & sym,MIRType & structType,ListOperand & srcOpnds,int32 offset,AArch64CallConvImpl & parmLocator,FieldID fieldID)7761 void AArch64CGFunc::SelectParmListDreadSmallAggregate(const MIRSymbol &sym, MIRType &structType, ListOperand &srcOpnds,
7762                                                       int32 offset, AArch64CallConvImpl &parmLocator, FieldID fieldID)
7763 {
7764     /*
7765      * in two param regs if possible
7766      * If struct is <= 8 bytes, then it fits into one param reg.
7767      * If struct is <= 16 bytes, then it fits into two param regs.
7768      * Otherwise, it goes onto the stack.
7769      * If the number of available param reg is less than what is
7770      * needed to fit the entire struct into them, then the param
7771      * reg is skipped and the struct goes onto the stack.
7772      * Example 1.
7773      *  struct size == 8 bytes.
7774      *  param regs x0 to x6 are used.
7775      *  struct is passed in x7.
7776      * Example 2.
7777      *  struct is 16 bytes.
7778      *  param regs x0 to x5 are used.
7779      *  struct is passed in x6 and x7.
7780      * Example 3.
7781      *  struct is 16 bytes.
7782      *  param regs x0 to x6 are used.  x7 alone is not enough to pass the struct.
7783      *  struct is passed on the stack.
7784      *  x7 is not used, as the following param will go onto the stack also.
7785      */
7786     int32 symSize = GetBecommon().GetTypeSize(structType.GetTypeIndex().GetIdx());
7787     CCLocInfo ploc;
7788     CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
7789     parmLocator.LocateNextParm(structType, ploc);
7790     if (ploc.reg0 == 0) {
7791         /* No param regs available, pass on stack. */
7792         /* If symSize is <= 8 bytes then use 1 reg, else 2 */
7793         CreateCallStructParamPassByStack(symSize, &sym, nullptr, ploc.memOffset);
7794     } else {
7795         /* pass by param regs. */
7796         RegOperand *parmOpnd0 = SelectParmListDreadAccessField(sym, fieldID, ploc, offset, 0);
7797         srcOpnds.PushOpnd(*parmOpnd0);
7798         if (ploc.reg1) {
7799             RegOperand *parmOpnd1 = SelectParmListDreadAccessField(sym, fieldID, ploc, offset, 1);
7800             srcOpnds.PushOpnd(*parmOpnd1);
7801         }
7802         if (ploc.reg2) {
7803             RegOperand *parmOpnd2 = SelectParmListDreadAccessField(sym, fieldID, ploc, offset, 2);
7804             srcOpnds.PushOpnd(*parmOpnd2);
7805         }
7806         if (ploc.reg3) {
7807             RegOperand *parmOpnd3 = SelectParmListDreadAccessField(sym, fieldID, ploc, offset, 3);
7808             srcOpnds.PushOpnd(*parmOpnd3);
7809         }
7810     }
7811 }
7812 
SelectParmListIreadSmallAggregate(const IreadNode & iread,MIRType & structType,ListOperand & srcOpnds,int32 offset,AArch64CallConvImpl & parmLocator)7813 void AArch64CGFunc::SelectParmListIreadSmallAggregate(const IreadNode &iread, MIRType &structType,
7814                                                       ListOperand &srcOpnds, int32 offset,
7815                                                       AArch64CallConvImpl &parmLocator)
7816 {
7817     int32 symSize = GetBecommon().GetTypeSize(structType.GetTypeIndex().GetIdx());
7818     RegOperand *addrOpnd0 = static_cast<RegOperand *>(HandleExpr(iread, *(iread.Opnd(0))));
7819     RegOperand *addrOpnd1 = &LoadIntoRegister(*addrOpnd0, iread.Opnd(0)->GetPrimType());
7820     CCLocInfo ploc;
7821     CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
7822     parmLocator.LocateNextParm(structType, ploc);
7823     if (ploc.reg0 == 0) {
7824         /* No param regs available, pass on stack. */
7825         CreateCallStructParamPassByStack(symSize, nullptr, addrOpnd1, ploc.memOffset);
7826     } else {
7827         /* pass by param regs. */
7828         fpParamState state = kStateUnknown;
7829         uint32 memSize = 0;
7830         switch (ploc.fpSize) {
7831             case k0BitSize:
7832                 state = kNotFp;
7833                 memSize = k64BitSize;
7834                 break;
7835             case k4BitSize:
7836                 state = kFp32Bit;
7837                 memSize = k32BitSize;
7838                 break;
7839             case k8BitSize:
7840                 state = kFp64Bit;
7841                 memSize = k64BitSize;
7842                 break;
7843             default:
7844                 break;
7845         }
7846         OfstOperand *offOpnd0 = &GetOrCreateOfstOpnd(static_cast<uint64>(static_cast<int64>(offset)), k32BitSize);
7847         MemOperand *mopnd =
7848             &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, memSize, addrOpnd1, nullptr, offOpnd0, nullptr);
7849         CreateCallStructParamPassByReg(ploc.reg0, *mopnd, srcOpnds, state);
7850         if (ploc.reg1) {
7851             OfstOperand *offOpnd1 = &GetOrCreateOfstOpnd(
7852                 ((ploc.fpSize ? ploc.fpSize : GetPointerSize()) + static_cast<uint32>(offset)), k32BitSize);
7853             mopnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, memSize, addrOpnd1, nullptr, offOpnd1, nullptr);
7854             CreateCallStructParamPassByReg(ploc.reg1, *mopnd, srcOpnds, state);
7855         }
7856         if (ploc.reg2) {
7857             OfstOperand *offOpnd2 = &GetOrCreateOfstOpnd(
7858                 ((ploc.fpSize ? (ploc.fpSize * k4BitShift) : GetPointerSize()) + static_cast<uint32>(offset)),
7859                 k32BitSize);
7860             mopnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, memSize, addrOpnd1, nullptr, offOpnd2, nullptr);
7861             CreateCallStructParamPassByReg(ploc.reg2, *mopnd, srcOpnds, state);
7862         }
7863         if (ploc.reg3) {
7864             OfstOperand *offOpnd3 = &GetOrCreateOfstOpnd(
7865                 ((ploc.fpSize ? (ploc.fpSize * k8BitShift) : GetPointerSize()) + static_cast<uint32>(offset)),
7866                 k32BitSize);
7867             mopnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, memSize, addrOpnd1, nullptr, offOpnd3, nullptr);
7868             CreateCallStructParamPassByReg(ploc.reg3, *mopnd, srcOpnds, state);
7869         }
7870     }
7871 }
7872 
SelectParmListDreadLargeAggregate(const MIRSymbol & sym,MIRType & structType,ListOperand & srcOpnds,AArch64CallConvImpl & parmLocator,int32 & structCopyOffset,int32 fromOffset)7873 void AArch64CGFunc::SelectParmListDreadLargeAggregate(const MIRSymbol &sym, MIRType &structType, ListOperand &srcOpnds,
7874                                                       AArch64CallConvImpl &parmLocator, int32 &structCopyOffset,
7875                                                       int32 fromOffset)
7876 {
7877     /*
7878      * Pass larger sized struct on stack.
7879      * Need to copy the entire structure onto the stack.
7880      * The pointer to the starting address of the copied struct is then
7881      * used as the parameter for the struct.
7882      * This pointer is passed as the next parameter.
7883      * Example 1:
7884      * struct is 23 bytes.
7885      * param regs x0 to x5 are used.
7886      * First around up 23 to 24, so 3 of 8-byte slots.
7887      * Copy struct to a created space on the stack.
7888      * Pointer of copied struct is passed in x6.
7889      * Example 2:
7890      * struct is 25 bytes.
7891      * param regs x0 to x7 are used.
7892      * First around up 25 to 32, so 4 of 8-byte slots.
7893      * Copy struct to a created space on the stack.
7894      * Pointer of copied struct is passed on stack as the 9th parameter.
7895      */
7896     uint64 symSize = GetBecommon().GetTypeSize(structType.GetTypeIndex().GetIdx());
7897     CCLocInfo ploc;
7898     CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
7899     parmLocator.LocateNextParm(structType, ploc);
7900     uint32 numMemOp = static_cast<uint32>(RoundUp(symSize, GetPointerSize()) / GetPointerSize()); /* round up */
7901     /* Create the struct copies. */
7902     RegOperand *parmOpnd =
7903         CreateCallStructParamCopyToStack(numMemOp, &sym, nullptr, structCopyOffset, fromOffset, ploc);
7904     if (parmOpnd) {
7905         srcOpnds.PushOpnd(*parmOpnd);
7906     }
7907     structCopyOffset += static_cast<int32>(numMemOp * GetPointerSize());
7908 }
7909 
SelectParmListIreadLargeAggregate(const IreadNode & iread,MIRType & structType,ListOperand & srcOpnds,AArch64CallConvImpl & parmLocator,int32 & structCopyOffset,int32 fromOffset)7910 void AArch64CGFunc::SelectParmListIreadLargeAggregate(const IreadNode &iread, MIRType &structType,
7911                                                       ListOperand &srcOpnds, AArch64CallConvImpl &parmLocator,
7912                                                       int32 &structCopyOffset, int32 fromOffset)
7913 {
7914     uint64 symSize = GetBecommon().GetTypeSize(structType.GetTypeIndex().GetIdx());
7915     RegOperand *addrOpnd0 = static_cast<RegOperand *>(HandleExpr(iread, *(iread.Opnd(0))));
7916     RegOperand *addrOpnd1 = &LoadIntoRegister(*addrOpnd0, iread.Opnd(0)->GetPrimType());
7917     CCLocInfo ploc;
7918     CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
7919     parmLocator.LocateNextParm(structType, ploc);
7920     uint32 numMemOp = static_cast<uint32>(RoundUp(symSize, GetPointerSize()) / GetPointerSize()); /* round up */
7921     RegOperand *parmOpnd =
7922         CreateCallStructParamCopyToStack(numMemOp, nullptr, addrOpnd1, structCopyOffset, fromOffset, ploc);
7923     structCopyOffset += static_cast<int32>(numMemOp * GetPointerSize());
7924     if (parmOpnd) {
7925         srcOpnds.PushOpnd(*parmOpnd);
7926     }
7927 }
7928 
CreateCallStructParamPassByStack(int32 symSize,const MIRSymbol * sym,RegOperand * addrOpnd,int32 baseOffset)7929 void AArch64CGFunc::CreateCallStructParamPassByStack(int32 symSize, const MIRSymbol *sym, RegOperand *addrOpnd,
7930                                                      int32 baseOffset)
7931 {
7932     MemOperand *ldMopnd = nullptr;
7933     MemOperand *stMopnd = nullptr;
7934     uint32 numRegNeeded = (static_cast<uint32>(symSize) <= k8ByteSize) ? kOneRegister : kTwoRegister;
7935     for (int j = 0; j < static_cast<int>(numRegNeeded); j++) {
7936         if (sym) {
7937             if (CGOptions::IsArm64ilp32()) {
7938                 ldMopnd = &GetOrCreateMemOpnd(*sym, (j * static_cast<int>(k8ByteSize)), k64BitSize);
7939             } else {
7940                 ldMopnd = &GetOrCreateMemOpnd(*sym, (j * static_cast<int>(GetPointerSize())), k64BitSize);
7941             }
7942         } else {
7943             if (CGOptions::IsArm64ilp32()) {
7944                 ldMopnd =
7945                     &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, addrOpnd, nullptr,
7946                                         &GetOrCreateOfstOpnd(static_cast<uint32>(j) * k8ByteSize, k32BitSize), nullptr);
7947             } else {
7948                 ldMopnd = &GetOrCreateMemOpnd(
7949                     MemOperand::kAddrModeBOi, k64BitSize, addrOpnd, nullptr,
7950                     &GetOrCreateOfstOpnd(static_cast<uint32>(j) * GetPointerSize(), k32BitSize), nullptr);
7951             }
7952         }
7953         RegOperand *vreg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
7954         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_i64), *vreg, *ldMopnd));
7955         if (CGOptions::IsArm64ilp32()) {
7956             stMopnd =
7957                 &CreateMemOpnd(RSP, (static_cast<int64>(baseOffset) + (j * static_cast<int>(k8ByteSize))), k64BitSize);
7958         } else {
7959             stMopnd = &CreateMemOpnd(RSP, (static_cast<int64>(baseOffset) + (j * GetPointerSize())), k64BitSize);
7960         }
7961         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(k64BitSize, PTY_i64), *vreg, *stMopnd));
7962     }
7963 }
7964 
SelectParmListDreadAccessField(const MIRSymbol & sym,FieldID fieldID,const CCLocInfo & ploc,int32 offset,uint32 parmNum)7965 RegOperand *AArch64CGFunc::SelectParmListDreadAccessField(const MIRSymbol &sym, FieldID fieldID, const CCLocInfo &ploc,
7966                                                           int32 offset, uint32 parmNum)
7967 {
7968     uint32 memSize;
7969     PrimType primType;
7970     RegOperand *parmOpnd;
7971     uint32 dataSizeBits;
7972     AArch64reg reg;
7973     switch (parmNum) {
7974         case 0:
7975             reg = static_cast<AArch64reg>(ploc.reg0);
7976             break;
7977         case 1:
7978             reg = static_cast<AArch64reg>(ploc.reg1);
7979             break;
7980         case 2: // parmNum 2 use reg2
7981             reg = static_cast<AArch64reg>(ploc.reg2);
7982             break;
7983         case 3: // parmNum 3 use reg3
7984             reg = static_cast<AArch64reg>(ploc.reg3);
7985             break;
7986         default:
7987             CHECK_FATAL(false, "Exceeded maximum allowed fp parameter registers for struct passing");
7988     }
7989     if (ploc.fpSize == 0) {
7990         memSize = k64BitSize;
7991         primType = PTY_i64;
7992         dataSizeBits = GetPrimTypeSize(PTY_i64) * kBitsPerByte;
7993         parmOpnd = &GetOrCreatePhysicalRegisterOperand(reg, k64BitSize, kRegTyInt);
7994     } else if (ploc.fpSize == k4ByteSize) {
7995         memSize = k32BitSize;
7996         primType = PTY_f32;
7997         dataSizeBits = GetPrimTypeSize(PTY_f32) * kBitsPerByte;
7998         parmOpnd = &GetOrCreatePhysicalRegisterOperand(reg, k32BitSize, kRegTyFloat);
7999     } else if (ploc.fpSize == k8ByteSize) {
8000         memSize = k64BitSize;
8001         primType = PTY_f64;
8002         dataSizeBits = GetPrimTypeSize(PTY_i64) * kBitsPerByte;
8003         parmOpnd = &GetOrCreatePhysicalRegisterOperand(reg, k64BitSize, kRegTyFloat);
8004     } else {
8005         CHECK_FATAL(false, "Unknown call parameter state");
8006     }
8007     MemOperand *memOpnd;
8008     if (sym.GetStorageClass() == kScFormal && fieldID > 0) {
8009         MemOperand &baseOpnd = GetOrCreateMemOpnd(sym, 0, memSize);
8010         RegOperand &base = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8011         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_i64), base, baseOpnd));
8012         memOpnd = &CreateMemOpnd(base, (static_cast<int64>(offset) + parmNum * GetPointerSize()), memSize);
8013     } else if (ploc.fpSize) {
8014         memOpnd = &GetOrCreateMemOpnd(sym, (ploc.fpSize * parmNum + static_cast<int64>(offset)), memSize);
8015     } else {
8016         if (CGOptions::IsArm64ilp32()) {
8017             memOpnd = &GetOrCreateMemOpnd(sym, (k8ByteSize * parmNum + static_cast<int64>(offset)), memSize);
8018         } else {
8019             memOpnd = &GetOrCreateMemOpnd(sym, (GetPointerSize() * parmNum + static_cast<int64>(offset)), memSize);
8020         }
8021     }
8022     MOperator selectedMop = PickLdInsn(dataSizeBits, primType);
8023     if ((memOpnd->GetAddrMode() == MemOperand::kAddrModeBOi) &&
8024         !IsOperandImmValid(selectedMop, memOpnd, kInsnSecondOpnd)) {
8025         memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, dataSizeBits);
8026     }
8027     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(selectedMop, *parmOpnd, *memOpnd));
8028 
8029     return parmOpnd;
8030 }
8031 
CreateCallStructParamPassByReg(regno_t regno,MemOperand & memOpnd,ListOperand & srcOpnds,fpParamState state)8032 void AArch64CGFunc::CreateCallStructParamPassByReg(regno_t regno, MemOperand &memOpnd, ListOperand &srcOpnds,
8033                                                    fpParamState state)
8034 {
8035     RegOperand *parmOpnd;
8036     uint32 dataSizeBits = 0;
8037     PrimType pType = PTY_void;
8038     parmOpnd = nullptr;
8039     AArch64reg reg = static_cast<AArch64reg>(regno);
8040     if (state == kNotFp) {
8041         parmOpnd = &GetOrCreatePhysicalRegisterOperand(reg, k64BitSize, kRegTyInt);
8042         dataSizeBits = GetPrimTypeSize(PTY_i64) * kBitsPerByte;
8043         pType = PTY_i64;
8044     } else if (state == kFp32Bit) {
8045         parmOpnd = &GetOrCreatePhysicalRegisterOperand(reg, k32BitSize, kRegTyFloat);
8046         dataSizeBits = GetPrimTypeSize(PTY_f32) * kBitsPerByte;
8047         pType = PTY_f32;
8048     } else if (state == kFp64Bit) {
8049         parmOpnd = &GetOrCreatePhysicalRegisterOperand(reg, k64BitSize, kRegTyFloat);
8050         dataSizeBits = GetPrimTypeSize(PTY_f64) * kBitsPerByte;
8051         pType = PTY_f64;
8052     } else {
8053         DEBUG_ASSERT(0, "CreateCallStructParamPassByReg: Unknown state");
8054     }
8055 
8056     MOperator selectedMop = PickLdInsn(dataSizeBits, pType);
8057     if (!IsOperandImmValid(selectedMop, &memOpnd, kInsnSecondOpnd)) {
8058         memOpnd = SplitOffsetWithAddInstruction(memOpnd, dataSizeBits);
8059     }
8060     DEBUG_ASSERT(parmOpnd != nullptr, "parmOpnd should not be nullptr");
8061     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(selectedMop, *parmOpnd, memOpnd));
8062     srcOpnds.PushOpnd(*parmOpnd);
8063 }
8064 
CreateCallStructParamMemcpy(const MIRSymbol * sym,RegOperand * addropnd,uint32 structSize,int32 copyOffset,int32 fromOffset)8065 void AArch64CGFunc::CreateCallStructParamMemcpy(const MIRSymbol *sym, RegOperand *addropnd, uint32 structSize,
8066                                                 int32 copyOffset, int32 fromOffset)
8067 {
8068     std::vector<Operand *> opndVec;
8069 
8070     RegOperand *vreg1 = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8BitSize));
8071     opndVec.push_back(vreg1); /* result */
8072 
8073     RegOperand *parmOpnd = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8BitSize));
8074     RegOperand *spReg = &GetOrCreatePhysicalRegisterOperand(RSP, k64BitSize, kRegTyInt);
8075     ImmOperand *offsetOpnd0 = &CreateImmOperand(copyOffset, k64BitSize, false);
8076     SelectAdd(*parmOpnd, *spReg, *offsetOpnd0, PTY_a64);
8077     opndVec.push_back(parmOpnd); /* param 0 */
8078 
8079     if (sym != nullptr) {
8080         if (sym->GetStorageClass() == kScGlobal || sym->GetStorageClass() == kScExtern) {
8081             StImmOperand &stopnd = CreateStImmOperand(*sym, fromOffset, 0);
8082             RegOperand &staddropnd = static_cast<RegOperand &>(CreateRegisterOperandOfType(PTY_u64));
8083             SelectAddrof(staddropnd, stopnd);
8084             opndVec.push_back(&staddropnd); /* param 1 */
8085         } else if (sym->GetStorageClass() == kScAuto || sym->GetStorageClass() == kScFormal) {
8086             RegOperand *parm1Reg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8087             AArch64SymbolAlloc *symloc =
8088                 static_cast<AArch64SymbolAlloc *>(GetMemlayout()->GetSymAllocInfo(sym->GetStIndex()));
8089             RegOperand *baseOpnd = static_cast<RegOperand *>(GetBaseReg(*symloc));
8090             int32 stoffset = GetBaseOffset(*symloc);
8091             ImmOperand *offsetOpnd1 = &CreateImmOperand(static_cast<int64>(stoffset), k64BitSize, false);
8092             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *parm1Reg, *baseOpnd, *offsetOpnd1));
8093             if (sym->GetStorageClass() == kScFormal) {
8094                 MemOperand *ldmopnd =
8095                     &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, parm1Reg, nullptr,
8096                                         &GetOrCreateOfstOpnd(0, k32BitSize), static_cast<MIRSymbol *>(nullptr));
8097                 RegOperand *tmpreg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8098                 RegOperand *vreg2 = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8099                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_a64), *tmpreg, *ldmopnd));
8100                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *vreg2, *tmpreg,
8101                                                                    CreateImmOperand(fromOffset, k64BitSize, false)));
8102                 parm1Reg = vreg2;
8103             }
8104             opndVec.push_back(parm1Reg); /* param 1 */
8105         } else if (sym->GetStorageClass() == kScPstatic || sym->GetStorageClass() == kScFstatic) {
8106             CHECK_FATAL(sym->GetSKind() != kStConst, "Unsupported sym const for struct param");
8107             StImmOperand *stopnd = &CreateStImmOperand(*sym, 0, 0);
8108             RegOperand &staddropnd = static_cast<RegOperand &>(CreateRegisterOperandOfType(PTY_u64));
8109             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrp, staddropnd, *stopnd));
8110             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrpl12, staddropnd, staddropnd, *stopnd));
8111             opndVec.push_back(&staddropnd); /* param 1 */
8112         } else {
8113             CHECK_FATAL(0, "Unsupported sym for struct param");
8114         }
8115     } else {
8116         opndVec.push_back(addropnd); /* param 1 */
8117     }
8118 
8119     RegOperand &vreg3 = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8BitSize));
8120     ImmOperand &sizeOpnd = CreateImmOperand(structSize, k64BitSize, false);
8121     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wmovri32, vreg3, sizeOpnd));
8122     opndVec.push_back(&vreg3); /* param 2 */
8123 
8124     SelectLibCall("memcpy", opndVec, PTY_a64, PTY_a64);
8125 }
8126 
CreateCallStructParamCopyToStack(uint32 numMemOp,const MIRSymbol * sym,RegOperand * addrOpd,int32 copyOffset,int32 fromOffset,const CCLocInfo & ploc)8127 RegOperand *AArch64CGFunc::CreateCallStructParamCopyToStack(uint32 numMemOp, const MIRSymbol *sym, RegOperand *addrOpd,
8128                                                             int32 copyOffset, int32 fromOffset, const CCLocInfo &ploc)
8129 {
8130     /* Create the struct copies. */
8131     MemOperand *ldMopnd = nullptr;
8132     MemOperand *stMopnd = nullptr;
8133     for (uint32 j = 0; j < numMemOp; j++) {
8134         if (sym != nullptr) {
8135             if (sym->GetStorageClass() == kScFormal) {
8136                 MemOperand &base = GetOrCreateMemOpnd(*sym, 0, k64BitSize);
8137                 RegOperand &vreg = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8138                 Insn &ldInsn = GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_i64), vreg, base);
8139                 GetCurBB()->AppendInsn(ldInsn);
8140                 ldMopnd = &GetOrCreateMemOpnd(
8141                     MemOperand::kAddrModeBOi, k64BitSize, &vreg, nullptr,
8142                     &GetOrCreateOfstOpnd((j * GetPointerSize() + static_cast<uint64>(fromOffset)), k32BitSize),
8143                     nullptr);
8144             } else {
8145                 if (CGOptions::IsArm64ilp32()) {
8146                     ldMopnd =
8147                         &GetOrCreateMemOpnd(*sym, (j * GetPointerSize() + static_cast<int64>(fromOffset)), k32BitSize);
8148                 } else {
8149                     ldMopnd =
8150                         &GetOrCreateMemOpnd(*sym, (j * GetPointerSize() + static_cast<int64>(fromOffset)), k64BitSize);
8151                 }
8152             }
8153         } else {
8154             ldMopnd = &GetOrCreateMemOpnd(
8155                 MemOperand::kAddrModeBOi, k64BitSize, addrOpd, nullptr,
8156                 &GetOrCreateOfstOpnd((j * GetPointerSize() + static_cast<uint64>(fromOffset)), k32BitSize), nullptr);
8157         }
8158         if (CGOptions::IsArm64ilp32()) {
8159             RegOperand *vreg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k4ByteSize));
8160             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(k32BitSize, PTY_i32), *vreg, *ldMopnd));
8161 
8162             stMopnd = &CreateMemOpnd(RSP, (static_cast<int64>(copyOffset) + (j * GetPointerSize())), k32BitSize);
8163             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(k32BitSize, PTY_i32), *vreg, *stMopnd));
8164         } else {
8165             RegOperand *vreg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8166             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_i64), *vreg, *ldMopnd));
8167 
8168             stMopnd = &CreateMemOpnd(RSP, (static_cast<int64>(copyOffset) + (j * GetPointerSize())), k64BitSize);
8169             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(k64BitSize, PTY_i64), *vreg, *stMopnd));
8170         }
8171     }
8172     /* Create the copy address parameter for the struct */
8173     RegOperand *fpopnd = &GetOrCreatePhysicalRegisterOperand(RSP, k64BitSize, kRegTyInt);
8174     ImmOperand *offset = &CreateImmOperand(copyOffset, k64BitSize, false);
8175     if (ploc.reg0 == kRinvalid) {
8176         RegOperand &res = CreateRegisterOperandOfType(PTY_u64);
8177         SelectAdd(res, *fpopnd, *offset, PTY_u64);
8178         MemOperand &stMopnd2 = CreateMemOpnd(RSP, ploc.memOffset, k64BitSize);
8179         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(k64BitSize, PTY_i64), res, stMopnd2));
8180         return nullptr;
8181     } else {
8182         RegOperand *parmOpnd =
8183             &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(ploc.reg0), k64BitSize, kRegTyInt);
8184         SelectAdd(*parmOpnd, *fpopnd, *offset, PTY_a64);
8185         return parmOpnd;
8186     }
8187 }
8188 
CreateCallStructMemcpyToParamReg(MIRType & structType,int32 structCopyOffset,AArch64CallConvImpl & parmLocator,ListOperand & srcOpnds)8189 void AArch64CGFunc::CreateCallStructMemcpyToParamReg(MIRType &structType, int32 structCopyOffset,
8190                                                      AArch64CallConvImpl &parmLocator, ListOperand &srcOpnds)
8191 {
8192     RegOperand &spReg = GetOrCreatePhysicalRegisterOperand(RSP, k64BitSize, kRegTyInt);
8193     ImmOperand &offsetOpnd = CreateImmOperand(structCopyOffset, k64BitSize, false);
8194 
8195     CCLocInfo ploc;
8196     parmLocator.LocateNextParm(structType, ploc);
8197     CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
8198     if (ploc.reg0 != 0) {
8199         RegOperand &res = GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(ploc.reg0), k64BitSize, kRegTyInt);
8200         SelectAdd(res, spReg, offsetOpnd, PTY_a64);
8201         srcOpnds.PushOpnd(res);
8202     } else {
8203         RegOperand &parmOpnd = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8204         SelectAdd(parmOpnd, spReg, offsetOpnd, PTY_a64);
8205         MemOperand &stmopnd = CreateMemOpnd(RSP, ploc.memOffset, k64BitSize);
8206         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(k64BitSize, PTY_i64), parmOpnd, stmopnd));
8207     }
8208 }
8209 
SelectParmListForAggregate(BaseNode & argExpr,ListOperand & srcOpnds,AArch64CallConvImpl & parmLocator,int32 & structCopyOffset)8210 void AArch64CGFunc::SelectParmListForAggregate(BaseNode &argExpr, ListOperand &srcOpnds,
8211                                                AArch64CallConvImpl &parmLocator, int32 &structCopyOffset)
8212 {
8213     uint64 symSize;
8214     int32 rhsOffset = 0;
8215     if (argExpr.GetOpCode() == OP_dread) {
8216         DreadNode &dread = static_cast<DreadNode &>(argExpr);
8217         MIRSymbol *sym = GetBecommon().GetMIRModule().CurFunction()->GetLocalOrGlobalSymbol(dread.GetStIdx());
8218         MIRType *ty = sym->GetType();
8219         if (dread.GetFieldID() != 0) {
8220             MIRStructType *structty = static_cast<MIRStructType *>(ty);
8221             ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(structty->GetFieldTyIdx(dread.GetFieldID()));
8222             rhsOffset = GetBecommon().GetFieldOffset(*structty, dread.GetFieldID()).first;
8223         }
8224         symSize = GetBecommon().GetTypeSize(ty->GetTypeIndex().GetIdx());
8225         if (symSize <= k16ByteSize) {
8226             SelectParmListDreadSmallAggregate(*sym, *ty, srcOpnds, rhsOffset, parmLocator, dread.GetFieldID());
8227         } else if (symSize > kParmMemcpySize) {
8228             CreateCallStructMemcpyToParamReg(*ty, structCopyOffset, parmLocator, srcOpnds);
8229             structCopyOffset += static_cast<int32>(RoundUp(symSize, GetPointerSize()));
8230         } else {
8231             SelectParmListDreadLargeAggregate(*sym, *ty, srcOpnds, parmLocator, structCopyOffset, rhsOffset);
8232         }
8233     } else if (argExpr.GetOpCode() == OP_iread) {
8234         IreadNode &iread = static_cast<IreadNode &>(argExpr);
8235         MIRPtrType *pointerty =
8236             static_cast<MIRPtrType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(iread.GetTyIdx()));
8237         MIRType *ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(pointerty->GetPointedTyIdx());
8238         if (iread.GetFieldID() != 0) {
8239             MIRStructType *structty = static_cast<MIRStructType *>(ty);
8240             ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(structty->GetFieldTyIdx(iread.GetFieldID()));
8241             rhsOffset = GetBecommon().GetFieldOffset(*structty, iread.GetFieldID()).first;
8242         }
8243         symSize = GetBecommon().GetTypeSize(ty->GetTypeIndex().GetIdx());
8244         if (symSize <= k16ByteSize) {
8245             SelectParmListIreadSmallAggregate(iread, *ty, srcOpnds, rhsOffset, parmLocator);
8246         } else if (symSize > kParmMemcpySize) {
8247             RegOperand *ireadOpnd = static_cast<RegOperand *>(HandleExpr(iread, *(iread.Opnd(0))));
8248             if (rhsOffset > 0) {
8249                 RegOperand *addrOpnd = &LoadIntoRegister(*ireadOpnd, iread.Opnd(0)->GetPrimType());
8250                 regno_t vRegNO = NewVReg(kRegTyInt, k8ByteSize);
8251                 RegOperand *result = &CreateVirtualRegisterOperand(vRegNO);
8252                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *result, *addrOpnd,
8253                                                                    CreateImmOperand(rhsOffset, k64BitSize, false)));
8254             }
8255 
8256             CreateCallStructMemcpyToParamReg(*ty, structCopyOffset, parmLocator, srcOpnds);
8257             structCopyOffset += static_cast<int32>(RoundUp(symSize, GetPointerSize()));
8258         } else {
8259             SelectParmListIreadLargeAggregate(iread, *ty, srcOpnds, parmLocator, structCopyOffset, rhsOffset);
8260         }
8261     } else {
8262         CHECK_FATAL(0, "NYI");
8263     }
8264 }
8265 
SelectParmListGetStructReturnSize(StmtNode & naryNode)8266 size_t AArch64CGFunc::SelectParmListGetStructReturnSize(StmtNode &naryNode)
8267 {
8268     if (naryNode.GetOpCode() == OP_call) {
8269         CallNode &callNode = static_cast<CallNode &>(naryNode);
8270         MIRFunction *callFunc = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(callNode.GetPUIdx());
8271         TyIdx retIdx = callFunc->GetReturnTyIdx();
8272         if (callFunc->IsFirstArgReturn()) {
8273             MIRType *ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(callFunc->GetFormalDefVec()[0].formalTyIdx);
8274             return GetBecommon().GetTypeSize(static_cast<MIRPtrType *>(ty)->GetPointedTyIdx());
8275         }
8276         size_t retSize = GetBecommon().GetTypeSize(retIdx.GetIdx());
8277         if ((retSize == 0) && callFunc->IsReturnStruct()) {
8278             TyIdx tyIdx = callFunc->GetFuncRetStructTyIdx();
8279             return GetBecommon().GetTypeSize(tyIdx);
8280         }
8281         return retSize;
8282     } else if (naryNode.GetOpCode() == OP_icall) {
8283         IcallNode &icallNode = static_cast<IcallNode &>(naryNode);
8284         CallReturnVector *p2nrets = &icallNode.GetReturnVec();
8285         if (p2nrets->size() == k1ByteSize) {
8286             StIdx stIdx = (*p2nrets)[0].first;
8287             MIRSymbol *sym = GetBecommon().GetMIRModule().CurFunction()->GetSymTab()->GetSymbolFromStIdx(stIdx.Idx());
8288             if (sym != nullptr) {
8289                 return GetBecommon().GetTypeSize(sym->GetTyIdx().GetIdx());
8290             }
8291         }
8292     } else if (naryNode.GetOpCode() == OP_icallproto) {
8293         IcallNode &icallProto = static_cast<IcallNode &>(naryNode);
8294         MIRFuncType *funcTy =
8295             static_cast<MIRFuncType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(icallProto.GetRetTyIdx()));
8296         if (funcTy->FirstArgReturn()) {
8297             MIRType *ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(funcTy->GetNthParamType(0));
8298             return GetBecommon().GetTypeSize(static_cast<MIRPtrType *>(ty)->GetPointedTyIdx());
8299         }
8300         return GetBecommon().GetTypeSize(funcTy->GetRetTyIdx());
8301     }
8302     return 0;
8303 }
8304 
SelectParmListPreprocessLargeStruct(BaseNode & argExpr,int32 & structCopyOffset)8305 void AArch64CGFunc::SelectParmListPreprocessLargeStruct(BaseNode &argExpr, int32 &structCopyOffset)
8306 {
8307     uint64 symSize;
8308     int32 rhsOffset = 0;
8309     if (argExpr.GetOpCode() == OP_dread) {
8310         DreadNode &dread = static_cast<DreadNode &>(argExpr);
8311         MIRSymbol *sym = GetBecommon().GetMIRModule().CurFunction()->GetLocalOrGlobalSymbol(dread.GetStIdx());
8312         MIRType *ty = sym->GetType();
8313         if (dread.GetFieldID() != 0) {
8314             MIRStructType *structty = static_cast<MIRStructType *>(ty);
8315             ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(structty->GetFieldTyIdx(dread.GetFieldID()));
8316             rhsOffset = GetBecommon().GetFieldOffset(*structty, dread.GetFieldID()).first;
8317         }
8318         symSize = GetBecommon().GetTypeSize(ty->GetTypeIndex().GetIdx());
8319         if (symSize > kParmMemcpySize) {
8320             CreateCallStructParamMemcpy(sym, nullptr, static_cast<uint32>(symSize), structCopyOffset, rhsOffset);
8321             structCopyOffset += static_cast<int32>(RoundUp(symSize, GetPointerSize()));
8322         } else if (symSize > k16ByteSize) {
8323             uint32 numMemOp = static_cast<uint32>(RoundUp(symSize, GetPointerSize()) / GetPointerSize());
8324             structCopyOffset += static_cast<int32>(numMemOp * GetPointerSize());
8325         }
8326     } else if (argExpr.GetOpCode() == OP_iread) {
8327         IreadNode &iread = static_cast<IreadNode &>(argExpr);
8328         MIRPtrType *pointerty =
8329             static_cast<MIRPtrType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(iread.GetTyIdx()));
8330         MIRType *ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(pointerty->GetPointedTyIdx());
8331         if (iread.GetFieldID() != 0) {
8332             MIRStructType *structty = static_cast<MIRStructType *>(ty);
8333             ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(structty->GetFieldTyIdx(iread.GetFieldID()));
8334             rhsOffset = GetBecommon().GetFieldOffset(*structty, iread.GetFieldID()).first;
8335         }
8336         symSize = GetBecommon().GetTypeSize(ty->GetTypeIndex().GetIdx());
8337         if (symSize > kParmMemcpySize) {
8338             RegOperand *ireadOpnd = static_cast<RegOperand *>(HandleExpr(iread, *(iread.Opnd(0))));
8339             RegOperand *addrOpnd = &LoadIntoRegister(*ireadOpnd, iread.Opnd(0)->GetPrimType());
8340             if (rhsOffset > 0) {
8341                 regno_t vRegNO = NewVReg(kRegTyInt, k8ByteSize);
8342                 RegOperand *result = &CreateVirtualRegisterOperand(vRegNO);
8343                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *result, *addrOpnd,
8344                                                                    CreateImmOperand(rhsOffset, k64BitSize, false)));
8345                 addrOpnd = result;
8346             }
8347 
8348             CreateCallStructParamMemcpy(nullptr, addrOpnd, static_cast<uint32>(symSize), structCopyOffset, rhsOffset);
8349             structCopyOffset += static_cast<int32>(RoundUp(symSize, GetPointerSize()));
8350         } else if (symSize > k16ByteSize) {
8351             uint32 numMemOp = static_cast<uint32>(RoundUp(symSize, GetPointerSize()) / GetPointerSize());
8352             structCopyOffset += static_cast<int32>(numMemOp * GetPointerSize());
8353         }
8354     }
8355 }
8356 
8357 /* preprocess call in parmlist */
MarkParmListCall(BaseNode & expr)8358 bool AArch64CGFunc::MarkParmListCall(BaseNode &expr)
8359 {
8360     if (!CGOptions::IsPIC()) {
8361         return false;
8362     }
8363     switch (expr.GetOpCode()) {
8364         case OP_addrof: {
8365             auto &addrNode = static_cast<AddrofNode &>(expr);
8366             MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(addrNode.GetStIdx());
8367             if (symbol->IsThreadLocal()) {
8368                 return true;
8369             }
8370             break;
8371         }
8372         default: {
8373             for (auto i = 0; i < expr.GetNumOpnds(); i++) {
8374                 if (expr.Opnd(i)) {
8375                     if (MarkParmListCall(*expr.Opnd(i))) {
8376                         return true;
8377                     }
8378                 }
8379             }
8380             break;
8381         }
8382     }
8383     return false;
8384 }
8385 
SelectParmListPreprocess(const StmtNode & naryNode,size_t start,std::set<size_t> & specialArgs)8386 void AArch64CGFunc::SelectParmListPreprocess(const StmtNode &naryNode, size_t start, std::set<size_t> &specialArgs)
8387 {
8388     size_t i = start;
8389     int32 structCopyOffset = GetMaxParamStackSize() - GetStructCopySize();
8390     for (; i < naryNode.NumOpnds(); ++i) {
8391         BaseNode *argExpr = naryNode.Opnd(i);
8392         PrimType primType = argExpr->GetPrimType();
8393         if (MarkParmListCall(*argExpr)) {
8394             (void)specialArgs.emplace(i);
8395         }
8396         DEBUG_ASSERT(primType != PTY_void, "primType should not be void");
8397         if (primType != PTY_agg) {
8398             continue;
8399         }
8400         SelectParmListPreprocessLargeStruct(*argExpr, structCopyOffset);
8401     }
8402 }
8403 
8404 /*
8405    SelectParmList generates an instrunction for each of the parameters
8406    to load the parameter value into the corresponding register.
8407    We return a list of registers to the call instruction because
8408    they may be needed in the register allocation phase.
8409  */
SelectParmList(StmtNode & naryNode,ListOperand & srcOpnds,bool isCallNative)8410 void AArch64CGFunc::SelectParmList(StmtNode &naryNode, ListOperand &srcOpnds, bool isCallNative)
8411 {
8412     size_t i = 0;
8413     if (naryNode.GetOpCode() == OP_icall || naryNode.GetOpCode() == OP_icallproto || isCallNative) {
8414         i++;
8415     }
8416     std::set<size_t> specialArgs;
8417     SelectParmListPreprocess(naryNode, i, specialArgs);
8418     bool specialArg = false;
8419     bool firstArgReturn = false;
8420     MIRFunction *callee = nullptr;
8421     if (dynamic_cast<CallNode *>(&naryNode) != nullptr) {
8422         auto calleePuIdx = static_cast<CallNode &>(naryNode).GetPUIdx();
8423         callee = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(calleePuIdx);
8424         firstArgReturn = callee->IsFirstArgReturn();
8425     } else if (naryNode.GetOpCode() == OP_icallproto) {
8426         IcallNode *icallnode = &static_cast<IcallNode &>(naryNode);
8427         MIRFuncType *funcType =
8428             static_cast<MIRFuncType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(icallnode->GetRetTyIdx()));
8429         firstArgReturn = funcType->FirstArgReturn();
8430     }
8431     BB *curBBrecord = GetCurBB();
8432     BB *tmpBB = nullptr;
8433     if (!specialArgs.empty()) {
8434         tmpBB = CreateNewBB();
8435         specialArg = true;
8436     }
8437     AArch64CallConvImpl parmLocator(GetBecommon());
8438     CCLocInfo ploc;
8439     int32 structCopyOffset = GetMaxParamStackSize() - GetStructCopySize();
8440     std::vector<Insn *> insnForStackArgs;
8441     uint32 stackArgsCount = 0;
8442     for (uint32 pnum = 0; i < naryNode.NumOpnds(); ++i, ++pnum) {
8443         if (specialArg) {
8444             DEBUG_ASSERT(tmpBB, "need temp bb for lower priority args");
8445             SetCurBB(specialArgs.count(i) ? *curBBrecord : *tmpBB);
8446         }
8447         bool is64x1vec = false;
8448         MIRType *ty = nullptr;
8449         BaseNode *argExpr = naryNode.Opnd(i);
8450         PrimType primType = argExpr->GetPrimType();
8451         DEBUG_ASSERT(primType != PTY_void, "primType should not be void");
8452         if (callee != nullptr && pnum < callee->GetFormalCount() && callee->GetFormal(pnum) != nullptr) {
8453             is64x1vec = callee->GetFormal(pnum)->GetAttr(ATTR_oneelem_simd);
8454         }
8455         switch (argExpr->op) {
8456             case OP_dread: {
8457                 DreadNode *dNode = static_cast<DreadNode *>(argExpr);
8458                 MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(dNode->GetStIdx());
8459                 if (dNode->GetFieldID() != 0) {
8460                     MIRStructType *structType = static_cast<MIRStructType *>(symbol->GetType());
8461                     DEBUG_ASSERT(structType != nullptr, "SelectParmList: non-zero fieldID for non-structure");
8462                     FieldAttrs fa = structType->GetFieldAttrs(dNode->GetFieldID());
8463                     is64x1vec = fa.GetAttr(FLDATTR_oneelem_simd);
8464                 } else {
8465                     is64x1vec = symbol->GetAttr(ATTR_oneelem_simd);
8466                 }
8467                 break;
8468             }
8469             case OP_iread: {
8470                 IreadNode *iNode = static_cast<IreadNode *>(argExpr);
8471                 MIRType *type = GlobalTables::GetTypeTable().GetTypeFromTyIdx(iNode->GetTyIdx());
8472                 MIRPtrType *ptrTyp = static_cast<MIRPtrType *>(type);
8473                 DEBUG_ASSERT(ptrTyp != nullptr, "expect a pointer type at iread node");
8474                 MIRType *pointedTy = GlobalTables::GetTypeTable().GetTypeFromTyIdx(ptrTyp->GetPointedTyIdx());
8475                 if (iNode->GetFieldID() != 0) {
8476                     MIRStructType *structType = static_cast<MIRStructType *>(pointedTy);
8477                     FieldAttrs fa = structType->GetFieldAttrs(iNode->GetFieldID());
8478                     is64x1vec = fa.GetAttr(FLDATTR_oneelem_simd);
8479                 } else {
8480                     TypeAttrs ta = static_cast<MIRPtrType *>(ptrTyp)->GetTypeAttrs();
8481                     is64x1vec = ta.GetAttr(ATTR_oneelem_simd);
8482                 }
8483                 break;
8484             }
8485             case OP_constval: {
8486                 CallNode *call = safe_cast<CallNode>(&naryNode);
8487                 if (call == nullptr) {
8488                     break;
8489                 }
8490                 MIRFunction *fn = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(call->GetPUIdx());
8491                 if (fn == nullptr || fn->GetFormalCount() == 0 || fn->GetFormalCount() <= pnum) {
8492                     break;
8493                 }
8494                 is64x1vec = fn->GetFormalDefAt(pnum).formalAttrs.GetAttr(ATTR_oneelem_simd);
8495                 break;
8496             }
8497             default:
8498                 break;
8499         }
8500         /* use alloca  */
8501         if (primType == PTY_agg) {
8502             SelectParmListForAggregate(*argExpr, srcOpnds, parmLocator, structCopyOffset);
8503             continue;
8504         }
8505         ty = GlobalTables::GetTypeTable().GetTypeTable()[static_cast<uint32>(primType)];
8506         RegOperand *expRegOpnd = nullptr;
8507         Operand *opnd = HandleExpr(naryNode, *argExpr);
8508         if (opnd->GetKind() == Operand::kOpdRegister && static_cast<RegOperand *>(opnd)->GetIF64Vec()) {
8509             is64x1vec = true;
8510         }
8511         if (!opnd->IsRegister()) {
8512             opnd = &LoadIntoRegister(*opnd, primType);
8513         }
8514         expRegOpnd = static_cast<RegOperand *>(opnd);
8515 
8516         if ((pnum == 0) && firstArgReturn) {
8517             parmLocator.InitCCLocInfo(ploc);
8518             ploc.reg0 = R8;
8519         } else {
8520             parmLocator.LocateNextParm(*ty, ploc);
8521         }
8522         /* is64x1vec should be an int64 value in an FP/simd reg for ABI compliance,
8523            convert R-reg to equivalent V-reg */
8524         PrimType destPrimType = primType;
8525         if (is64x1vec && ploc.reg0 != kRinvalid && ploc.reg0 < R7) {
8526             ploc.reg0 = AArch64Abi::floatParmRegs[static_cast<int>(ploc.reg0) - 1];
8527             destPrimType = PTY_f64;
8528         }
8529 
8530         /* skip unused args */
8531         if (callee != nullptr && callee->GetFuncDesc().IsArgUnused(pnum))
8532             continue;
8533 
8534         if (ploc.reg0 != kRinvalid) { /* load to the register. */
8535             CHECK_FATAL(expRegOpnd != nullptr, "null ptr check");
8536             RegOperand &parmRegOpnd = GetOrCreatePhysicalRegisterOperand(
8537                 static_cast<AArch64reg>(ploc.reg0), expRegOpnd->GetSize(), GetRegTyFromPrimTy(destPrimType));
8538             SelectCopy(parmRegOpnd, destPrimType, *expRegOpnd, primType);
8539             srcOpnds.PushOpnd(parmRegOpnd);
8540         } else { /* store to the memory segment for stack-passsed arguments. */
8541             if (CGOptions::IsBigEndian()) {
8542                 if (GetPrimTypeBitSize(primType) < k64BitSize) {
8543                     ploc.memOffset = ploc.memOffset + static_cast<int32>(k4BitSize);
8544                 }
8545             }
8546             MemOperand &actMemOpnd = CreateMemOpnd(RSP, ploc.memOffset, GetPrimTypeBitSize(primType));
8547             Insn &strInsn = GetInsnBuilder()->BuildInsn(PickStInsn(GetPrimTypeBitSize(primType), primType), *expRegOpnd,
8548                                                         actMemOpnd);
8549             actMemOpnd.SetStackArgMem(true);
8550             if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel1 && stackArgsCount < kShiftAmount12) {
8551                 (void)insnForStackArgs.emplace_back(&strInsn);
8552                 stackArgsCount++;
8553             } else {
8554                 GetCurBB()->AppendInsn(strInsn);
8555             }
8556         }
8557         DEBUG_ASSERT(ploc.reg1 == 0, "SelectCall NYI");
8558     }
8559     if (specialArg) {
8560         DEBUG_ASSERT(tmpBB, "need temp bb for lower priority args");
8561         curBBrecord->InsertAtEnd(*tmpBB);
8562         SetCurBB(*curBBrecord);
8563     }
8564     for (auto &strInsn : insnForStackArgs) {
8565         GetCurBB()->AppendInsn(*strInsn);
8566     }
8567 }
8568 
SelectParmListNotC(StmtNode & naryNode,ListOperand & srcOpnds)8569 void AArch64CGFunc::SelectParmListNotC(StmtNode &naryNode, ListOperand &srcOpnds)
8570 {
8571     size_t i = 0;
8572     if (naryNode.GetOpCode() == OP_icall || naryNode.GetOpCode() == OP_icallproto) {
8573         i++;
8574     }
8575 
8576     CCImpl &parmLocator = *GetOrCreateLocator(CCImpl::GetCallConvKind(naryNode));
8577     CCLocInfo ploc;
8578     std::vector<Insn *> insnForStackArgs;
8579     uint32 stackArgsCount = 0;
8580     for (uint32 pnum = 0; i < naryNode.NumOpnds(); ++i, ++pnum) {
8581         MIRType *ty = nullptr;
8582         BaseNode *argExpr = naryNode.Opnd(i);
8583         PrimType primType = argExpr->GetPrimType();
8584         DEBUG_ASSERT(primType != PTY_void, "primType should not be void");
8585         /* use alloca  */
8586         ty = GlobalTables::GetTypeTable().GetTypeTable()[static_cast<uint32>(primType)];
8587         RegOperand *expRegOpnd = nullptr;
8588         Operand *opnd = HandleExpr(naryNode, *argExpr);
8589         if (!opnd->IsRegister()) {
8590             opnd = &LoadIntoRegister(*opnd, primType);
8591         }
8592         expRegOpnd = static_cast<RegOperand *>(opnd);
8593 
8594         parmLocator.LocateNextParm(*ty, ploc);
8595         PrimType destPrimType = primType;
8596         if (ploc.reg0 != kRinvalid) { /* load to the register. */
8597             CHECK_FATAL(expRegOpnd != nullptr, "null ptr check");
8598             RegOperand &parmRegOpnd = GetOrCreatePhysicalRegisterOperand(
8599                 static_cast<AArch64reg>(ploc.reg0), expRegOpnd->GetSize(), GetRegTyFromPrimTy(destPrimType));
8600             SelectCopy(parmRegOpnd, destPrimType, *expRegOpnd, primType);
8601             srcOpnds.PushOpnd(parmRegOpnd);
8602         } else { /* store to the memory segment for stack-passsed arguments. */
8603             if (CGOptions::IsBigEndian()) {
8604                 if (GetPrimTypeBitSize(primType) < k64BitSize) {
8605                     ploc.memOffset = ploc.memOffset + static_cast<int32>(k4BitSize);
8606                 }
8607             }
8608             MemOperand &actMemOpnd = CreateMemOpnd(RSP, ploc.memOffset, GetPrimTypeBitSize(primType));
8609             Insn &strInsn = GetInsnBuilder()->BuildInsn(PickStInsn(GetPrimTypeBitSize(primType), primType), *expRegOpnd,
8610                                                         actMemOpnd);
8611             actMemOpnd.SetStackArgMem(true);
8612             if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel1 && stackArgsCount < kShiftAmount12) {
8613                 (void)insnForStackArgs.emplace_back(&strInsn);
8614                 stackArgsCount++;
8615             } else {
8616                 GetCurBB()->AppendInsn(strInsn);
8617             }
8618         }
8619         DEBUG_ASSERT(ploc.reg1 == 0, "SelectCall NYI");
8620     }
8621     for (auto &strInsn : insnForStackArgs) {
8622         GetCurBB()->AppendInsn(*strInsn);
8623     }
8624 }
8625 
8626 // based on call conv, choose how to prepare args
SelectParmListWrapper(StmtNode & naryNode,ListOperand & srcOpnds,bool isCallNative)8627 void AArch64CGFunc::SelectParmListWrapper(StmtNode &naryNode, ListOperand &srcOpnds, bool isCallNative)
8628 {
8629     if (CCImpl::GetCallConvKind(naryNode) == kCCall) {
8630         SelectParmList(naryNode, srcOpnds, isCallNative);
8631     } else if (CCImpl::GetCallConvKind(naryNode) == kWebKitJS || CCImpl::GetCallConvKind(naryNode) == kGHC) {
8632         SelectParmListNotC(naryNode, srcOpnds);
8633     } else {
8634         CHECK_FATAL(false, "niy");
8635     }
8636 }
8637 /*
8638  * for MCC_DecRefResetPair(addrof ptr %Reg17_R5592, addrof ptr %Reg16_R6202) or
8639  * MCC_ClearLocalStackRef(addrof ptr %Reg17_R5592), the parameter (addrof ptr xxx) is converted to asm as follow:
8640  * add vreg, x29, #imm
8641  * mov R0/R1, vreg
8642  * this function is used to prepare parameters, the generated vreg is returned, and #imm is saved in offsetValue.
8643  */
SelectClearStackCallParam(const AddrofNode & expr,int64 & offsetValue)8644 Operand *AArch64CGFunc::SelectClearStackCallParam(const AddrofNode &expr, int64 &offsetValue)
8645 {
8646     MIRSymbol *symbol = GetMirModule().CurFunction()->GetLocalOrGlobalSymbol(expr.GetStIdx());
8647     PrimType ptype = expr.GetPrimType();
8648     regno_t vRegNO = NewVReg(kRegTyInt, GetPrimTypeSize(ptype));
8649     Operand &result = CreateVirtualRegisterOperand(vRegNO);
8650     CHECK_FATAL(expr.GetFieldID() == 0, "the fieldID of parameter in clear stack reference call must be 0");
8651     if (!CGOptions::IsQuiet()) {
8652         maple::LogInfo::MapleLogger(kLlErr)
8653             << "Warning: we expect AddrOf with StImmOperand is not used for local variables";
8654     }
8655     auto *symLoc = static_cast<AArch64SymbolAlloc *>(GetMemlayout()->GetSymAllocInfo(symbol->GetStIndex()));
8656     ImmOperand *offset = nullptr;
8657     if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsArgsStkPassed) {
8658         offset = &CreateImmOperand(GetBaseOffset(*symLoc), k64BitSize, false, kUnAdjustVary);
8659     } else if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsRefLocals) {
8660         auto it = immOpndsRequiringOffsetAdjustmentForRefloc.find(symLoc);
8661         if (it != immOpndsRequiringOffsetAdjustmentForRefloc.end()) {
8662             offset = (*it).second;
8663         } else {
8664             offset = &CreateImmOperand(GetBaseOffset(*symLoc), k64BitSize, false);
8665             immOpndsRequiringOffsetAdjustmentForRefloc[symLoc] = offset;
8666         }
8667     } else {
8668         CHECK_FATAL(false, "the symLoc of parameter in clear stack reference call is unreasonable");
8669     }
8670     DEBUG_ASSERT(offset != nullptr, "offset should not be nullptr");
8671     offsetValue = offset->GetValue();
8672     SelectAdd(result, *GetBaseReg(*symLoc), *offset, PTY_u64);
8673     if (GetCG()->GenerateVerboseCG()) {
8674         /* Add a comment */
8675         Insn *insn = GetCurBB()->GetLastInsn();
8676         std::string comm = "local/formal var: ";
8677         comm += symbol->GetName();
8678         insn->SetComment(comm);
8679     }
8680     return &result;
8681 }
8682 
8683 /* select paramters for MCC_DecRefResetPair and MCC_ClearLocalStackRef function */
SelectClearStackCallParmList(const StmtNode & naryNode,ListOperand & srcOpnds,std::vector<int64> & stackPostion)8684 void AArch64CGFunc::SelectClearStackCallParmList(const StmtNode &naryNode, ListOperand &srcOpnds,
8685                                                  std::vector<int64> &stackPostion)
8686 {
8687     CHECK_FATAL(false, "should not go here");
8688     AArch64CallConvImpl parmLocator(GetBecommon());
8689     CCLocInfo ploc;
8690     for (size_t i = 0; i < naryNode.NumOpnds(); ++i) {
8691         MIRType *ty = nullptr;
8692         BaseNode *argExpr = naryNode.Opnd(i);
8693         PrimType primType = argExpr->GetPrimType();
8694         DEBUG_ASSERT(primType != PTY_void, "primType check");
8695         /* use alloc */
8696         CHECK_FATAL(primType != PTY_agg, "the type of argument is unreasonable");
8697         ty = GlobalTables::GetTypeTable().GetTypeTable()[static_cast<uint32>(primType)];
8698         CHECK_FATAL(argExpr->GetOpCode() == OP_addrof, "the argument of clear stack call is unreasonable");
8699         auto *expr = static_cast<AddrofNode *>(argExpr);
8700         int64 offsetValue = 0;
8701         Operand *opnd = SelectClearStackCallParam(*expr, offsetValue);
8702         stackPostion.emplace_back(offsetValue);
8703         auto *expRegOpnd = static_cast<RegOperand *>(opnd);
8704         CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
8705         parmLocator.LocateNextParm(*ty, ploc);
8706         CHECK_FATAL(ploc.reg0 != 0, "the parameter of ClearStackCall must be passed by register");
8707         CHECK_FATAL(expRegOpnd != nullptr, "null ptr check");
8708         RegOperand &parmRegOpnd = GetOrCreatePhysicalRegisterOperand(
8709             static_cast<AArch64reg>(ploc.reg0), expRegOpnd->GetSize(), GetRegTyFromPrimTy(primType));
8710         SelectCopy(parmRegOpnd, primType, *expRegOpnd, primType);
8711         srcOpnds.PushOpnd(parmRegOpnd);
8712         DEBUG_ASSERT(ploc.reg1 == 0, "SelectCall NYI");
8713     }
8714 }
8715 
8716 /*
8717  * intrinsify Unsafe.getAndAddInt and Unsafe.getAndAddLong
8718  * generate an intrinsic instruction instead of a function call
8719  * intrinsic_get_add_int w0, xt, ws, ws, x1, x2, w3, label
8720  */
IntrinsifyGetAndAddInt(ListOperand & srcOpnds,PrimType pty)8721 void AArch64CGFunc::IntrinsifyGetAndAddInt(ListOperand &srcOpnds, PrimType pty)
8722 {
8723     MapleList<RegOperand *> &opnds = srcOpnds.GetOperands();
8724     /* Unsafe.getAndAddInt has more than 4 parameters */
8725     DEBUG_ASSERT(opnds.size() >= 4, "ensure the operands number");
8726     auto iter = opnds.begin();
8727     RegOperand *objOpnd = *(++iter);
8728     RegOperand *offOpnd = *(++iter);
8729     RegOperand *deltaOpnd = *(++iter);
8730     auto &retVal = static_cast<RegOperand &>(GetTargetRetOperand(pty, -1));
8731     LabelIdx labIdx = CreateLabel();
8732     LabelOperand &targetOpnd = GetOrCreateLabelOperand(labIdx);
8733     RegOperand &tempOpnd0 = CreateRegisterOperandOfType(PTY_i64);
8734     RegOperand &tempOpnd1 = CreateRegisterOperandOfType(pty);
8735     RegOperand &tempOpnd2 = CreateRegisterOperandOfType(PTY_i32);
8736     MOperator mOp = (pty == PTY_i64) ? MOP_get_and_addL : MOP_get_and_addI;
8737     std::vector<Operand *> intrnOpnds;
8738     intrnOpnds.emplace_back(&retVal);
8739     intrnOpnds.emplace_back(&tempOpnd0);
8740     intrnOpnds.emplace_back(&tempOpnd1);
8741     intrnOpnds.emplace_back(&tempOpnd2);
8742     intrnOpnds.emplace_back(objOpnd);
8743     intrnOpnds.emplace_back(offOpnd);
8744     intrnOpnds.emplace_back(deltaOpnd);
8745     intrnOpnds.emplace_back(&targetOpnd);
8746     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, intrnOpnds));
8747 }
8748 
8749 /*
8750  * intrinsify Unsafe.getAndSetInt and Unsafe.getAndSetLong
8751  * generate an intrinsic instruction instead of a function call
8752  */
IntrinsifyGetAndSetInt(ListOperand & srcOpnds,PrimType pty)8753 void AArch64CGFunc::IntrinsifyGetAndSetInt(ListOperand &srcOpnds, PrimType pty)
8754 {
8755     MapleList<RegOperand *> &opnds = srcOpnds.GetOperands();
8756     /* Unsafe.getAndSetInt has 4 parameters */
8757     DEBUG_ASSERT(opnds.size() == 4, "ensure the operands number");
8758     auto iter = opnds.begin();
8759     RegOperand *objOpnd = *(++iter);
8760     RegOperand *offOpnd = *(++iter);
8761     RegOperand *newValueOpnd = *(++iter);
8762     auto &retVal = static_cast<RegOperand &>(GetTargetRetOperand(pty, -1));
8763     LabelIdx labIdx = CreateLabel();
8764     LabelOperand &targetOpnd = GetOrCreateLabelOperand(labIdx);
8765     RegOperand &tempOpnd0 = CreateRegisterOperandOfType(PTY_i64);
8766     RegOperand &tempOpnd1 = CreateRegisterOperandOfType(PTY_i32);
8767 
8768     MOperator mOp = (pty == PTY_i64) ? MOP_get_and_setL : MOP_get_and_setI;
8769     std::vector<Operand *> intrnOpnds;
8770     intrnOpnds.emplace_back(&retVal);
8771     intrnOpnds.emplace_back(&tempOpnd0);
8772     intrnOpnds.emplace_back(&tempOpnd1);
8773     intrnOpnds.emplace_back(objOpnd);
8774     intrnOpnds.emplace_back(offOpnd);
8775     intrnOpnds.emplace_back(newValueOpnd);
8776     intrnOpnds.emplace_back(&targetOpnd);
8777     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, intrnOpnds));
8778 }
8779 
8780 /*
8781  * intrinsify Unsafe.compareAndSwapInt and Unsafe.compareAndSwapLong
8782  * generate an intrinsic instruction instead of a function call
8783  */
IntrinsifyCompareAndSwapInt(ListOperand & srcOpnds,PrimType pty)8784 void AArch64CGFunc::IntrinsifyCompareAndSwapInt(ListOperand &srcOpnds, PrimType pty)
8785 {
8786     MapleList<RegOperand *> &opnds = srcOpnds.GetOperands();
8787     /* Unsafe.compareAndSwapInt has more than 5 parameters */
8788     DEBUG_ASSERT(opnds.size() >= 5, "ensure the operands number");
8789     auto iter = opnds.begin();
8790     RegOperand *objOpnd = *(++iter);
8791     RegOperand *offOpnd = *(++iter);
8792     RegOperand *expectedValueOpnd = *(++iter);
8793     RegOperand *newValueOpnd = *(++iter);
8794     auto &retVal = static_cast<RegOperand &>(GetTargetRetOperand(PTY_i64, -1));
8795     RegOperand &tempOpnd0 = CreateRegisterOperandOfType(PTY_i64);
8796     RegOperand &tempOpnd1 = CreateRegisterOperandOfType(pty);
8797     LabelIdx labIdx1 = CreateLabel();
8798     LabelOperand &label1Opnd = GetOrCreateLabelOperand(labIdx1);
8799     LabelIdx labIdx2 = CreateLabel();
8800     LabelOperand &label2Opnd = GetOrCreateLabelOperand(labIdx2);
8801     MOperator mOp = (pty == PTY_i32) ? MOP_compare_and_swapI : MOP_compare_and_swapL;
8802     std::vector<Operand *> intrnOpnds;
8803     intrnOpnds.emplace_back(&retVal);
8804     intrnOpnds.emplace_back(&tempOpnd0);
8805     intrnOpnds.emplace_back(&tempOpnd1);
8806     intrnOpnds.emplace_back(objOpnd);
8807     intrnOpnds.emplace_back(offOpnd);
8808     intrnOpnds.emplace_back(expectedValueOpnd);
8809     intrnOpnds.emplace_back(newValueOpnd);
8810     intrnOpnds.emplace_back(&label1Opnd);
8811     intrnOpnds.emplace_back(&label2Opnd);
8812     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, intrnOpnds));
8813 }
8814 
8815 /*
8816  * the lowest bit of count field is used to indicate whether or not the string is compressed
8817  * if the string is not compressed, jump to jumpLabIdx
8818  */
CheckStringIsCompressed(BB & bb,RegOperand & str,int32 countOffset,PrimType countPty,LabelIdx jumpLabIdx)8819 RegOperand *AArch64CGFunc::CheckStringIsCompressed(BB &bb, RegOperand &str, int32 countOffset, PrimType countPty,
8820                                                    LabelIdx jumpLabIdx)
8821 {
8822     MemOperand &memOpnd = CreateMemOpnd(str, countOffset, str.GetSize());
8823     uint32 bitSize = GetPrimTypeBitSize(countPty);
8824     MOperator loadOp = PickLdInsn(bitSize, countPty);
8825     RegOperand &countOpnd = CreateRegisterOperandOfType(countPty);
8826     bb.AppendInsn(GetInsnBuilder()->BuildInsn(loadOp, countOpnd, memOpnd));
8827     ImmOperand &immValueOne = CreateImmOperand(countPty, 1);
8828     RegOperand &countLowestBitOpnd = CreateRegisterOperandOfType(countPty);
8829     MOperator andOp = bitSize == k64BitSize ? MOP_xandrri13 : MOP_wandrri12;
8830     bb.AppendInsn(GetInsnBuilder()->BuildInsn(andOp, countLowestBitOpnd, countOpnd, immValueOne));
8831     RegOperand &wzr = GetZeroOpnd(bitSize);
8832     MOperator cmpOp = (bitSize == k64BitSize) ? MOP_xcmprr : MOP_wcmprr;
8833     Operand &rflag = GetOrCreateRflag();
8834     bb.AppendInsn(GetInsnBuilder()->BuildInsn(cmpOp, rflag, wzr, countLowestBitOpnd));
8835     bb.AppendInsn(GetInsnBuilder()->BuildInsn(MOP_beq, rflag, GetOrCreateLabelOperand(jumpLabIdx)));
8836     bb.SetKind(BB::kBBIf);
8837     return &countOpnd;
8838 }
8839 
8840 /*
8841  * count field stores the length shifted one bit to the left
8842  * if the length is less than eight, jump to jumpLabIdx
8843  */
CheckStringLengthLessThanEight(BB & bb,RegOperand & countOpnd,PrimType countPty,LabelIdx jumpLabIdx)8844 RegOperand *AArch64CGFunc::CheckStringLengthLessThanEight(BB &bb, RegOperand &countOpnd, PrimType countPty,
8845                                                           LabelIdx jumpLabIdx)
8846 {
8847     RegOperand &lengthOpnd = CreateRegisterOperandOfType(countPty);
8848     uint32 bitSize = GetPrimTypeBitSize(countPty);
8849     MOperator lsrOp = (bitSize == k64BitSize) ? MOP_xlsrrri6 : MOP_wlsrrri5;
8850     ImmOperand &immValueOne = CreateImmOperand(countPty, 1);
8851     bb.AppendInsn(GetInsnBuilder()->BuildInsn(lsrOp, lengthOpnd, countOpnd, immValueOne));
8852     constexpr int kConstIntEight = 8;
8853     ImmOperand &immValueEight = CreateImmOperand(countPty, kConstIntEight);
8854     MOperator cmpImmOp = (bitSize == k64BitSize) ? MOP_xcmpri : MOP_wcmpri;
8855     Operand &rflag = GetOrCreateRflag();
8856     bb.AppendInsn(GetInsnBuilder()->BuildInsn(cmpImmOp, rflag, lengthOpnd, immValueEight));
8857     bb.AppendInsn(GetInsnBuilder()->BuildInsn(MOP_blt, rflag, GetOrCreateLabelOperand(jumpLabIdx)));
8858     bb.SetKind(BB::kBBIf);
8859     return &lengthOpnd;
8860 }
8861 
GenerateIntrnInsnForStrIndexOf(BB & bb,RegOperand & srcString,RegOperand & patternString,RegOperand & srcCountOpnd,RegOperand & patternLengthOpnd,PrimType countPty,LabelIdx jumpLabIdx)8862 void AArch64CGFunc::GenerateIntrnInsnForStrIndexOf(BB &bb, RegOperand &srcString, RegOperand &patternString,
8863                                                    RegOperand &srcCountOpnd, RegOperand &patternLengthOpnd,
8864                                                    PrimType countPty, LabelIdx jumpLabIdx)
8865 {
8866     RegOperand &srcLengthOpnd = CreateRegisterOperandOfType(countPty);
8867     ImmOperand &immValueOne = CreateImmOperand(countPty, 1);
8868     uint32 bitSize = GetPrimTypeBitSize(countPty);
8869     MOperator lsrOp = (bitSize == k64BitSize) ? MOP_xlsrrri6 : MOP_wlsrrri5;
8870     bb.AppendInsn(GetInsnBuilder()->BuildInsn(lsrOp, srcLengthOpnd, srcCountOpnd, immValueOne));
8871 #ifdef USE_32BIT_REF
8872     const int64 stringBaseObjSize = 16; /* shadow(4)+monitor(4)+count(4)+hash(4) */
8873 #else
8874     const int64 stringBaseObjSize = 20; /* shadow(8)+monitor(4)+count(4)+hash(4) */
8875 #endif /* USE_32BIT_REF */
8876     PrimType pty = (srcString.GetSize() == k64BitSize) ? PTY_i64 : PTY_i32;
8877     ImmOperand &immStringBaseOffset = CreateImmOperand(pty, stringBaseObjSize);
8878     MOperator addOp = (pty == PTY_i64) ? MOP_xaddrri12 : MOP_waddrri12;
8879     RegOperand &srcStringBaseOpnd = CreateRegisterOperandOfType(pty);
8880     bb.AppendInsn(GetInsnBuilder()->BuildInsn(addOp, srcStringBaseOpnd, srcString, immStringBaseOffset));
8881     RegOperand &patternStringBaseOpnd = CreateRegisterOperandOfType(pty);
8882     bb.AppendInsn(GetInsnBuilder()->BuildInsn(addOp, patternStringBaseOpnd, patternString, immStringBaseOffset));
8883     auto &retVal = static_cast<RegOperand &>(GetTargetRetOperand(PTY_i32, -1));
8884     std::vector<Operand *> intrnOpnds;
8885     intrnOpnds.emplace_back(&retVal);
8886     intrnOpnds.emplace_back(&srcStringBaseOpnd);
8887     intrnOpnds.emplace_back(&srcLengthOpnd);
8888     intrnOpnds.emplace_back(&patternStringBaseOpnd);
8889     intrnOpnds.emplace_back(&patternLengthOpnd);
8890     const uint32 tmpRegOperandNum = 6;
8891     for (uint32 i = 0; i < tmpRegOperandNum - 1; ++i) {
8892         RegOperand &tmpOpnd = CreateRegisterOperandOfType(PTY_i64);
8893         intrnOpnds.emplace_back(&tmpOpnd);
8894     }
8895     intrnOpnds.emplace_back(&CreateRegisterOperandOfType(PTY_i32));
8896     const uint32 labelNum = 7;
8897     for (uint32 i = 0; i < labelNum; ++i) {
8898         LabelIdx labIdx = CreateLabel();
8899         LabelOperand &labelOpnd = GetOrCreateLabelOperand(labIdx);
8900         intrnOpnds.emplace_back(&labelOpnd);
8901     }
8902     bb.AppendInsn(GetInsnBuilder()->BuildInsn(MOP_string_indexof, intrnOpnds));
8903     bb.AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xuncond, GetOrCreateLabelOperand(jumpLabIdx)));
8904     bb.SetKind(BB::kBBGoto);
8905 }
8906 
8907 /*
8908  * intrinsify String.indexOf
8909  * generate an intrinsic instruction instead of a function call if both the source string and the specified substring
8910  * are compressed and the length of the substring is not less than 8, i.e.
8911  * bl  String.indexOf, srcString, patternString ===>>
8912  *
8913  * ldr srcCountOpnd, [srcString, offset]
8914  * and srcCountLowestBitOpnd, srcCountOpnd, #1
8915  * cmp wzr, srcCountLowestBitOpnd
8916  * beq Label.call
8917  * ldr patternCountOpnd, [patternString, offset]
8918  * and patternCountLowestBitOpnd, patternCountOpnd, #1
8919  * cmp wzr, patternCountLowestBitOpnd
8920  * beq Label.call
8921  * lsr patternLengthOpnd, patternCountOpnd, #1
8922  * cmp patternLengthOpnd, #8
8923  * blt Label.call
8924  * lsr srcLengthOpnd, srcCountOpnd, #1
8925  * add srcStringBaseOpnd, srcString, immStringBaseOffset
8926  * add patternStringBaseOpnd, patternString, immStringBaseOffset
8927  * intrinsic_string_indexof retVal, srcStringBaseOpnd, srcLengthOpnd, patternStringBaseOpnd, patternLengthOpnd,
8928  *                          tmpOpnd1, tmpOpnd2, tmpOpnd3, tmpOpnd4, tmpOpnd5, tmpOpnd6,
8929  *                          label1, label2, label3, lable3, label4, label5, label6, label7
8930  * b   Label.joint
8931  * Label.call:
8932  * bl  String.indexOf, srcString, patternString
8933  * Label.joint:
8934  */
IntrinsifyStringIndexOf(ListOperand & srcOpnds,const MIRSymbol & funcSym)8935 void AArch64CGFunc::IntrinsifyStringIndexOf(ListOperand &srcOpnds, const MIRSymbol &funcSym)
8936 {
8937     MapleList<RegOperand *> &opnds = srcOpnds.GetOperands();
8938     /* String.indexOf opnd size must be more than 2 */
8939     DEBUG_ASSERT(opnds.size() >= 2, "ensure the operands number");
8940     auto iter = opnds.begin();
8941     RegOperand *srcString = *iter;
8942     RegOperand *patternString = *(++iter);
8943     GStrIdx gStrIdx = GlobalTables::GetStrTable().GetStrIdxFromName(namemangler::kJavaLangStringStr);
8944     MIRType *type =
8945         GlobalTables::GetTypeTable().GetTypeFromTyIdx(GlobalTables::GetTypeNameTable().GetTyIdxFromGStrIdx(gStrIdx));
8946     auto stringType = static_cast<MIRStructType *>(type);
8947     CHECK_FATAL(stringType != nullptr, "Ljava_2Flang_2FString_3B type can not be null");
8948     FieldID fieldID = GetMirModule().GetMIRBuilder()->GetStructFieldIDFromFieldNameParentFirst(stringType, "count");
8949     MIRType *fieldType = stringType->GetFieldType(fieldID);
8950     PrimType countPty = fieldType->GetPrimType();
8951     int32 offset = GetBecommon().GetFieldOffset(*stringType, fieldID).first;
8952     LabelIdx callBBLabIdx = CreateLabel();
8953     RegOperand *srcCountOpnd = CheckStringIsCompressed(*GetCurBB(), *srcString, offset, countPty, callBBLabIdx);
8954 
8955     BB *srcCompressedBB = CreateNewBB();
8956     GetCurBB()->AppendBB(*srcCompressedBB);
8957     RegOperand *patternCountOpnd =
8958         CheckStringIsCompressed(*srcCompressedBB, *patternString, offset, countPty, callBBLabIdx);
8959 
8960     BB *patternCompressedBB = CreateNewBB();
8961     RegOperand *patternLengthOpnd =
8962         CheckStringLengthLessThanEight(*patternCompressedBB, *patternCountOpnd, countPty, callBBLabIdx);
8963 
8964     BB *intrinsicBB = CreateNewBB();
8965     LabelIdx jointLabIdx = CreateLabel();
8966     GenerateIntrnInsnForStrIndexOf(*intrinsicBB, *srcString, *patternString, *srcCountOpnd, *patternLengthOpnd,
8967                                    countPty, jointLabIdx);
8968 
8969     BB *callBB = CreateNewBB();
8970     callBB->AddLabel(callBBLabIdx);
8971     SetLab2BBMap(callBBLabIdx, *callBB);
8972     SetCurBB(*callBB);
8973     Insn &callInsn = AppendCall(funcSym, srcOpnds);
8974     MIRType *retType = funcSym.GetFunction()->GetReturnType();
8975     if (retType != nullptr) {
8976         callInsn.SetRetSize(static_cast<uint32>(retType->GetSize()));
8977     }
8978     GetFunction().SetHasCall();
8979 
8980     BB *jointBB = CreateNewBB();
8981     jointBB->AddLabel(jointLabIdx);
8982     SetLab2BBMap(jointLabIdx, *jointBB);
8983     srcCompressedBB->AppendBB(*patternCompressedBB);
8984     patternCompressedBB->AppendBB(*intrinsicBB);
8985     intrinsicBB->AppendBB(*callBB);
8986     callBB->AppendBB(*jointBB);
8987     SetCurBB(*jointBB);
8988 }
8989 
8990 /* Lmbc calls have no argument, they are all explicit iassignspoff or
8991    blkassign.  Info collected and to be emitted here */
LmbcSelectParmList(ListOperand * srcOpnds,bool isArgReturn)8992 void AArch64CGFunc::LmbcSelectParmList(ListOperand *srcOpnds, bool isArgReturn)
8993 {
8994     if (GetLmbcArgInfo() == nullptr) {
8995         return; /* no arg */
8996     }
8997     CHECK_FATAL(GetMirModule().GetFlavor() == MIRFlavor::kFlavorLmbc, "To be called for Lmbc model only");
8998     MapleVector<RegOperand *> &args = GetLmbcCallArgs();
8999     MapleVector<PrimType> &types = GetLmbcCallArgTypes();
9000     MapleVector<int32> &offsets = GetLmbcCallArgOffsets();
9001     MapleVector<int32> &regs = GetLmbcCallArgNumOfRegs();
9002     int iCnt = 0;
9003     int fCnt = 0;
9004     for (size_t i = isArgReturn ? 1 : 0; i < args.size(); i++) {
9005         RegType ty = args[i]->GetRegisterType();
9006         PrimType pTy = types[i];
9007         AArch64reg reg;
9008         if (args[i]->IsOfIntClass() && (iCnt + regs[i]) <= static_cast<int32>(k8ByteSize)) {
9009             reg = static_cast<AArch64reg>(R0 + iCnt++);
9010             RegOperand *res = &GetOrCreatePhysicalRegisterOperand(reg, GetPrimTypeSize(pTy) * kBitsPerByte, ty);
9011             SelectCopy(*res, pTy, *args[i], pTy);
9012             srcOpnds->PushOpnd(*res);
9013         } else if (!args[i]->IsOfIntClass() && (fCnt + regs[i]) <= static_cast<int32>(k8ByteSize)) {
9014             reg = static_cast<AArch64reg>(V0 + fCnt++);
9015             RegOperand *res = &GetOrCreatePhysicalRegisterOperand(reg, GetPrimTypeSize(pTy) * kBitsPerByte, ty);
9016             SelectCopy(*res, pTy, *args[i], pTy);
9017             srcOpnds->PushOpnd(*res);
9018         } else {
9019             uint32 pSize = GetPrimTypeSize(pTy);
9020             Operand &memOpd = CreateMemOpnd(RSP, offsets[i], pSize);
9021             GetCurBB()->AppendInsn(
9022                 GetInsnBuilder()->BuildInsn(PickStInsn(pSize * kBitsPerByte, pTy), *args[i], memOpd));
9023         }
9024     }
9025     /* Load x8 if 1st arg is for agg return */
9026     if (isArgReturn) {
9027         AArch64reg reg = static_cast<AArch64reg>(R8);
9028         RegOperand *res = &GetOrCreatePhysicalRegisterOperand(reg, GetPrimTypeSize(PTY_a64) * kBitsPerByte, kRegTyInt);
9029         SelectCopy(*res, PTY_a64, *args[0], PTY_a64);
9030         srcOpnds->PushOpnd(*res);
9031     }
9032     ResetLmbcArgInfo(); /* reset */
9033     ResetLmbcArgsInRegs();
9034     ResetLmbcTotalArgs();
9035 }
9036 
SelectCall(CallNode & callNode)9037 void AArch64CGFunc::SelectCall(CallNode &callNode)
9038 {
9039     MIRFunction *fn = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(callNode.GetPUIdx());
9040     MIRSymbol *fsym = GetFunction().GetLocalOrGlobalSymbol(fn->GetStIdx(), false);
9041     MIRType *retType = fn->GetReturnType();
9042 
9043     if (GetCG()->GenerateVerboseCG()) {
9044         const std::string &comment = fsym->GetName();
9045         GetCurBB()->AppendInsn(CreateCommentInsn(comment));
9046     }
9047 
9048     ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
9049     if (GetMirModule().GetFlavor() == MIRFlavor::kFlavorLmbc) {
9050         SetLmbcCallReturnType(nullptr);
9051         bool largeStructRet = false;
9052         if (fn->IsFirstArgReturn()) {
9053             MIRPtrType *ptrTy = static_cast<MIRPtrType *>(
9054                 GlobalTables::GetTypeTable().GetTypeFromTyIdx(fn->GetFormalDefVec()[0].formalTyIdx));
9055             MIRType *sTy = GlobalTables::GetTypeTable().GetTypeFromTyIdx(ptrTy->GetPointedTyIdx());
9056             largeStructRet = sTy->GetSize() > k16ByteSize;
9057             SetLmbcCallReturnType(sTy);
9058         } else {
9059             MIRType *ty = fn->GetReturnType();
9060             SetLmbcCallReturnType(ty);
9061         }
9062         LmbcSelectParmList(srcOpnds, largeStructRet);
9063     }
9064 
9065     SelectParmListWrapper(callNode, *srcOpnds, false);
9066 
9067     const std::string &funcName = fsym->GetName();
9068     if (Globals::GetInstance()->GetOptimLevel() >= CGOptions::kLevel2 &&
9069         funcName == "Ljava_2Flang_2FString_3B_7CindexOf_7C_28Ljava_2Flang_2FString_3B_29I") {
9070         GStrIdx strIdx = GlobalTables::GetStrTable().GetStrIdxFromName(funcName);
9071         MIRSymbol *st = GlobalTables::GetGsymTable().GetSymbolFromStrIdx(strIdx, true);
9072         IntrinsifyStringIndexOf(*srcOpnds, *st);
9073         return;
9074     }
9075 
9076     Insn &callInsn = AppendCall(*fsym, *srcOpnds);
9077     GetCurBB()->SetHasCall();
9078     if (retType != nullptr) {
9079         callInsn.SetRetSize(static_cast<uint32>(retType->GetSize()));
9080         callInsn.SetIsCallReturnUnsigned(IsUnsignedInteger(retType->GetPrimType()));
9081     }
9082     const auto &deoptBundleInfo = callNode.GetDeoptBundleInfo();
9083     for (const auto &elem : deoptBundleInfo) {
9084         auto valueKind = elem.second.GetMapleValueKind();
9085         if (valueKind == MapleValue::kPregKind) {
9086             auto *opnd = GetOpndFromPregIdx(elem.second.GetPregIdx());
9087             CHECK_FATAL(opnd != nullptr, "pregIdx has not been assigned Operand");
9088             callInsn.AddDeoptBundleInfo(elem.first, *opnd);
9089         } else if (valueKind == MapleValue::kConstKind) {
9090             auto *opnd = SelectIntConst(static_cast<const MIRIntConst &>(elem.second.GetConstValue()));
9091             callInsn.AddDeoptBundleInfo(elem.first, *opnd);
9092         } else {
9093             CHECK_FATAL(false, "not supported currently");
9094         }
9095     }
9096     AppendStackMapInsn(callInsn);
9097 
9098     /* check if this call use stack slot to return */
9099     if (fn->IsFirstArgReturn()) {
9100         SetStackProtectInfo(kRetureStackSlot);
9101     }
9102 
9103     GetFunction().SetHasCall();
9104     if (GetMirModule().IsCModule()) { /* do not mark abort BB in C at present */
9105         if (fsym->GetName() == "__builtin_unreachable") {
9106             GetCurBB()->ClearInsns();
9107             GetCurBB()->SetUnreachable(true);
9108         }
9109         return;
9110     }
9111 }
9112 
SelectIcall(IcallNode & icallNode,Operand & srcOpnd)9113 void AArch64CGFunc::SelectIcall(IcallNode &icallNode, Operand &srcOpnd)
9114 {
9115     ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
9116     SelectParmListWrapper(icallNode, *srcOpnds, false);
9117 
9118     Operand *fptrOpnd = &srcOpnd;
9119     if (fptrOpnd->GetKind() != Operand::kOpdRegister) {
9120         PrimType ty = icallNode.Opnd(0)->GetPrimType();
9121         fptrOpnd = &SelectCopy(srcOpnd, ty, ty);
9122     }
9123     DEBUG_ASSERT(fptrOpnd->IsRegister(), "SelectIcall: function pointer not RegOperand");
9124     RegOperand *regOpnd = static_cast<RegOperand *>(fptrOpnd);
9125     Insn &callInsn = GetInsnBuilder()->BuildInsn(MOP_xblr, *regOpnd, *srcOpnds);
9126 
9127     MIRType *retType = icallNode.GetCallReturnType();
9128     if (retType != nullptr) {
9129         callInsn.SetRetSize(static_cast<uint32>(retType->GetSize()));
9130         callInsn.SetIsCallReturnUnsigned(IsUnsignedInteger(retType->GetPrimType()));
9131     }
9132 
9133     /* check if this icall use stack slot to return */
9134     CallReturnVector *p2nrets = &icallNode.GetReturnVec();
9135     if (p2nrets->size() == k1ByteSize) {
9136         StIdx stIdx = (*p2nrets)[0].first;
9137         MIRSymbol *sym = GetBecommon().GetMIRModule().CurFunction()->GetSymTab()->GetSymbolFromStIdx(stIdx.Idx());
9138         if (sym != nullptr && (GetBecommon().GetTypeSize(sym->GetTyIdx().GetIdx()) > k16ByteSize)) {
9139             SetStackProtectInfo(kRetureStackSlot);
9140         }
9141     }
9142 
9143     GetCurBB()->AppendInsn(callInsn);
9144     GetCurBB()->SetHasCall();
9145     DEBUG_ASSERT(GetCurBB()->GetLastInsn()->IsCall(), "lastInsn should be a call");
9146     GetFunction().SetHasCall();
9147     const auto &deoptBundleInfo = icallNode.GetDeoptBundleInfo();
9148     for (const auto &elem : deoptBundleInfo) {
9149         auto valueKind = elem.second.GetMapleValueKind();
9150         if (valueKind == MapleValue::kPregKind) {
9151             auto *opnd = GetOpndFromPregIdx(elem.second.GetPregIdx());
9152             CHECK_FATAL(opnd != nullptr, "pregIdx has not been assigned Operand");
9153             callInsn.AddDeoptBundleInfo(elem.first, *opnd);
9154         } else if (valueKind == MapleValue::kConstKind) {
9155             auto *opnd = SelectIntConst(static_cast<const MIRIntConst &>(elem.second.GetConstValue()));
9156             callInsn.AddDeoptBundleInfo(elem.first, *opnd);
9157         } else {
9158             CHECK_FATAL(false, "not supported currently");
9159         }
9160     }
9161     AppendStackMapInsn(callInsn);
9162 }
9163 
HandleCatch()9164 void AArch64CGFunc::HandleCatch()
9165 {
9166     if (Globals::GetInstance()->GetOptimLevel() >= CGOptions::kLevel1) {
9167         regno_t regNO = uCatch.regNOCatch;
9168         RegOperand &vregOpnd = GetOrCreateVirtualRegisterOperand(regNO);
9169         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(
9170             MOP_xmovrr, vregOpnd, GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, kRegTyInt)));
9171     } else {
9172         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(
9173             PickStInsn(uCatch.opndCatch->GetSize(), PTY_a64),
9174             GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, kRegTyInt), *uCatch.opndCatch));
9175     }
9176 }
9177 
SelectMembar(StmtNode & membar)9178 void AArch64CGFunc::SelectMembar(StmtNode &membar)
9179 {
9180     switch (membar.GetOpCode()) {
9181         case OP_membaracquire:
9182             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_dmb_ishld, AArch64CG::kMd[MOP_dmb_ishld]));
9183             break;
9184         case OP_membarrelease:
9185         case OP_membarstoreload:
9186             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_dmb_ish, AArch64CG::kMd[MOP_dmb_ish]));
9187             break;
9188         case OP_membarstorestore:
9189             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_dmb_ishst, AArch64CG::kMd[MOP_dmb_ishst]));
9190             break;
9191         default:
9192             DEBUG_ASSERT(false, "NYI");
9193             break;
9194     }
9195 }
9196 
SelectComment(CommentNode & comment)9197 void AArch64CGFunc::SelectComment(CommentNode &comment)
9198 {
9199     GetCurBB()->AppendInsn(CreateCommentInsn(comment.GetComment()));
9200 }
9201 
SelectReturn(Operand * opnd0)9202 void AArch64CGFunc::SelectReturn(Operand *opnd0)
9203 {
9204     bool is64x1vec = GetFunction().GetAttr(FUNCATTR_oneelem_simd) ? true : false;
9205     MIRType *floatType = GlobalTables::GetTypeTable().GetDouble();
9206     MIRType *retTyp = is64x1vec ? floatType : GetFunction().GetReturnType();
9207     CCImpl &retLocator = *GetOrCreateLocator(GetCurCallConvKind());
9208     CCLocInfo retMech;
9209     retLocator.InitReturnInfo(*retTyp, retMech);
9210     if ((retMech.GetRegCount() > 0) && (opnd0 != nullptr)) {
9211         RegType regTyp = is64x1vec ? kRegTyFloat : GetRegTyFromPrimTy(retMech.GetPrimTypeOfReg0());
9212         PrimType oriPrimType = is64x1vec ? GetFunction().GetReturnType()->GetPrimType() : retMech.GetPrimTypeOfReg0();
9213         AArch64reg retReg = static_cast<AArch64reg>(retMech.GetReg0());
9214         if (opnd0->IsRegister()) {
9215             RegOperand *regOpnd = static_cast<RegOperand *>(opnd0);
9216             if (regOpnd->GetRegisterNumber() != retMech.GetReg0()) {
9217                 RegOperand &retOpnd = GetOrCreatePhysicalRegisterOperand(retReg, regOpnd->GetSize(), regTyp);
9218                 SelectCopy(retOpnd, retMech.GetPrimTypeOfReg0(), *regOpnd, oriPrimType);
9219             }
9220         } else if (opnd0->IsMemoryAccessOperand()) {
9221             auto *memopnd = static_cast<MemOperand *>(opnd0);
9222             RegOperand &retOpnd =
9223                 GetOrCreatePhysicalRegisterOperand(retReg, GetPrimTypeBitSize(retMech.GetPrimTypeOfReg0()), regTyp);
9224             MOperator mOp = PickLdInsn(memopnd->GetSize(), retMech.GetPrimTypeOfReg0());
9225             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, retOpnd, *memopnd));
9226         } else if (opnd0->IsConstImmediate()) {
9227             ImmOperand *immOpnd = static_cast<ImmOperand *>(opnd0);
9228             if (!is64x1vec) {
9229                 RegOperand &retOpnd =
9230                     GetOrCreatePhysicalRegisterOperand(retReg, GetPrimTypeBitSize(retMech.GetPrimTypeOfReg0()),
9231                                                        GetRegTyFromPrimTy(retMech.GetPrimTypeOfReg0()));
9232                 SelectCopy(retOpnd, retMech.GetPrimTypeOfReg0(), *immOpnd, retMech.GetPrimTypeOfReg0());
9233             } else {
9234                 PrimType rType = GetFunction().GetReturnType()->GetPrimType();
9235                 RegOperand *reg = &CreateRegisterOperandOfType(rType);
9236                 SelectCopy(*reg, rType, *immOpnd, rType);
9237                 RegOperand &retOpnd = GetOrCreatePhysicalRegisterOperand(retReg, GetPrimTypeBitSize(PTY_f64),
9238                                                                          GetRegTyFromPrimTy(PTY_f64));
9239                 Insn &insn = GetInsnBuilder()->BuildInsn(MOP_xvmovdr, retOpnd, *reg);
9240                 GetCurBB()->AppendInsn(insn);
9241             }
9242         } else {
9243             CHECK_FATAL(false, "nyi");
9244         }
9245     }
9246     GetExitBBsVec().emplace_back(GetCurBB());
9247 }
9248 
GetOrCreateSpecialRegisterOperand(PregIdx sregIdx,PrimType primType)9249 RegOperand &AArch64CGFunc::GetOrCreateSpecialRegisterOperand(PregIdx sregIdx, PrimType primType)
9250 {
9251     AArch64reg reg = R0;
9252     switch (sregIdx) {
9253         case kSregSp:
9254             reg = RSP;
9255             break;
9256         case kSregFp:
9257             reg = RFP;
9258             break;
9259         case kSregGp: {
9260             MIRSymbol *sym = GetCG()->GetGP();
9261             if (sym == nullptr) {
9262                 sym = GetFunction().GetSymTab()->CreateSymbol(kScopeLocal);
9263                 std::string strBuf("__file__local__GP");
9264                 sym->SetNameStrIdx(GetMirModule().GetMIRBuilder()->GetOrCreateStringIndex(strBuf));
9265                 GetCG()->SetGP(sym);
9266             }
9267             RegOperand &result = GetOrCreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
9268             SelectAddrof(result, CreateStImmOperand(*sym, 0, 0));
9269             return result;
9270         }
9271         case kSregThrownval: { /* uses x0 == R0 */
9272             DEBUG_ASSERT(uCatch.regNOCatch > 0, "regNOCatch should greater than 0.");
9273             if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) {
9274                 RegOperand &regOpnd = GetOrCreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8BitSize));
9275                 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(uCatch.opndCatch->GetSize(), PTY_a64),
9276                                                                    regOpnd, *uCatch.opndCatch));
9277                 return regOpnd;
9278             } else {
9279                 return GetOrCreateVirtualRegisterOperand(uCatch.regNOCatch);
9280             }
9281         }
9282         case kSregRetval0:
9283             if (!IsPrimitiveInteger(primType) || IsPrimitiveVectorFloat(primType)) {
9284                 reg = V0;
9285             }
9286             break;
9287         case kSregMethodhdl:
9288             if (methodHandleVreg == regno_t(-1)) {
9289                 methodHandleVreg = NewVReg(kRegTyInt, k8BitSize);
9290             }
9291             return GetOrCreateVirtualRegisterOperand(methodHandleVreg);
9292         default:
9293             DEBUG_ASSERT(false, "Special pseudo registers NYI");
9294             break;
9295     }
9296     return GetOrCreatePhysicalRegisterOperand(reg, k64BitSize, kRegTyInt);
9297 }
9298 
GetOrCreatePhysicalRegisterOperand(std::string & asmAttr)9299 RegOperand &AArch64CGFunc::GetOrCreatePhysicalRegisterOperand(std::string &asmAttr)
9300 {
9301     DEBUG_ASSERT(!asmAttr.empty(), "Get inline asm string failed in GetOrCreatePhysicalRegisterOperand");
9302     RegType rKind = kRegTyUndef;
9303     uint32 rSize = 0;
9304     /* Get Register Type and Size */
9305     switch (asmAttr[0]) {
9306         case 'x': {
9307             rKind = kRegTyInt;
9308             rSize = k64BitSize;
9309             break;
9310         }
9311         case 'w': {
9312             rKind = kRegTyInt;
9313             rSize = k32BitSize;
9314             break;
9315         }
9316         default: {
9317             LogInfo::MapleLogger() << "Unsupport asm string : " << asmAttr << "\n";
9318             CHECK_FATAL(false, "Have not support this kind of register ");
9319         }
9320     }
9321     AArch64reg rNO = kRinvalid;
9322     /* Get Register Number */
9323     uint32 regNumPos = 1;
9324     char numberChar = asmAttr[regNumPos++];
9325     if (numberChar >= '0' && numberChar <= '9') {
9326         uint32 val = static_cast<uint32>(numberChar - '0');
9327         if (regNumPos < asmAttr.length()) {
9328             char numberCharSecond = asmAttr[regNumPos++];
9329             DEBUG_ASSERT(regNumPos == asmAttr.length(), "Invalid asm attribute");
9330             if (numberCharSecond >= '0' && numberCharSecond <= '9') {
9331                 val = val * kDecimalMax + static_cast<uint32>((numberCharSecond - '0'));
9332             }
9333         }
9334         rNO = static_cast<AArch64reg>(static_cast<uint32>(R0) + val);
9335         if (val > (kAsmInputRegPrefixOpnd + 1)) {
9336             LogInfo::MapleLogger() << "Unsupport asm string : " << asmAttr << "\n";
9337             CHECK_FATAL(false, "have not support this kind of register ");
9338         }
9339     } else if (numberChar == 0) {
9340         return CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
9341     } else {
9342         CHECK_FATAL(false, "Unexpect input in GetOrCreatePhysicalRegisterOperand");
9343     }
9344     return GetOrCreatePhysicalRegisterOperand(rNO, rSize, rKind);
9345 }
9346 
GetOrCreatePhysicalRegisterOperand(AArch64reg regNO,uint32 size,RegType kind,uint32 flag)9347 RegOperand &AArch64CGFunc::GetOrCreatePhysicalRegisterOperand(AArch64reg regNO, uint32 size, RegType kind, uint32 flag)
9348 {
9349     uint64 aarch64PhyRegIdx = regNO;
9350     DEBUG_ASSERT(flag == 0, "Do not expect flag here");
9351     if (size <= k32BitSize) {
9352         size = k32BitSize;
9353         aarch64PhyRegIdx = aarch64PhyRegIdx << 1;
9354     } else if (size <= k64BitSize) {
9355         size = k64BitSize;
9356         aarch64PhyRegIdx = (aarch64PhyRegIdx << 1) + 1;
9357     } else {
9358         size = (size == k128BitSize) ? k128BitSize : k64BitSize;
9359         aarch64PhyRegIdx = aarch64PhyRegIdx << k4BitShift;
9360     }
9361     RegOperand *phyRegOpnd = nullptr;
9362     auto phyRegIt = phyRegOperandTable.find(aarch64PhyRegIdx);
9363     if (phyRegIt != phyRegOperandTable.end()) {
9364         phyRegOpnd = phyRegOperandTable[aarch64PhyRegIdx];
9365     } else {
9366         phyRegOpnd = memPool->New<RegOperand>(regNO, size, kind, flag);
9367         phyRegOperandTable.emplace(aarch64PhyRegIdx, phyRegOpnd);
9368     }
9369     return *phyRegOpnd;
9370 }
9371 
GetLabelOperand(LabelIdx labIdx) const9372 const LabelOperand *AArch64CGFunc::GetLabelOperand(LabelIdx labIdx) const
9373 {
9374     const MapleUnorderedMap<LabelIdx, LabelOperand *>::const_iterator it = hashLabelOpndTable.find(labIdx);
9375     if (it != hashLabelOpndTable.end()) {
9376         return it->second;
9377     }
9378     return nullptr;
9379 }
9380 
GetOrCreateLabelOperand(LabelIdx labIdx)9381 LabelOperand &AArch64CGFunc::GetOrCreateLabelOperand(LabelIdx labIdx)
9382 {
9383     MapleUnorderedMap<LabelIdx, LabelOperand *>::iterator it = hashLabelOpndTable.find(labIdx);
9384     if (it != hashLabelOpndTable.end()) {
9385         return *(it->second);
9386     }
9387     const char *funcName = GetShortFuncName().c_str();
9388     LabelOperand *res = memPool->New<LabelOperand>(funcName, labIdx);
9389     hashLabelOpndTable[labIdx] = res;
9390     return *res;
9391 }
9392 
GetOrCreateLabelOperand(BB & bb)9393 LabelOperand &AArch64CGFunc::GetOrCreateLabelOperand(BB &bb)
9394 {
9395     LabelIdx labelIdx = bb.GetLabIdx();
9396     if (labelIdx == MIRLabelTable::GetDummyLabel()) {
9397         labelIdx = CreateLabel();
9398         bb.AddLabel(labelIdx);
9399         SetLab2BBMap(labelIdx, bb);
9400     }
9401     return GetOrCreateLabelOperand(labelIdx);
9402 }
9403 
GetAggCopySize(uint32 offset1,uint32 offset2,uint32 alignment) const9404 uint32 AArch64CGFunc::GetAggCopySize(uint32 offset1, uint32 offset2, uint32 alignment) const
9405 {
9406     /* Generating a larger sized mem op than alignment if allowed by aggregate starting address */
9407     uint32 offsetAlign1 = (offset1 == 0) ? k8ByteSize : offset1;
9408     uint32 offsetAlign2 = (offset2 == 0) ? k8ByteSize : offset2;
9409     uint32 alignOffset =
9410         1U << (std::min(__builtin_ffs(static_cast<int>(offsetAlign1)), __builtin_ffs(static_cast<int>(offsetAlign2))) -
9411                1);
9412     if (alignOffset == k8ByteSize || alignOffset == k4ByteSize || alignOffset == k2ByteSize) {
9413         return alignOffset;
9414     } else if (alignOffset > k8ByteSize) {
9415         return k8ByteSize;
9416     } else {
9417         return alignment;
9418     }
9419 }
9420 
GetOrCreateOfstOpnd(uint64 offset,uint32 size)9421 OfstOperand &AArch64CGFunc::GetOrCreateOfstOpnd(uint64 offset, uint32 size)
9422 {
9423     uint64 aarch64OfstRegIdx = offset;
9424     aarch64OfstRegIdx = (aarch64OfstRegIdx << 1);
9425     if (size == k64BitSize) {
9426         ++aarch64OfstRegIdx;
9427     }
9428     DEBUG_ASSERT(size == k32BitSize || size == k64BitSize, "ofStOpnd size check");
9429     auto it = hashOfstOpndTable.find(aarch64OfstRegIdx);
9430     if (it != hashOfstOpndTable.end()) {
9431         return *it->second;
9432     }
9433     OfstOperand *res = &CreateOfstOpnd(offset, size);
9434     hashOfstOpndTable[aarch64OfstRegIdx] = res;
9435     return *res;
9436 }
9437 
SelectAddrofAfterRa(Operand & result,StImmOperand & stImm,std::vector<Insn * > & rematInsns)9438 void AArch64CGFunc::SelectAddrofAfterRa(Operand &result, StImmOperand &stImm, std::vector<Insn *> &rematInsns)
9439 {
9440     const MIRSymbol *symbol = stImm.GetSymbol();
9441     DEBUG_ASSERT((symbol->GetStorageClass() != kScAuto) || (symbol->GetStorageClass() != kScFormal), "");
9442     Operand *srcOpnd = &result;
9443     rematInsns.emplace_back(&GetInsnBuilder()->BuildInsn(MOP_xadrp, result, stImm));
9444     if (CGOptions::IsPIC() && symbol->NeedPIC()) {
9445         /* ldr     x0, [x0, #:got_lo12:Ljava_2Flang_2FSystem_3B_7Cout] */
9446         OfstOperand &offset = CreateOfstOpnd(*stImm.GetSymbol(), stImm.GetOffset(), stImm.GetRelocs());
9447         MemOperand &memOpnd = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, GetPointerSize() * kBitsPerByte,
9448                                                  static_cast<RegOperand *>(srcOpnd), nullptr, &offset, nullptr);
9449         rematInsns.emplace_back(
9450             &GetInsnBuilder()->BuildInsn(memOpnd.GetSize() == k64BitSize ? MOP_xldr : MOP_wldr, result, memOpnd));
9451 
9452         if (stImm.GetOffset() > 0) {
9453             ImmOperand &immOpnd = CreateImmOperand(stImm.GetOffset(), result.GetSize(), false);
9454             rematInsns.emplace_back(&GetInsnBuilder()->BuildInsn(MOP_xaddrri12, result, result, immOpnd));
9455             return;
9456         }
9457     } else {
9458         rematInsns.emplace_back(&GetInsnBuilder()->BuildInsn(MOP_xadrpl12, result, *srcOpnd, stImm));
9459     }
9460 }
9461 
GetOrCreateMemOpndAfterRa(const MIRSymbol & symbol,int32 offset,uint32 size,bool needLow12,RegOperand * regOp,std::vector<Insn * > & rematInsns)9462 MemOperand &AArch64CGFunc::GetOrCreateMemOpndAfterRa(const MIRSymbol &symbol, int32 offset, uint32 size, bool needLow12,
9463                                                      RegOperand *regOp, std::vector<Insn *> &rematInsns)
9464 {
9465     MIRStorageClass storageClass = symbol.GetStorageClass();
9466     if ((storageClass == kScGlobal) || (storageClass == kScExtern)) {
9467         StImmOperand &stOpnd = CreateStImmOperand(symbol, offset, 0);
9468         RegOperand &stAddrOpnd = *regOp;
9469         SelectAddrofAfterRa(stAddrOpnd, stOpnd, rematInsns);
9470         /* MemOperand::AddrMode_B_OI */
9471         return *CreateMemOperand(MemOperand::kAddrModeBOi, size, stAddrOpnd, nullptr,
9472                                  &GetOrCreateOfstOpnd(0, k32BitSize), &symbol);
9473     } else if ((storageClass == kScPstatic) || (storageClass == kScFstatic)) {
9474         if (symbol.GetSKind() == kStConst) {
9475             DEBUG_ASSERT(offset == 0, "offset should be 0 for constant literals");
9476             return *CreateMemOperand(MemOperand::kAddrModeLiteral, size, symbol);
9477         } else {
9478             if (needLow12) {
9479                 StImmOperand &stOpnd = CreateStImmOperand(symbol, offset, 0);
9480                 RegOperand &stAddrOpnd = *regOp;
9481                 SelectAddrofAfterRa(stAddrOpnd, stOpnd, rematInsns);
9482                 return *CreateMemOperand(MemOperand::kAddrModeBOi, size, stAddrOpnd, nullptr,
9483                                          &GetOrCreateOfstOpnd(0, k32BitSize), &symbol);
9484             } else {
9485                 StImmOperand &stOpnd = CreateStImmOperand(symbol, offset, 0);
9486                 RegOperand &stAddrOpnd = *regOp;
9487                 /* adrp    x1, _PTR__cinf_Ljava_2Flang_2FSystem_3B */
9488                 Insn &insn = GetInsnBuilder()->BuildInsn(MOP_xadrp, stAddrOpnd, stOpnd);
9489                 rematInsns.emplace_back(&insn);
9490                 /* ldr     x1, [x1, #:lo12:_PTR__cinf_Ljava_2Flang_2FSystem_3B] */
9491                 return *CreateMemOperand(MemOperand::kAddrModeLo12Li, size, stAddrOpnd, nullptr,
9492                                          &GetOrCreateOfstOpnd(static_cast<uint64>(offset), k32BitSize), &symbol);
9493             }
9494         }
9495     } else {
9496         CHECK_FATAL(false, "NYI");
9497     }
9498 }
9499 
GetOrCreateMemOpnd(const MIRSymbol & symbol,int64 offset,uint32 size,bool forLocalRef,bool needLow12,RegOperand * regOp)9500 MemOperand &AArch64CGFunc::GetOrCreateMemOpnd(const MIRSymbol &symbol, int64 offset, uint32 size, bool forLocalRef,
9501                                               bool needLow12, RegOperand *regOp)
9502 {
9503     MIRStorageClass storageClass = symbol.GetStorageClass();
9504     if ((storageClass == kScAuto) || (storageClass == kScFormal)) {
9505         AArch64SymbolAlloc *symLoc =
9506             static_cast<AArch64SymbolAlloc *>(GetMemlayout()->GetSymAllocInfo(symbol.GetStIndex()));
9507         if (forLocalRef) {
9508             auto p = GetMemlayout()->GetLocalRefLocMap().find(symbol.GetStIdx());
9509             CHECK_FATAL(p != GetMemlayout()->GetLocalRefLocMap().end(), "sym loc should have been defined");
9510             symLoc = static_cast<AArch64SymbolAlloc *>(p->second);
9511         }
9512         DEBUG_ASSERT(symLoc != nullptr, "sym loc should have been defined");
9513         /* At this point, we don't know which registers the callee needs to save. */
9514         DEBUG_ASSERT((IsFPLRAddedToCalleeSavedList() || (SizeOfCalleeSaved() == 0)),
9515                      "CalleeSaved won't be known until after Register Allocation");
9516         StIdx idx = symbol.GetStIdx();
9517         auto it = memOpndsRequiringOffsetAdjustment.find(idx);
9518         DEBUG_ASSERT((!IsFPLRAddedToCalleeSavedList() ||
9519                       ((it != memOpndsRequiringOffsetAdjustment.end()) || (storageClass == kScFormal))),
9520                      "Memory operand of this symbol should have been added to the hash table");
9521         int32 stOffset = GetBaseOffset(*symLoc);
9522         if (it != memOpndsRequiringOffsetAdjustment.end()) {
9523             if (GetMemlayout()->IsLocalRefLoc(symbol)) {
9524                 if (!forLocalRef) {
9525                     return *(it->second);
9526                 }
9527             } else if (mirModule.IsJavaModule()) {
9528                 return *(it->second);
9529             } else {
9530                 Operand *offOpnd = (it->second)->GetOffset();
9531                 if (((static_cast<OfstOperand *>(offOpnd))->GetOffsetValue() == (stOffset + offset)) &&
9532                     (it->second->GetSize() == size)) {
9533                     return *(it->second);
9534                 }
9535             }
9536         }
9537         it = memOpndsForStkPassedArguments.find(idx);
9538         if (it != memOpndsForStkPassedArguments.end()) {
9539             if (GetMemlayout()->IsLocalRefLoc(symbol)) {
9540                 if (!forLocalRef) {
9541                     return *(it->second);
9542                 }
9543             } else {
9544                 return *(it->second);
9545             }
9546         }
9547 
9548         RegOperand *baseOpnd = static_cast<RegOperand *>(GetBaseReg(*symLoc));
9549         int32 totalOffset = stOffset + static_cast<int32>(offset);
9550         /* needs a fresh copy of ImmOperand as we may adjust its offset at a later stage. */
9551         OfstOperand *offsetOpnd = nullptr;
9552         if (CGOptions::IsBigEndian()) {
9553             if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsArgsStkPassed && size < k64BitSize) {
9554                 offsetOpnd = &CreateOfstOpnd(k4BitSize + static_cast<uint32>(totalOffset), k64BitSize);
9555             } else {
9556                 offsetOpnd = &CreateOfstOpnd(static_cast<uint64>(static_cast<int64>(totalOffset)), k64BitSize);
9557             }
9558         } else {
9559             offsetOpnd = &CreateOfstOpnd(static_cast<uint64>(static_cast<int64>(totalOffset)), k64BitSize);
9560         }
9561         if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsArgsStkPassed &&
9562             MemOperand::IsPIMMOffsetOutOfRange(totalOffset, size)) {
9563             ImmOperand *offsetOprand;
9564             offsetOprand = &CreateImmOperand(totalOffset, k64BitSize, true, kUnAdjustVary);
9565             Operand *resImmOpnd = &SelectCopy(*offsetOprand, PTY_i64, PTY_i64);
9566             return *CreateMemOperand(MemOperand::kAddrModeBOrX, size, *baseOpnd, static_cast<RegOperand &>(*resImmOpnd),
9567                                      nullptr, symbol, true);
9568         } else {
9569             if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsArgsStkPassed) {
9570                 offsetOpnd->SetVary(kUnAdjustVary);
9571             }
9572             MemOperand *res = CreateMemOperand(MemOperand::kAddrModeBOi, size, *baseOpnd, nullptr, offsetOpnd, &symbol);
9573             if ((symbol.GetType()->GetKind() != kTypeClass) && !forLocalRef) {
9574                 memOpndsRequiringOffsetAdjustment[idx] = res;
9575             }
9576             return *res;
9577         }
9578     } else if ((storageClass == kScGlobal) || (storageClass == kScExtern)) {
9579         StImmOperand &stOpnd = CreateStImmOperand(symbol, offset, 0);
9580         if (!regOp) {
9581             regOp = static_cast<RegOperand *>(&CreateRegisterOperandOfType(PTY_u64));
9582         }
9583         RegOperand &stAddrOpnd = *regOp;
9584         SelectAddrof(stAddrOpnd, stOpnd);
9585         /* MemOperand::AddrMode_B_OI */
9586         return *CreateMemOperand(MemOperand::kAddrModeBOi, size, stAddrOpnd, nullptr,
9587                                  &GetOrCreateOfstOpnd(0, k32BitSize), &symbol);
9588     } else if ((storageClass == kScPstatic) || (storageClass == kScFstatic)) {
9589         if (symbol.GetSKind() == kStConst) {
9590             DEBUG_ASSERT(offset == 0, "offset should be 0 for constant literals");
9591             return *CreateMemOperand(MemOperand::kAddrModeLiteral, size, symbol);
9592         } else {
9593             /* not guaranteed align for uninitialized symbol */
9594             if (needLow12 || (!symbol.IsConst() && CGOptions::IsPIC())) {
9595                 StImmOperand &stOpnd = CreateStImmOperand(symbol, offset, 0);
9596                 if (!regOp) {
9597                     regOp = static_cast<RegOperand *>(&CreateRegisterOperandOfType(PTY_u64));
9598                 }
9599                 RegOperand &stAddrOpnd = *regOp;
9600                 SelectAddrof(stAddrOpnd, stOpnd);
9601                 return *CreateMemOperand(MemOperand::kAddrModeBOi, size, stAddrOpnd, nullptr,
9602                                          &GetOrCreateOfstOpnd(0, k32BitSize), &symbol);
9603             } else {
9604                 StImmOperand &stOpnd = CreateStImmOperand(symbol, offset, 0);
9605                 if (!regOp) {
9606                     regOp = static_cast<RegOperand *>(&CreateRegisterOperandOfType(PTY_u64));
9607                 }
9608                 RegOperand &stAddrOpnd = *regOp;
9609                 /* adrp    x1, _PTR__cinf_Ljava_2Flang_2FSystem_3B */
9610                 Insn &insn = GetInsnBuilder()->BuildInsn(MOP_xadrp, stAddrOpnd, stOpnd);
9611                 GetCurBB()->AppendInsn(insn);
9612                 /* ldr     x1, [x1, #:lo12:_PTR__cinf_Ljava_2Flang_2FSystem_3B] */
9613                 return *CreateMemOperand(MemOperand::kAddrModeLo12Li, size, stAddrOpnd, nullptr,
9614                                          &GetOrCreateOfstOpnd(static_cast<uint64>(offset), k32BitSize), &symbol);
9615             }
9616         }
9617     } else {
9618         CHECK_FATAL(false, "NYI");
9619     }
9620 }
9621 
HashMemOpnd(MemOperand & tMemOpnd)9622 MemOperand &AArch64CGFunc::HashMemOpnd(MemOperand &tMemOpnd)
9623 {
9624     auto it = hashMemOpndTable.find(tMemOpnd);
9625     if (it != hashMemOpndTable.end()) {
9626         return *(it->second);
9627     }
9628     auto *res = memPool->New<MemOperand>(tMemOpnd);
9629     hashMemOpndTable[tMemOpnd] = res;
9630     return *res;
9631 }
9632 
GetOrCreateMemOpnd(MemOperand::AArch64AddressingMode mode,uint32 size,RegOperand * base,RegOperand * index,ImmOperand * offset,const MIRSymbol * st)9633 MemOperand &AArch64CGFunc::GetOrCreateMemOpnd(MemOperand::AArch64AddressingMode mode, uint32 size, RegOperand *base,
9634                                               RegOperand *index, ImmOperand *offset, const MIRSymbol *st)
9635 {
9636     DEBUG_ASSERT(base != nullptr, "nullptr check");
9637     MemOperand tMemOpnd(mode, size, *base, index, offset, st);
9638     if (base->GetRegisterNumber() == RFP || base->GetRegisterNumber() == RSP) {
9639         tMemOpnd.SetStackMem(true);
9640     }
9641     return HashMemOpnd(tMemOpnd);
9642 }
9643 
GetOrCreateMemOpnd(MemOperand::AArch64AddressingMode mode,uint32 size,RegOperand * base,RegOperand * index,int32 shift,bool isSigned)9644 MemOperand &AArch64CGFunc::GetOrCreateMemOpnd(MemOperand::AArch64AddressingMode mode, uint32 size, RegOperand *base,
9645                                               RegOperand *index, int32 shift, bool isSigned)
9646 {
9647     DEBUG_ASSERT(base != nullptr, "nullptr check");
9648     MemOperand tMemOpnd(mode, size, *base, *index, shift, isSigned);
9649     if (base->GetRegisterNumber() == RFP || base->GetRegisterNumber() == RSP) {
9650         tMemOpnd.SetStackMem(true);
9651     }
9652     return HashMemOpnd(tMemOpnd);
9653 }
9654 
GetOrCreateMemOpnd(MemOperand & oldMem)9655 MemOperand &AArch64CGFunc::GetOrCreateMemOpnd(MemOperand &oldMem)
9656 {
9657     return HashMemOpnd(oldMem);
9658 }
9659 
9660 /* offset: base offset from FP or SP */
CreateMemOpnd(RegOperand & baseOpnd,int64 offset,uint32 size)9661 MemOperand &AArch64CGFunc::CreateMemOpnd(RegOperand &baseOpnd, int64 offset, uint32 size)
9662 {
9663     OfstOperand &offsetOpnd = CreateOfstOpnd(static_cast<uint64>(offset), k32BitSize);
9664     /* do not need to check bit size rotate of sign immediate */
9665     bool checkSimm = (offset > kMinSimm64 && offset < kMaxSimm64Pair);
9666     if (!checkSimm && !ImmOperand::IsInBitSizeRot(kMaxImmVal12Bits, offset)) {
9667         Operand *resImmOpnd = &SelectCopy(CreateImmOperand(offset, k32BitSize, true), PTY_i32, PTY_i32);
9668         return *CreateMemOperand(MemOperand::kAddrModeBOrX, size, baseOpnd, static_cast<RegOperand *>(resImmOpnd),
9669                                  nullptr, nullptr);
9670     } else {
9671         return *CreateMemOperand(MemOperand::kAddrModeBOi, size, baseOpnd, nullptr, &offsetOpnd, nullptr);
9672     }
9673 }
9674 
9675 /* offset: base offset + #:lo12:Label+immediate */
CreateMemOpnd(RegOperand & baseOpnd,int64 offset,uint32 size,const MIRSymbol & sym)9676 MemOperand &AArch64CGFunc::CreateMemOpnd(RegOperand &baseOpnd, int64 offset, uint32 size, const MIRSymbol &sym)
9677 {
9678     OfstOperand &offsetOpnd = CreateOfstOpnd(static_cast<uint64>(offset), k32BitSize);
9679     DEBUG_ASSERT(ImmOperand::IsInBitSizeRot(kMaxImmVal12Bits, offset), "");
9680     return *CreateMemOperand(MemOperand::kAddrModeBOi, size, baseOpnd, nullptr, &offsetOpnd, &sym);
9681 }
9682 
GenStructParamIndex(RegOperand & base,const BaseNode & indexExpr,int shift,PrimType baseType,PrimType targetType)9683 RegOperand &AArch64CGFunc::GenStructParamIndex(RegOperand &base, const BaseNode &indexExpr, int shift,
9684                                                PrimType baseType, PrimType targetType)
9685 {
9686     RegOperand *index = &LoadIntoRegister(*HandleExpr(indexExpr, *(indexExpr.Opnd(0))), PTY_a64);
9687     RegOperand *srcOpnd = &CreateRegisterOperandOfType(PTY_a64);
9688     ImmOperand *imm = &CreateImmOperand(PTY_a64, shift);
9689     SelectShift(*srcOpnd, *index, *imm, kShiftLeft, PTY_a64);
9690     RegOperand *result = &CreateRegisterOperandOfType(PTY_a64);
9691     SelectAdd(*result, base, *srcOpnd, PTY_a64);
9692 
9693     OfstOperand *offopnd = &CreateOfstOpnd(0, k32BitSize);
9694     MemOperand &mo = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, result, nullptr, offopnd, nullptr);
9695     RegOperand &structAddr = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
9696     GetCurBB()->AppendInsn(
9697         GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(baseType), targetType), structAddr, mo));
9698     return structAddr;
9699 }
9700 
9701 /*
9702  *  case 1: iread a64 <* <* void>> 0 (add a64 (
9703  *  addrof a64 $__reg_jni_func_tab$$libcore_all_bytecode,
9704  *  mul a64 (
9705  *    cvt a64 i32 (constval i32 21),
9706  *    constval a64 8)))
9707  *
9708  * case 2 : iread u32 <* u8> 0 (add a64 (regread a64 %61, constval a64 3))
9709  * case 3 : iread u32 <* u8> 0 (add a64 (regread a64 %61, regread a64 %65))
9710  * case 4 : iread u32 <* u8> 0 (add a64 (cvt a64 i32(regread  %n)))
9711  */
CheckAndCreateExtendMemOpnd(PrimType ptype,const BaseNode & addrExpr,int64 offset,AArch64isa::MemoryOrdering memOrd)9712 MemOperand *AArch64CGFunc::CheckAndCreateExtendMemOpnd(PrimType ptype, const BaseNode &addrExpr, int64 offset,
9713                                                        AArch64isa::MemoryOrdering memOrd)
9714 {
9715     aggParamReg = nullptr;
9716     if (memOrd != AArch64isa::kMoNone || addrExpr.GetOpCode() != OP_add || offset != 0) {
9717         return nullptr;
9718     }
9719     BaseNode *baseExpr = addrExpr.Opnd(0);
9720     BaseNode *addendExpr = addrExpr.Opnd(1);
9721 
9722     if (baseExpr->GetOpCode() == OP_regread) {
9723         /* case 2 */
9724         if (addendExpr->GetOpCode() == OP_constval) {
9725             DEBUG_ASSERT(addrExpr.GetNumOpnds() == kOpndNum2, "Unepect expr operand in CheckAndCreateExtendMemOpnd");
9726             ConstvalNode *constOfstNode = static_cast<ConstvalNode *>(addendExpr);
9727             DEBUG_ASSERT(constOfstNode->GetConstVal()->GetKind() == kConstInt, "expect MIRIntConst");
9728             MIRIntConst *intOfst = safe_cast<MIRIntConst>(constOfstNode->GetConstVal());
9729             CHECK_FATAL(intOfst != nullptr, "just checking");
9730             /* discard large offset and negative offset */
9731             if (intOfst->GetExtValue() > INT32_MAX || intOfst->IsNegative()) {
9732                 return nullptr;
9733             }
9734             uint32 scale = static_cast<uint32>(intOfst->GetExtValue());
9735             OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(scale, k32BitSize);
9736             uint32 dsize = GetPrimTypeBitSize(ptype);
9737             MemOperand *memOpnd =
9738                 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, GetPrimTypeBitSize(ptype),
9739                                     SelectRegread(*static_cast<RegreadNode *>(baseExpr)), nullptr, &ofstOpnd, nullptr);
9740             return IsOperandImmValid(PickLdInsn(dsize, ptype), memOpnd, kInsnSecondOpnd) ? memOpnd : nullptr;
9741             /* case 3 */
9742         } else if (addendExpr->GetOpCode() == OP_regread) {
9743             CHECK_FATAL(addrExpr.GetNumOpnds() == kOpndNum2, "Unepect expr operand in CheckAndCreateExtendMemOpnd");
9744             if (GetPrimTypeSize(baseExpr->GetPrimType()) != GetPrimTypeSize(addendExpr->GetPrimType())) {
9745                 return nullptr;
9746             }
9747 
9748             auto *baseReg = SelectRegread(*static_cast<RegreadNode *>(baseExpr));
9749             auto *indexReg = SelectRegread(*static_cast<RegreadNode *>(addendExpr));
9750             MemOperand *memOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, GetPrimTypeBitSize(ptype), baseReg,
9751                                                       indexReg, nullptr, nullptr);
9752             return memOpnd;
9753             /* case 4 */
9754         } else if (addendExpr->GetOpCode() == OP_cvt && addendExpr->GetNumOpnds() == 1) {
9755             int shiftAmount = 0;
9756             BaseNode *cvtRegreadNode = addendExpr->Opnd(kInsnFirstOpnd);
9757             if (cvtRegreadNode->GetOpCode() == OP_regread && cvtRegreadNode->IsLeaf()) {
9758                 uint32 fromSize = GetPrimTypeBitSize(cvtRegreadNode->GetPrimType());
9759                 uint32 toSize = GetPrimTypeBitSize(addendExpr->GetPrimType());
9760 
9761                 if (toSize < fromSize) {
9762                     return nullptr;
9763                 }
9764 
9765                 MemOperand *memOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, GetPrimTypeBitSize(ptype),
9766                                                           SelectRegread(*static_cast<RegreadNode *>(baseExpr)),
9767                                                           SelectRegread(*static_cast<RegreadNode *>(cvtRegreadNode)),
9768                                                           shiftAmount, toSize != fromSize);
9769                 return memOpnd;
9770             }
9771         }
9772     }
9773     if (addendExpr->GetOpCode() != OP_mul || !IsPrimitiveInteger(ptype)) {
9774         return nullptr;
9775     }
9776     BaseNode *indexExpr, *scaleExpr;
9777     indexExpr = addendExpr->Opnd(0);
9778     scaleExpr = addendExpr->Opnd(1);
9779     if (scaleExpr->GetOpCode() != OP_constval) {
9780         return nullptr;
9781     }
9782     ConstvalNode *constValNode = static_cast<ConstvalNode *>(scaleExpr);
9783     CHECK_FATAL(constValNode->GetConstVal()->GetKind() == kConstInt, "expect MIRIntConst");
9784     MIRIntConst *mirIntConst = safe_cast<MIRIntConst>(constValNode->GetConstVal());
9785     CHECK_FATAL(mirIntConst != nullptr, "just checking");
9786     int32 scale = mirIntConst->GetExtValue();
9787     if (scale < 0) {
9788         return nullptr;
9789     }
9790     uint32 unsignedScale = static_cast<uint32>(scale);
9791     if (unsignedScale != GetPrimTypeSize(ptype) || indexExpr->GetOpCode() != OP_cvt) {
9792         return nullptr;
9793     }
9794     /* 8 is 1 << 3; 4 is 1 << 2; 2 is 1 << 1; 1 is 1 << 0 */
9795     int32 shift = (unsignedScale == 8) ? 3 : ((unsignedScale == 4) ? 2 : ((unsignedScale == 2) ? 1 : 0));
9796     RegOperand &base = static_cast<RegOperand &>(LoadIntoRegister(*HandleExpr(addrExpr, *baseExpr), PTY_a64));
9797     TypeCvtNode *typeCvtNode = static_cast<TypeCvtNode *>(indexExpr);
9798     PrimType fromType = typeCvtNode->FromType();
9799     PrimType toType = typeCvtNode->GetPrimType();
9800     if (isAggParamInReg) {
9801         aggParamReg = &GenStructParamIndex(base, *indexExpr, shift, ptype, fromType);
9802         return nullptr;
9803     }
9804     MemOperand *memOpnd = nullptr;
9805     if ((fromType == PTY_i32) && (toType == PTY_a64)) {
9806         RegOperand &index =
9807             static_cast<RegOperand &>(LoadIntoRegister(*HandleExpr(*indexExpr, *indexExpr->Opnd(0)), PTY_i32));
9808         memOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, GetPrimTypeBitSize(ptype), &base, &index, shift, true);
9809     } else if ((fromType == PTY_u32) && (toType == PTY_a64)) {
9810         RegOperand &index =
9811             static_cast<RegOperand &>(LoadIntoRegister(*HandleExpr(*indexExpr, *indexExpr->Opnd(0)), PTY_u32));
9812         memOpnd =
9813             &GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, GetPrimTypeBitSize(ptype), &base, &index, shift, false);
9814     }
9815     return memOpnd;
9816 }
9817 
CreateNonExtendMemOpnd(PrimType ptype,const BaseNode & parent,BaseNode & addrExpr,int64 offset)9818 MemOperand &AArch64CGFunc::CreateNonExtendMemOpnd(PrimType ptype, const BaseNode &parent, BaseNode &addrExpr,
9819                                                   int64 offset)
9820 {
9821     Operand *addrOpnd = nullptr;
9822     if ((addrExpr.GetOpCode() == OP_add || addrExpr.GetOpCode() == OP_sub) &&
9823         addrExpr.Opnd(1)->GetOpCode() == OP_constval) {
9824         addrOpnd = HandleExpr(addrExpr, *addrExpr.Opnd(0));
9825         ConstvalNode *constOfstNode = static_cast<ConstvalNode *>(addrExpr.Opnd(1));
9826         DEBUG_ASSERT(constOfstNode->GetConstVal()->GetKind() == kConstInt, "expect MIRIntConst");
9827         MIRIntConst *intOfst = safe_cast<MIRIntConst>(constOfstNode->GetConstVal());
9828         CHECK_FATAL(intOfst != nullptr, "just checking");
9829         offset = (addrExpr.GetOpCode() == OP_add) ? offset + intOfst->GetSXTValue() : offset - intOfst->GetSXTValue();
9830     } else {
9831         addrOpnd = HandleExpr(parent, addrExpr);
9832     }
9833     addrOpnd = static_cast<RegOperand *>(&LoadIntoRegister(*addrOpnd, PTY_a64));
9834     Insn *lastInsn = GetCurBB() == nullptr ? nullptr : GetCurBB()->GetLastInsn();
9835     if ((addrExpr.GetOpCode() == OP_CG_array_elem_add) && (offset == 0) && lastInsn &&
9836         (lastInsn->GetMachineOpcode() == MOP_xadrpl12) &&
9837         (&lastInsn->GetOperand(kInsnFirstOpnd) == &lastInsn->GetOperand(kInsnSecondOpnd))) {
9838         Operand &opnd = lastInsn->GetOperand(kInsnThirdOpnd);
9839         StImmOperand &stOpnd = static_cast<StImmOperand &>(opnd);
9840 
9841         OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(static_cast<uint64>(stOpnd.GetOffset()), k32BitSize);
9842         MemOperand &tmpMemOpnd =
9843             GetOrCreateMemOpnd(MemOperand::kAddrModeLo12Li, GetPrimTypeBitSize(ptype),
9844                                static_cast<RegOperand *>(addrOpnd), nullptr, &ofstOpnd, stOpnd.GetSymbol());
9845         GetCurBB()->RemoveInsn(*GetCurBB()->GetLastInsn());
9846         return tmpMemOpnd;
9847     } else {
9848         OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(static_cast<uint64>(offset), k64BitSize);
9849         return GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, GetPrimTypeBitSize(ptype),
9850                                   static_cast<RegOperand *>(addrOpnd), nullptr, &ofstOpnd, nullptr);
9851     }
9852 }
9853 
9854 /*
9855  * Create a memory operand with specified data type and memory ordering, making
9856  * use of aarch64 extend register addressing mode when possible.
9857  */
CreateMemOpnd(PrimType ptype,const BaseNode & parent,BaseNode & addrExpr,int64 offset,AArch64isa::MemoryOrdering memOrd)9858 MemOperand &AArch64CGFunc::CreateMemOpnd(PrimType ptype, const BaseNode &parent, BaseNode &addrExpr, int64 offset,
9859                                          AArch64isa::MemoryOrdering memOrd)
9860 {
9861     MemOperand *memOpnd = CheckAndCreateExtendMemOpnd(ptype, addrExpr, offset, memOrd);
9862     if (memOpnd != nullptr) {
9863         return *memOpnd;
9864     }
9865     return CreateNonExtendMemOpnd(ptype, parent, addrExpr, offset);
9866 }
9867 
CreateMemOpndOrNull(PrimType ptype,const BaseNode & parent,BaseNode & addrExpr,int64 offset,AArch64isa::MemoryOrdering memOrd)9868 MemOperand *AArch64CGFunc::CreateMemOpndOrNull(PrimType ptype, const BaseNode &parent, BaseNode &addrExpr, int64 offset,
9869                                                AArch64isa::MemoryOrdering memOrd)
9870 {
9871     MemOperand *memOpnd = CheckAndCreateExtendMemOpnd(ptype, addrExpr, offset, memOrd);
9872     if (memOpnd != nullptr) {
9873         return memOpnd;
9874     } else if (aggParamReg != nullptr) {
9875         return nullptr;
9876     }
9877     return &CreateNonExtendMemOpnd(ptype, parent, addrExpr, offset);
9878 }
9879 
GetOrCreateFuncNameOpnd(const MIRSymbol & symbol) const9880 Operand &AArch64CGFunc::GetOrCreateFuncNameOpnd(const MIRSymbol &symbol) const
9881 {
9882     return *memPool->New<FuncNameOperand>(symbol);
9883 }
9884 
GetOrCreateRflag()9885 Operand &AArch64CGFunc::GetOrCreateRflag()
9886 {
9887     if (rcc == nullptr) {
9888         rcc = &CreateRflagOperand();
9889     }
9890     return *rcc;
9891 }
9892 
GetRflag() const9893 const Operand *AArch64CGFunc::GetRflag() const
9894 {
9895     return rcc;
9896 }
9897 
GetOrCreatevaryreg()9898 Operand &AArch64CGFunc::GetOrCreatevaryreg()
9899 {
9900     if (vary == nullptr) {
9901         regno_t vRegNO = NewVReg(kRegTyVary, k8ByteSize);
9902         vary = &CreateVirtualRegisterOperand(vRegNO);
9903     }
9904     return *vary;
9905 }
9906 
9907 /* the first operand in opndvec is return opnd */
SelectLibCall(const std::string & funcName,std::vector<Operand * > & opndVec,PrimType primType,PrimType retPrimType,bool is2ndRet)9908 void AArch64CGFunc::SelectLibCall(const std::string &funcName, std::vector<Operand *> &opndVec, PrimType primType,
9909                                   PrimType retPrimType, bool is2ndRet)
9910 {
9911     std::vector<PrimType> pt;
9912     pt.push_back(retPrimType);
9913     for (size_t i = 0; i < opndVec.size(); ++i) {
9914         pt.push_back(primType);
9915     }
9916     SelectLibCallNArg(funcName, opndVec, pt, retPrimType, is2ndRet);
9917     return;
9918 }
9919 
SelectLibCallNArg(const std::string & funcName,std::vector<Operand * > & opndVec,std::vector<PrimType> pt,PrimType retPrimType,bool is2ndRet)9920 void AArch64CGFunc::SelectLibCallNArg(const std::string &funcName, std::vector<Operand *> &opndVec,
9921                                       std::vector<PrimType> pt, PrimType retPrimType, bool is2ndRet)
9922 {
9923     std::string newName = funcName;
9924     // Check whether we have a maple version of libcall and we want to use it instead.
9925     if (!CGOptions::IsDuplicateAsmFileEmpty() && asmMap.find(funcName) != asmMap.end()) {
9926         newName = asmMap.at(funcName);
9927     }
9928     MIRSymbol *st = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
9929     st->SetNameStrIdx(newName);
9930     st->SetStorageClass(kScExtern);
9931     st->SetSKind(kStFunc);
9932     /* setup the type of the callee function */
9933     std::vector<TyIdx> vec;
9934     std::vector<TypeAttrs> vecAt;
9935     for (size_t i = 1; i < opndVec.size(); ++i) {
9936         (void)vec.emplace_back(GlobalTables::GetTypeTable().GetTypeTable()[static_cast<size_t>(pt[i])]->GetTypeIndex());
9937         vecAt.emplace_back(TypeAttrs());
9938     }
9939 
9940     MIRType *retType = GlobalTables::GetTypeTable().GetTypeTable().at(static_cast<size_t>(retPrimType));
9941     st->SetTyIdx(GetBecommon().BeGetOrCreateFunctionType(retType->GetTypeIndex(), vec, vecAt)->GetTypeIndex());
9942 
9943     if (GetCG()->GenerateVerboseCG()) {
9944         const std::string &comment = "lib call : " + newName;
9945         GetCurBB()->AppendInsn(CreateCommentInsn(comment));
9946     }
9947     // only create c lib call here
9948     AArch64CallConvImpl parmLocator(GetBecommon());
9949     CCLocInfo ploc;
9950     /* setup actual parameters */
9951     ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
9952     for (size_t i = 1; i < opndVec.size(); ++i) {
9953         DEBUG_ASSERT(pt[i] != PTY_void, "primType check");
9954         MIRType *ty;
9955         ty = GlobalTables::GetTypeTable().GetTypeTable()[static_cast<size_t>(pt[i])];
9956         Operand *stOpnd = opndVec[i];
9957         if (stOpnd->GetKind() != Operand::kOpdRegister) {
9958             stOpnd = &SelectCopy(*stOpnd, pt[i], pt[i]);
9959         }
9960         RegOperand *expRegOpnd = static_cast<RegOperand *>(stOpnd);
9961         parmLocator.LocateNextParm(*ty, ploc);
9962         if (ploc.reg0 != 0) { /* load to the register */
9963             RegOperand &parmRegOpnd = GetOrCreatePhysicalRegisterOperand(
9964                 static_cast<AArch64reg>(ploc.reg0), expRegOpnd->GetSize(), GetRegTyFromPrimTy(pt[i]));
9965             SelectCopy(parmRegOpnd, pt[i], *expRegOpnd, pt[i]);
9966             srcOpnds->PushOpnd(parmRegOpnd);
9967         }
9968         DEBUG_ASSERT(ploc.reg1 == 0, "SelectCall NYI");
9969     }
9970 
9971     MIRSymbol *sym = GetFunction().GetLocalOrGlobalSymbol(st->GetStIdx(), false);
9972     Insn &callInsn = AppendCall(*sym, *srcOpnds);
9973     MIRType *callRetType = GlobalTables::GetTypeTable().GetTypeTable().at(static_cast<int32>(retPrimType));
9974     if (callRetType != nullptr) {
9975         callInsn.SetRetSize(static_cast<uint32>(callRetType->GetSize()));
9976         callInsn.SetIsCallReturnUnsigned(IsUnsignedInteger(callRetType->GetPrimType()));
9977     }
9978     GetFunction().SetHasCall();
9979     /* get return value */
9980     Operand *opnd0 = opndVec[0];
9981     CCLocInfo retMech;
9982     parmLocator.InitReturnInfo(*(GlobalTables::GetTypeTable().GetTypeTable().at(retPrimType)), retMech);
9983     if (retMech.GetRegCount() <= 0) {
9984         CHECK_FATAL(false, "should return from register");
9985     }
9986     if (!opnd0->IsRegister()) {
9987         CHECK_FATAL(false, "nyi");
9988     }
9989     RegOperand *regOpnd = static_cast<RegOperand *>(opnd0);
9990     AArch64reg regNum = static_cast<AArch64reg>(is2ndRet ? retMech.GetReg1() : retMech.GetReg0());
9991     if (regOpnd->GetRegisterNumber() != regNum) {
9992         RegOperand &retOpnd =
9993             GetOrCreatePhysicalRegisterOperand(regNum, regOpnd->GetSize(), GetRegTyFromPrimTy(retPrimType));
9994         SelectCopy(*opnd0, retPrimType, retOpnd, retPrimType);
9995     }
9996 }
9997 
GetBaseReg(const AArch64SymbolAlloc & symAlloc)9998 Operand *AArch64CGFunc::GetBaseReg(const AArch64SymbolAlloc &symAlloc)
9999 {
10000     MemSegmentKind sgKind = symAlloc.GetMemSegment()->GetMemSegmentKind();
10001     DEBUG_ASSERT(((sgKind == kMsArgsRegPassed) || (sgKind == kMsLocals) || (sgKind == kMsRefLocals) ||
10002                   (sgKind == kMsArgsToStkPass) || (sgKind == kMsArgsStkPassed)),
10003                  "NYI");
10004 
10005     if (sgKind == kMsArgsStkPassed) {
10006         return &GetOrCreatevaryreg();
10007     }
10008 
10009     if (fsp == nullptr) {
10010         fsp = &GetOrCreatePhysicalRegisterOperand(RFP, GetPointerSize() * kBitsPerByte, kRegTyInt);
10011     }
10012     return fsp;
10013 }
10014 
GetBaseOffset(const SymbolAlloc & sa)10015 int32 AArch64CGFunc::GetBaseOffset(const SymbolAlloc &sa)
10016 {
10017     const AArch64SymbolAlloc *symAlloc = static_cast<const AArch64SymbolAlloc *>(&sa);
10018     /* Call Frame layout of AArch64
10019      * Refer to V2 in aarch64_memlayout.h.
10020      * Do Not change this unless you know what you do
10021      */
10022     const int32 sizeofFplr = 2 * kIntregBytelen;
10023     MemSegmentKind sgKind = symAlloc->GetMemSegment()->GetMemSegmentKind();
10024     AArch64MemLayout *memLayout = static_cast<AArch64MemLayout *>(this->GetMemlayout());
10025     if (sgKind == kMsArgsStkPassed) { /* for callees */
10026         int32 offset = static_cast<int32>(symAlloc->GetOffset());
10027         return offset;
10028     } else if (sgKind == kMsArgsRegPassed) {
10029         int32 baseOffset = memLayout->GetSizeOfLocals() + symAlloc->GetOffset() + memLayout->GetSizeOfRefLocals();
10030         return baseOffset + sizeofFplr;
10031     } else if (sgKind == kMsRefLocals) {
10032         int32 baseOffset = symAlloc->GetOffset() + memLayout->GetSizeOfLocals();
10033         return baseOffset + sizeofFplr;
10034     } else if (sgKind == kMsLocals) {
10035         int32 baseOffset = symAlloc->GetOffset();
10036         return baseOffset + sizeofFplr;
10037     } else if (sgKind == kMsSpillReg) {
10038         if (GetCG()->IsLmbc()) {
10039             return symAlloc->GetOffset() + memLayout->SizeOfArgsToStackPass();
10040         }
10041         int32 baseOffset = symAlloc->GetOffset() + memLayout->SizeOfArgsRegisterPassed() +
10042                            memLayout->GetSizeOfLocals() + memLayout->GetSizeOfRefLocals();
10043         return baseOffset + sizeofFplr;
10044     } else if (sgKind == kMsArgsToStkPass) { /* this is for callers */
10045         return static_cast<int32>(symAlloc->GetOffset());
10046     } else {
10047         CHECK_FATAL(false, "sgKind check");
10048     }
10049     return 0;
10050 }
10051 
AppendCall(const MIRSymbol & funcSymbol)10052 void AArch64CGFunc::AppendCall(const MIRSymbol &funcSymbol)
10053 {
10054     ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
10055     AppendCall(funcSymbol, *srcOpnds);
10056 }
10057 
DBGFixCallFrameLocationOffsets()10058 void AArch64CGFunc::DBGFixCallFrameLocationOffsets()
10059 {
10060     for (DBGExprLoc *el : GetDbgCallFrameLocations()) {
10061         if (el->GetSimpLoc()->GetDwOp() == DW_OP_fbreg) {
10062             SymbolAlloc *symloc = static_cast<SymbolAlloc *>(el->GetSymLoc());
10063             int32 offset = GetBaseOffset(*symloc) - GetDbgCallFrameOffset();
10064             el->SetFboffset(offset);
10065         }
10066     }
10067 }
10068 
SelectAddAfterInsn(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType,bool isDest,Insn & insn)10069 void AArch64CGFunc::SelectAddAfterInsn(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType, bool isDest,
10070                                        Insn &insn)
10071 {
10072     uint32 dsize = GetPrimTypeBitSize(primType);
10073     bool is64Bits = (dsize == k64BitSize);
10074     DEBUG_ASSERT(opnd0.GetKind() == Operand::kOpdRegister, "Spill memory operand should based on register");
10075     DEBUG_ASSERT((opnd1.GetKind() == Operand::kOpdImmediate || opnd1.GetKind() == Operand::kOpdOffset),
10076                  "Spill memory operand should be with a immediate offset.");
10077 
10078     ImmOperand *immOpnd = static_cast<ImmOperand *>(&opnd1);
10079 
10080     MOperator mOpCode = MOP_undef;
10081     Insn *curInsn = &insn;
10082     /* lower 24 bits has 1, higher bits are all 0 */
10083     if (immOpnd->IsInBitSize(kMaxImmVal24Bits, 0)) {
10084         /* lower 12 bits and higher 12 bits both has 1 */
10085         Operand *newOpnd0 = &opnd0;
10086         if (!(immOpnd->IsInBitSize(kMaxImmVal12Bits, 0) || immOpnd->IsInBitSize(kMaxImmVal12Bits, kMaxImmVal12Bits))) {
10087             /* process higher 12 bits */
10088             ImmOperand &immOpnd2 =
10089                 CreateImmOperand(static_cast<int64>(static_cast<uint64>(immOpnd->GetValue()) >> kMaxImmVal12Bits),
10090                                  immOpnd->GetSize(), immOpnd->IsSignedValue());
10091             mOpCode = is64Bits ? MOP_xaddrri24 : MOP_waddrri24;
10092             BitShiftOperand &shiftopnd = CreateBitShiftOperand(BitShiftOperand::kLSL, kShiftAmount12, k64BitSize);
10093             Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, opnd0, immOpnd2, shiftopnd);
10094             DEBUG_ASSERT(IsOperandImmValid(mOpCode, &immOpnd2, kInsnThirdOpnd), "immOpnd2 appears invalid");
10095             if (isDest) {
10096                 insn.GetBB()->InsertInsnAfter(insn, newInsn);
10097             } else {
10098                 insn.GetBB()->InsertInsnBefore(insn, newInsn);
10099             }
10100             /* get lower 12 bits value */
10101             immOpnd->ModuloByPow2(static_cast<int32>(kMaxImmVal12Bits));
10102             newOpnd0 = &resOpnd;
10103             curInsn = &newInsn;
10104         }
10105         /* process lower 12 bits value */
10106         mOpCode = is64Bits ? MOP_xaddrri12 : MOP_waddrri12;
10107         Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, *newOpnd0, *immOpnd);
10108         DEBUG_ASSERT(IsOperandImmValid(mOpCode, immOpnd, kInsnThirdOpnd), "immOpnd appears invalid");
10109         if (isDest) {
10110             insn.GetBB()->InsertInsnAfter(*curInsn, newInsn);
10111         } else {
10112             insn.GetBB()->InsertInsnBefore(insn, newInsn);
10113         }
10114     } else {
10115         /* load into register */
10116         RegOperand &movOpnd = GetOrCreatePhysicalRegisterOperand(R16, dsize, kRegTyInt);
10117         mOpCode = is64Bits ? MOP_xmovri64 : MOP_wmovri32;
10118         Insn &movInsn = GetInsnBuilder()->BuildInsn(mOpCode, movOpnd, *immOpnd);
10119         mOpCode = is64Bits ? MOP_xaddrrr : MOP_waddrrr;
10120         Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, opnd0, movOpnd);
10121         if (isDest) {
10122             (void)insn.GetBB()->InsertInsnAfter(insn, newInsn);
10123             (void)insn.GetBB()->InsertInsnAfter(insn, movInsn);
10124         } else {
10125             (void)insn.GetBB()->InsertInsnBefore(insn, movInsn);
10126             (void)insn.GetBB()->InsertInsnBefore(insn, newInsn);
10127         }
10128     }
10129 }
10130 
AdjustMemOperandIfOffsetOutOfRange(MemOperand * memOpnd,regno_t vrNum,bool isDest,Insn & insn,AArch64reg regNum,bool & isOutOfRange)10131 MemOperand *AArch64CGFunc::AdjustMemOperandIfOffsetOutOfRange(MemOperand *memOpnd, regno_t vrNum, bool isDest,
10132                                                               Insn &insn, AArch64reg regNum, bool &isOutOfRange)
10133 {
10134     if (vrNum >= vRegTable.size()) {
10135         CHECK_FATAL(false, "index out of range in AArch64CGFunc::AdjustMemOperandIfOffsetOutOfRange");
10136     }
10137     uint32 dataSize = GetOrCreateVirtualRegisterOperand(vrNum).GetSize();
10138     if (IsImmediateOffsetOutOfRange(*memOpnd, dataSize) && CheckIfSplitOffsetWithAdd(*memOpnd, dataSize)) {
10139         isOutOfRange = true;
10140         memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, dataSize, regNum, isDest, &insn);
10141     } else {
10142         isOutOfRange = false;
10143     }
10144     return memOpnd;
10145 }
10146 
FreeSpillRegMem(regno_t vrNum)10147 void AArch64CGFunc::FreeSpillRegMem(regno_t vrNum)
10148 {
10149     MemOperand *memOpnd = nullptr;
10150 
10151     auto p = spillRegMemOperands.find(vrNum);
10152     if (p != spillRegMemOperands.end()) {
10153         memOpnd = p->second;
10154     }
10155 
10156     if ((memOpnd == nullptr) && IsVRegNOForPseudoRegister(vrNum)) {
10157         auto pSecond = pRegSpillMemOperands.find(GetPseudoRegIdxFromVirtualRegNO(vrNum));
10158         if (pSecond != pRegSpillMemOperands.end()) {
10159             memOpnd = pSecond->second;
10160         }
10161     }
10162 
10163     if (memOpnd == nullptr) {
10164         DEBUG_ASSERT(false, "free spillreg have no mem");
10165         return;
10166     }
10167 
10168     uint32 size = memOpnd->GetSize();
10169     MapleUnorderedMap<uint32, SpillMemOperandSet *>::iterator iter;
10170     if ((iter = reuseSpillLocMem.find(size)) != reuseSpillLocMem.end()) {
10171         iter->second->Add(*memOpnd);
10172     } else {
10173         reuseSpillLocMem[size] = memPool->New<SpillMemOperandSet>(*GetFuncScopeAllocator());
10174         reuseSpillLocMem[size]->Add(*memOpnd);
10175     }
10176 }
10177 
GetOrCreatSpillMem(regno_t vrNum,uint32 memSize)10178 MemOperand *AArch64CGFunc::GetOrCreatSpillMem(regno_t vrNum, uint32 memSize)
10179 {
10180     /* NOTES: must used in RA, not used in other place. */
10181     if (IsVRegNOForPseudoRegister(vrNum)) {
10182         auto p = pRegSpillMemOperands.find(GetPseudoRegIdxFromVirtualRegNO(vrNum));
10183         if (p != pRegSpillMemOperands.end()) {
10184             return p->second;
10185         }
10186     }
10187 
10188     auto p = spillRegMemOperands.find(vrNum);
10189     if (p == spillRegMemOperands.end()) {
10190         if (vrNum >= vRegTable.size()) {
10191             CHECK_FATAL(false, "index out of range in AArch64CGFunc::FreeSpillRegMem");
10192         }
10193         uint32 memBitSize = (memSize <= k32BitSize) ? k32BitSize :
10194             (memSize <= k64BitSize) ? k64BitSize : k128BitSize;
10195         auto it = reuseSpillLocMem.find(memBitSize);
10196         if (it != reuseSpillLocMem.end()) {
10197             MemOperand *memOpnd = it->second->GetOne();
10198             if (memOpnd != nullptr) {
10199                 (void)spillRegMemOperands.emplace(std::pair<regno_t, MemOperand*>(vrNum, memOpnd));
10200                 return memOpnd;
10201             }
10202         }
10203 
10204         RegOperand &baseOpnd = GetOrCreateStackBaseRegOperand();
10205         int64 offset = GetOrCreatSpillRegLocation(vrNum, memBitSize / kBitsPerByte);
10206         MemOperand *memOpnd = nullptr;
10207         OfstOperand *offsetOpnd = &CreateOfstOpnd(static_cast<uint64>(offset), k64BitSize);
10208         memOpnd = CreateMemOperand(MemOperand::kAddrModeBOi, memBitSize, baseOpnd, nullptr, offsetOpnd, nullptr);
10209         (void)spillRegMemOperands.emplace(std::pair<regno_t, MemOperand*>(vrNum, memOpnd));
10210         return memOpnd;
10211     } else {
10212         return p->second;
10213     }
10214 }
10215 
GetPseudoRegisterSpillMemoryOperand(PregIdx i)10216 MemOperand *AArch64CGFunc::GetPseudoRegisterSpillMemoryOperand(PregIdx i)
10217 {
10218     MapleUnorderedMap<PregIdx, MemOperand *>::iterator p;
10219     if (GetCG()->GetOptimizeLevel() == CGOptions::kLevel0) {
10220         p = pRegSpillMemOperands.end();
10221     } else {
10222         p = pRegSpillMemOperands.find(i);
10223     }
10224     if (p != pRegSpillMemOperands.end()) {
10225         return p->second;
10226     }
10227     int64 offset = GetPseudoRegisterSpillLocation(i);
10228     MIRPreg *preg = GetFunction().GetPregTab()->PregFromPregIdx(i);
10229     uint32 bitLen = GetPrimTypeSize(preg->GetPrimType()) * kBitsPerByte;
10230     RegOperand &base = GetOrCreateFramePointerRegOperand();
10231 
10232     OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(static_cast<uint64>(offset), k32BitSize);
10233     MemOperand &memOpnd = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, bitLen, &base, nullptr, &ofstOpnd, nullptr);
10234     if (IsImmediateOffsetOutOfRange(memOpnd, bitLen)) {
10235         MemOperand &newMemOpnd = SplitOffsetWithAddInstruction(memOpnd, bitLen);
10236         (void)pRegSpillMemOperands.emplace(std::pair<PregIdx, MemOperand *>(i, &newMemOpnd));
10237         return &newMemOpnd;
10238     }
10239     (void)pRegSpillMemOperands.emplace(std::pair<PregIdx, MemOperand *>(i, &memOpnd));
10240     return &memOpnd;
10241 }
10242 
GetPseudoRegFromVirtualRegNO(const regno_t vRegNO,bool afterSSA) const10243 MIRPreg *AArch64CGFunc::GetPseudoRegFromVirtualRegNO(const regno_t vRegNO, bool afterSSA) const
10244 {
10245     PregIdx pri = afterSSA ? VRegNOToPRegIdx(vRegNO) : GetPseudoRegIdxFromVirtualRegNO(vRegNO);
10246     if (pri == -1)
10247         return nullptr;
10248     return GetFunction().GetPregTab()->PregFromPregIdx(pri);
10249 }
10250 
10251 /* Get the number of return register of current function. */
GetReturnRegisterNumber()10252 AArch64reg AArch64CGFunc::GetReturnRegisterNumber()
10253 {
10254     CCImpl &retLocator = *GetOrCreateLocator(GetCurCallConvKind());
10255     CCLocInfo retMech;
10256     retLocator.InitReturnInfo(*(GetFunction().GetReturnType()), retMech);
10257     if (retMech.GetRegCount() > 0) {
10258         return static_cast<AArch64reg>(retMech.GetReg0());
10259     }
10260     return kRinvalid;
10261 }
10262 
CanLazyBinding(const Insn & ldrInsn) const10263 bool AArch64CGFunc::CanLazyBinding(const Insn &ldrInsn) const
10264 {
10265     Operand &memOpnd = ldrInsn.GetOperand(1);
10266     auto &aarchMemOpnd = static_cast<MemOperand &>(memOpnd);
10267     if (aarchMemOpnd.GetAddrMode() != MemOperand::kAddrModeLo12Li) {
10268         return false;
10269     }
10270 
10271     const MIRSymbol *sym = aarchMemOpnd.GetSymbol();
10272     CHECK_FATAL(sym != nullptr, "sym can't be nullptr");
10273     if (sym->IsMuidFuncDefTab() || sym->IsMuidFuncUndefTab() || sym->IsMuidDataDefTab() || sym->IsMuidDataUndefTab() ||
10274         (sym->IsReflectionClassInfo() && !sym->IsReflectionArrayClassInfo())) {
10275         return true;
10276     }
10277 
10278     return false;
10279 }
10280 
10281 /*
10282  *  add reg, reg, __PTR_C_STR_...
10283  *  ldr reg1, [reg]
10284  *  =>
10285  *  ldr reg1, [reg, #:lo12:__Ptr_C_STR_...]
10286  */
ConvertAdrpl12LdrToLdr()10287 void AArch64CGFunc::ConvertAdrpl12LdrToLdr()
10288 {
10289     FOR_ALL_BB(bb, this) {
10290         FOR_BB_INSNS_SAFE(insn, bb, nextInsn) {
10291             nextInsn = insn->GetNextMachineInsn();
10292             if (nextInsn == nullptr) {
10293                 break;
10294             }
10295             if (!insn->IsMachineInstruction()) {
10296                 continue;
10297             }
10298             /* check first insn */
10299             MOperator thisMop = insn->GetMachineOpcode();
10300             if (thisMop != MOP_xadrpl12) {
10301                 continue;
10302             }
10303             /* check second insn */
10304             MOperator nextMop = nextInsn->GetMachineOpcode();
10305             if (!(((nextMop >= MOP_wldrsb) && (nextMop <= MOP_dldp)) ||
10306                   ((nextMop >= MOP_wstrb) && (nextMop <= MOP_dstp)))) {
10307                 continue;
10308             }
10309 
10310             /* Check if base register of nextInsn and the dest operand of insn are identical. */
10311             MemOperand *memOpnd = static_cast<MemOperand *>(nextInsn->GetMemOpnd());
10312             CHECK_FATAL(memOpnd != nullptr, "memOpnd can't be nullptr");
10313 
10314             /* Only for AddrMode_B_OI addressing mode. */
10315             if (memOpnd->GetAddrMode() != MemOperand::kAddrModeBOi) {
10316                 continue;
10317             }
10318 
10319             /* Only for intact memory addressing. */
10320             if (!memOpnd->IsIntactIndexed()) {
10321                 continue;
10322             }
10323 
10324             auto &regOpnd = static_cast<RegOperand &>(insn->GetOperand(0));
10325 
10326             /* Check if dest operand of insn is idential with base register of nextInsn. */
10327             RegOperand *baseReg = memOpnd->GetBaseRegister();
10328             CHECK_FATAL(baseReg != nullptr, "baseReg can't be nullptr");
10329             if (baseReg->GetRegisterNumber() != regOpnd.GetRegisterNumber()) {
10330                 continue;
10331             }
10332 
10333             StImmOperand &stImmOpnd = static_cast<StImmOperand &>(insn->GetOperand(kInsnThirdOpnd));
10334             OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(
10335                 static_cast<uint64>(stImmOpnd.GetOffset() + memOpnd->GetOffsetImmediate()->GetOffsetValue()),
10336                 k32BitSize);
10337             RegOperand &newBaseOpnd = static_cast<RegOperand &>(insn->GetOperand(kInsnSecondOpnd));
10338             MemOperand &newMemOpnd = GetOrCreateMemOpnd(MemOperand::kAddrModeLo12Li, memOpnd->GetSize(), &newBaseOpnd,
10339                                                         nullptr, &ofstOpnd, stImmOpnd.GetSymbol());
10340             nextInsn->SetOperand(1, newMemOpnd);
10341             bb->RemoveInsn(*insn);
10342         }
10343     }
10344 }
10345 
10346 /*
10347  * adrp reg1, __muid_func_undef_tab..
10348  * ldr reg2, [reg1, #:lo12:__muid_func_undef_tab..]
10349  * =>
10350  * intrinsic_adrp_ldr reg2, __muid_func_undef_tab...
10351  */
ConvertAdrpLdrToIntrisic()10352 void AArch64CGFunc::ConvertAdrpLdrToIntrisic()
10353 {
10354     FOR_ALL_BB(bb, this) {
10355         FOR_BB_INSNS_SAFE(insn, bb, nextInsn) {
10356             nextInsn = insn->GetNextMachineInsn();
10357             if (nextInsn == nullptr) {
10358                 break;
10359             }
10360             if (!insn->IsMachineInstruction()) {
10361                 continue;
10362             }
10363 
10364             MOperator firstMop = insn->GetMachineOpcode();
10365             MOperator secondMop = nextInsn->GetMachineOpcode();
10366             if (!((firstMop == MOP_xadrp) && ((secondMop == MOP_wldr) || (secondMop == MOP_xldr)))) {
10367                 continue;
10368             }
10369 
10370             if (CanLazyBinding(*nextInsn)) {
10371                 bb->ReplaceInsn(
10372                     *insn, GetInsnBuilder()->BuildInsn(MOP_adrp_ldr, nextInsn->GetOperand(0), insn->GetOperand(1)));
10373                 bb->RemoveInsn(*nextInsn);
10374             }
10375         }
10376     }
10377 }
10378 
ProcessLazyBinding()10379 void AArch64CGFunc::ProcessLazyBinding()
10380 {
10381     ConvertAdrpl12LdrToLdr();
10382     ConvertAdrpLdrToIntrisic();
10383 }
10384 
10385 /*
10386  * Generate global long call
10387  *  adrp  VRx, symbol
10388  *  ldr VRx, [VRx, #:lo12:symbol]
10389  *  blr VRx
10390  *
10391  * Input:
10392  *  insn       : insert new instruction after the 'insn'
10393  *  func       : the symbol of the function need to be called
10394  *  srcOpnds   : list operand of the function need to be called
10395  *  isCleanCall: when generate clean call insn, set isCleanCall as true
10396  * Return: the 'blr' instruction
10397  */
GenerateGlobalLongCallAfterInsn(const MIRSymbol & func,ListOperand & srcOpnds)10398 Insn &AArch64CGFunc::GenerateGlobalLongCallAfterInsn(const MIRSymbol &func, ListOperand &srcOpnds)
10399 {
10400     MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(func.GetStIdx());
10401     symbol->SetStorageClass(kScGlobal);
10402     RegOperand &tmpReg = CreateRegisterOperandOfType(PTY_u64);
10403     StImmOperand &stOpnd = CreateStImmOperand(*symbol, 0, 0);
10404     OfstOperand &offsetOpnd = CreateOfstOpnd(*symbol, 0);
10405     Insn &adrpInsn = GetInsnBuilder()->BuildInsn(MOP_xadrp, tmpReg, stOpnd);
10406     GetCurBB()->AppendInsn(adrpInsn);
10407     MemOperand &memOrd = GetOrCreateMemOpnd(MemOperand::kAddrModeLo12Li, GetPointerSize() * kBitsPerByte,
10408                                             static_cast<RegOperand *>(&tmpReg), nullptr, &offsetOpnd, symbol);
10409     Insn &ldrInsn = GetInsnBuilder()->BuildInsn(memOrd.GetSize() == k64BitSize ? MOP_xldr : MOP_wldr, tmpReg, memOrd);
10410     GetCurBB()->AppendInsn(ldrInsn);
10411 
10412     Insn &callInsn = GetInsnBuilder()->BuildInsn(MOP_xblr, tmpReg, srcOpnds);
10413     GetCurBB()->AppendInsn(callInsn);
10414     GetCurBB()->SetHasCall();
10415     return callInsn;
10416 }
10417 
10418 /*
10419  * Generate local long call
10420  *  adrp  VRx, symbol
10421  *  add VRx, VRx, #:lo12:symbol
10422  *  blr VRx
10423  *
10424  * Input:
10425  *  insn       : insert new instruction after the 'insn'
10426  *  func       : the symbol of the function need to be called
10427  *  srcOpnds   : list operand of the function need to be called
10428  *  isCleanCall: when generate clean call insn, set isCleanCall as true
10429  * Return: the 'blr' instruction
10430  */
GenerateLocalLongCallAfterInsn(const MIRSymbol & func,ListOperand & srcOpnds)10431 Insn &AArch64CGFunc::GenerateLocalLongCallAfterInsn(const MIRSymbol &func, ListOperand &srcOpnds)
10432 {
10433     RegOperand &tmpReg = CreateRegisterOperandOfType(PTY_u64);
10434     StImmOperand &stOpnd = CreateStImmOperand(func, 0, 0);
10435     Insn &adrpInsn = GetInsnBuilder()->BuildInsn(MOP_xadrp, tmpReg, stOpnd);
10436     GetCurBB()->AppendInsn(adrpInsn);
10437     Insn &addInsn = GetInsnBuilder()->BuildInsn(MOP_xadrpl12, tmpReg, tmpReg, stOpnd);
10438     GetCurBB()->AppendInsn(addInsn);
10439     Insn *callInsn = &GetInsnBuilder()->BuildInsn(MOP_xblr, tmpReg, srcOpnds);
10440     GetCurBB()->AppendInsn(*callInsn);
10441     GetCurBB()->SetHasCall();
10442     return *callInsn;
10443 }
10444 
AppendCall(const MIRSymbol & sym,ListOperand & srcOpnds)10445 Insn &AArch64CGFunc::AppendCall(const MIRSymbol &sym, ListOperand &srcOpnds)
10446 {
10447     Insn *callInsn = nullptr;
10448     if (CGOptions::IsLongCalls()) {
10449         MIRFunction *mirFunc = sym.GetFunction();
10450         if (IsDuplicateAsmList(sym) || (mirFunc && mirFunc->GetAttr(FUNCATTR_local))) {
10451             callInsn = &GenerateLocalLongCallAfterInsn(sym, srcOpnds);
10452         } else {
10453             callInsn = &GenerateGlobalLongCallAfterInsn(sym, srcOpnds);
10454         }
10455     } else {
10456         Operand &targetOpnd = GetOrCreateFuncNameOpnd(sym);
10457         callInsn = &GetInsnBuilder()->BuildInsn(MOP_xbl, targetOpnd, srcOpnds);
10458         GetCurBB()->AppendInsn(*callInsn);
10459         GetCurBB()->SetHasCall();
10460     }
10461     return *callInsn;
10462 }
10463 
IsDuplicateAsmList(const MIRSymbol & sym) const10464 bool AArch64CGFunc::IsDuplicateAsmList(const MIRSymbol &sym) const
10465 {
10466     if (CGOptions::IsDuplicateAsmFileEmpty()) {
10467         return false;
10468     }
10469 
10470     const std::string &name = sym.GetName();
10471     if ((name == "strlen") || (name == "strncmp") || (name == "memcpy") || (name == "memmove") || (name == "strcmp") ||
10472         (name == "memcmp") || (name == "memcmpMpl")) {
10473         return true;
10474     }
10475     return false;
10476 }
10477 
SelectMPLProfCounterInc(const IntrinsiccallNode & intrnNode)10478 void AArch64CGFunc::SelectMPLProfCounterInc(const IntrinsiccallNode &intrnNode)
10479 {
10480     if (Options::profileGen) {
10481         DEBUG_ASSERT(intrnNode.NumOpnds() == 1, "must be 1 operand");
10482         BaseNode *arg1 = intrnNode.Opnd(0);
10483         DEBUG_ASSERT(arg1 != nullptr, "nullptr check");
10484         regno_t vRegNO1 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
10485         RegOperand &vReg1 = CreateVirtualRegisterOperand(vRegNO1);
10486         vReg1.SetRegNotBBLocal();
10487         static const MIRSymbol *bbProfileTab = nullptr;
10488 
10489         // Ref: MeProfGen::InstrumentFunc on ctrTbl namiLogicalShiftLeftOperandng
10490         std::string ctrTblName = namemangler::kprefixProfCtrTbl + GetMirModule().GetFileName() + "_" + GetName();
10491         std::replace(ctrTblName.begin(), ctrTblName.end(), '.', '_');
10492         std::replace(ctrTblName.begin(), ctrTblName.end(), '-', '_');
10493         std::replace(ctrTblName.begin(), ctrTblName.end(), '/', '_');
10494 
10495         if (!bbProfileTab || bbProfileTab->GetName() != ctrTblName) {
10496             bbProfileTab = GetMirModule().GetMIRBuilder()->GetGlobalDecl(ctrTblName);
10497             CHECK_FATAL(bbProfileTab != nullptr, "expect counter table");
10498         }
10499 
10500         ConstvalNode *constvalNode = static_cast<ConstvalNode *>(arg1);
10501         MIRConst *mirConst = constvalNode->GetConstVal();
10502         DEBUG_ASSERT(mirConst != nullptr, "nullptr check");
10503         CHECK_FATAL(mirConst->GetKind() == kConstInt, "expect MIRIntConst type");
10504         MIRIntConst *mirIntConst = safe_cast<MIRIntConst>(mirConst);
10505         int64 offset = GetPrimTypeSize(PTY_u64) * mirIntConst->GetExtValue();
10506 
10507         if (!CGOptions::IsQuiet()) {
10508             maple::LogInfo::MapleLogger(kLlInfo) << "At counter table offset: " << offset << std::endl;
10509         }
10510         MemOperand *memOpnd = &GetOrCreateMemOpnd(*bbProfileTab, offset, k64BitSize);
10511         if (IsImmediateOffsetOutOfRange(*memOpnd, k64BitSize)) {
10512             memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, k64BitSize);
10513         }
10514         Operand *reg = &SelectCopy(*memOpnd, PTY_u64, PTY_u64);
10515         ImmOperand &one = CreateImmOperand(1, k64BitSize, false);
10516         SelectAdd(*reg, *reg, one, PTY_u64);
10517         SelectCopy(*memOpnd, PTY_u64, *reg, PTY_u64);
10518         return;
10519     }
10520 
10521     DEBUG_ASSERT(intrnNode.NumOpnds() == 1, "must be 1 operand");
10522     BaseNode *arg1 = intrnNode.Opnd(0);
10523     DEBUG_ASSERT(arg1 != nullptr, "nullptr check");
10524     regno_t vRegNO1 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
10525     RegOperand &vReg1 = CreateVirtualRegisterOperand(vRegNO1);
10526     vReg1.SetRegNotBBLocal();
10527     static const MIRSymbol *bbProfileTab = nullptr;
10528     if (!bbProfileTab) {
10529         std::string bbProfileName = namemangler::kBBProfileTabPrefixStr + GetMirModule().GetFileNameAsPostfix();
10530         bbProfileTab = GetMirModule().GetMIRBuilder()->GetGlobalDecl(bbProfileName);
10531         CHECK_FATAL(bbProfileTab != nullptr, "expect bb profile tab");
10532     }
10533     ConstvalNode *constvalNode = static_cast<ConstvalNode *>(arg1);
10534     MIRConst *mirConst = constvalNode->GetConstVal();
10535     DEBUG_ASSERT(mirConst != nullptr, "nullptr check");
10536     CHECK_FATAL(mirConst->GetKind() == kConstInt, "expect MIRIntConst type");
10537     MIRIntConst *mirIntConst = safe_cast<MIRIntConst>(mirConst);
10538     int64 idx = GetPrimTypeSize(PTY_u32) * mirIntConst->GetExtValue();
10539     if (!CGOptions::IsQuiet()) {
10540         maple::LogInfo::MapleLogger(kLlErr) << "Id index " << idx << std::endl;
10541     }
10542     StImmOperand &stOpnd = CreateStImmOperand(*bbProfileTab, idx, 0);
10543     Insn &newInsn = GetInsnBuilder()->BuildInsn(MOP_counter, vReg1, stOpnd);
10544     newInsn.SetDoNotRemove(true);
10545     GetCurBB()->AppendInsn(newInsn);
10546 }
10547 
SelectMPLClinitCheck(const IntrinsiccallNode & intrnNode)10548 void AArch64CGFunc::SelectMPLClinitCheck(const IntrinsiccallNode &intrnNode)
10549 {
10550     DEBUG_ASSERT(intrnNode.NumOpnds() == 1, "must be 1 operand");
10551     BaseNode *arg = intrnNode.Opnd(0);
10552     Operand *stOpnd = nullptr;
10553     bool bClinitSeperate = false;
10554     DEBUG_ASSERT(CGOptions::IsPIC(), "must be doPIC");
10555     if (arg->GetOpCode() == OP_addrof) {
10556         AddrofNode *addrof = static_cast<AddrofNode *>(arg);
10557         MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(addrof->GetStIdx());
10558         DEBUG_ASSERT(symbol->GetName().find(CLASSINFO_PREFIX_STR) == 0, "must be a symbol with __classinfo__");
10559 
10560         if (!symbol->IsMuidDataUndefTab()) {
10561             std::string ptrName = namemangler::kPtrPrefixStr + symbol->GetName();
10562             MIRType *ptrType = GlobalTables::GetTypeTable().GetPtr();
10563             symbol = GetMirModule().GetMIRBuilder()->GetOrCreateGlobalDecl(ptrName, *ptrType);
10564             bClinitSeperate = true;
10565             symbol->SetStorageClass(kScFstatic);
10566         }
10567         stOpnd = &CreateStImmOperand(*symbol, 0, 0);
10568     } else {
10569         arg = arg->Opnd(0);
10570         BaseNode *arg0 = arg->Opnd(0);
10571         BaseNode *arg1 = arg->Opnd(1);
10572         DEBUG_ASSERT(arg0 != nullptr, "nullptr check");
10573         DEBUG_ASSERT(arg1 != nullptr, "nullptr check");
10574         DEBUG_ASSERT(arg0->GetOpCode() == OP_addrof, "expect the operand to be addrof");
10575         AddrofNode *addrof = static_cast<AddrofNode *>(arg0);
10576         MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(addrof->GetStIdx());
10577         DEBUG_ASSERT(addrof->GetFieldID() == 0, "For debug SelectMPLClinitCheck.");
10578         ConstvalNode *constvalNode = static_cast<ConstvalNode *>(arg1);
10579         MIRConst *mirConst = constvalNode->GetConstVal();
10580         DEBUG_ASSERT(mirConst != nullptr, "nullptr check");
10581         CHECK_FATAL(mirConst->GetKind() == kConstInt, "expect MIRIntConst type");
10582         MIRIntConst *mirIntConst = safe_cast<MIRIntConst>(mirConst);
10583         stOpnd = &CreateStImmOperand(*symbol, mirIntConst->GetExtValue(), 0);
10584     }
10585 
10586     regno_t vRegNO2 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
10587     RegOperand &vReg2 = CreateVirtualRegisterOperand(vRegNO2);
10588     vReg2.SetRegNotBBLocal();
10589     if (bClinitSeperate) {
10590         /* Seperate MOP_clinit to MOP_adrp_ldr + MOP_clinit_tail. */
10591         Insn &newInsn = GetInsnBuilder()->BuildInsn(MOP_adrp_ldr, vReg2, *stOpnd);
10592         GetCurBB()->AppendInsn(newInsn);
10593         newInsn.SetDoNotRemove(true);
10594         Insn &insn = GetInsnBuilder()->BuildInsn(MOP_clinit_tail, vReg2);
10595         insn.SetDoNotRemove(true);
10596         GetCurBB()->AppendInsn(insn);
10597     } else {
10598         Insn &newInsn = GetInsnBuilder()->BuildInsn(MOP_clinit, vReg2, *stOpnd);
10599         GetCurBB()->AppendInsn(newInsn);
10600     }
10601 }
GenCVaStartIntrin(RegOperand & opnd,uint32 stkSize)10602 void AArch64CGFunc::GenCVaStartIntrin(RegOperand &opnd, uint32 stkSize)
10603 {
10604     /* FPLR only pushed in regalloc() after intrin function */
10605     Operand &stkOpnd = GetOrCreatePhysicalRegisterOperand(RFP, k64BitSize, kRegTyInt);
10606 
10607     /* __stack */
10608     ImmOperand *offsOpnd;
10609     if (GetMirModule().GetFlavor() != MIRFlavor::kFlavorLmbc) {
10610         offsOpnd = &CreateImmOperand(0, k64BitSize, true, kUnAdjustVary); /* isvary reset StackFrameSize */
10611     } else {
10612         offsOpnd = &CreateImmOperand(0, k64BitSize, true);
10613     }
10614     ImmOperand *offsOpnd2 = &CreateImmOperand(stkSize, k64BitSize, false);
10615     RegOperand &vReg = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, GetPrimTypeSize(GetLoweredPtrType())));
10616     if (stkSize) {
10617         SelectAdd(vReg, *offsOpnd, *offsOpnd2, GetLoweredPtrType());
10618         SelectAdd(vReg, stkOpnd, vReg, GetLoweredPtrType());
10619     } else {
10620         SelectAdd(vReg, stkOpnd, *offsOpnd, GetLoweredPtrType()); /* stack pointer */
10621     }
10622     OfstOperand *offOpnd = &GetOrCreateOfstOpnd(0, k64BitSize); /* va_list ptr */
10623     /* mem operand in va_list struct (lhs) */
10624     MemOperand *strOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, &opnd, nullptr, offOpnd,
10625                                               static_cast<MIRSymbol *>(nullptr));
10626     GetCurBB()->AppendInsn(
10627         GetInsnBuilder()->BuildInsn(vReg.GetSize() == k64BitSize ? MOP_xstr : MOP_wstr, vReg, *strOpnd));
10628 
10629     /* __gr_top   ; it's the same as __stack before the 1st va_arg */
10630     if (CGOptions::IsArm64ilp32()) {
10631         offOpnd = &GetOrCreateOfstOpnd(GetPointerSize(), k64BitSize);
10632     } else {
10633         offOpnd = &GetOrCreateOfstOpnd(k8BitSize, k64BitSize);
10634     }
10635     strOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, &opnd, nullptr, offOpnd,
10636                                   static_cast<MIRSymbol *>(nullptr));
10637     SelectAdd(vReg, stkOpnd, *offsOpnd, GetLoweredPtrType());
10638     GetCurBB()->AppendInsn(
10639         GetInsnBuilder()->BuildInsn(vReg.GetSize() == k64BitSize ? MOP_xstr : MOP_wstr, vReg, *strOpnd));
10640 
10641     /* __vr_top */
10642     int32 grAreaSize = static_cast<int32>(static_cast<AArch64MemLayout *>(GetMemlayout())->GetSizeOfGRSaveArea());
10643     if (CGOptions::IsArm64ilp32()) {
10644         offsOpnd2 = &CreateImmOperand(static_cast<int64>(RoundUp(static_cast<uint64>(grAreaSize), k8ByteSize << 1)),
10645                                       k64BitSize, false);
10646     } else {
10647         offsOpnd2 = &CreateImmOperand(
10648             static_cast<int64>(RoundUp(static_cast<uint64>(grAreaSize), GetPointerSize() << 1)), k64BitSize, false);
10649     }
10650     SelectSub(vReg, *offsOpnd, *offsOpnd2, GetLoweredPtrType()); /* if 1st opnd is register => sub */
10651     SelectAdd(vReg, stkOpnd, vReg, GetLoweredPtrType());
10652     offOpnd = &GetOrCreateOfstOpnd(GetPointerSize() << 1, k64BitSize);
10653     strOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, &opnd, nullptr, offOpnd,
10654                                   static_cast<MIRSymbol *>(nullptr));
10655     GetCurBB()->AppendInsn(
10656         GetInsnBuilder()->BuildInsn(vReg.GetSize() == k64BitSize ? MOP_xstr : MOP_wstr, vReg, *strOpnd));
10657 
10658     /* __gr_offs */
10659     int32 offs = 0 - grAreaSize;
10660     offsOpnd = &CreateImmOperand(offs, k32BitSize, false);
10661     RegOperand *tmpReg = &CreateRegisterOperandOfType(PTY_i32); /* offs value to be assigned (rhs) */
10662     SelectCopyImm(*tmpReg, *offsOpnd, PTY_i32);
10663     // write __gr_offs : offset from gr_top to next GP register arg
10664     // accroding to typedef struct va_list
10665     // the field offset of __gr_offs is 3 pointer from beginning
10666     offOpnd = &GetOrCreateOfstOpnd(GetPointerSize() * 3, k32BitSize);
10667     strOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k32BitSize, &opnd, nullptr, offOpnd,
10668                                   static_cast<MIRSymbol *>(nullptr));
10669     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wstr, *tmpReg, *strOpnd));
10670 
10671     /* __vr_offs */
10672     offs =
10673         static_cast<int32>(UINT32_MAX - (static_cast<AArch64MemLayout *>(GetMemlayout())->GetSizeOfVRSaveArea() - 1UL));
10674     offsOpnd = &CreateImmOperand(offs, k32BitSize, false);
10675     tmpReg = &CreateRegisterOperandOfType(PTY_i32);
10676     SelectCopyImm(*tmpReg, *offsOpnd, PTY_i32);
10677     // write __vr_offs : offset from vr_top to next FP/SIMD register arg
10678     // accroding to typedef struct va_list
10679     // the field offset of __vr_offs is 3 pointer + sizeof(int) from beginning
10680     offOpnd = &GetOrCreateOfstOpnd((GetPointerSize() * 3 + sizeof(int32)), k32BitSize);
10681     strOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k32BitSize, &opnd, nullptr, offOpnd,
10682                                   static_cast<MIRSymbol *>(nullptr));
10683     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wstr, *tmpReg, *strOpnd));
10684 }
10685 
SelectCVaStart(const IntrinsiccallNode & intrnNode)10686 void AArch64CGFunc::SelectCVaStart(const IntrinsiccallNode &intrnNode)
10687 {
10688     DEBUG_ASSERT(intrnNode.NumOpnds() == 2, "must be 2 operands"); // must be 2 operands
10689     /* 2 operands, but only 1 needed. Don't need to emit code for second operand
10690      *
10691      * va_list is a passed struct with an address, load its address
10692      */
10693     isIntrnCallForC = true;
10694     BaseNode *argExpr = intrnNode.Opnd(0);
10695     Operand *opnd = HandleExpr(intrnNode, *argExpr);
10696     RegOperand &opnd0 = LoadIntoRegister(*opnd, GetLoweredPtrType()); /* first argument of intrinsic */
10697 
10698     /* Find beginning of unnamed arg on stack.
10699      * Ex. void foo(int i1, int i2, ... int i8, struct S r, struct S s, ...)
10700      *     where struct S has size 32, address of r and s are on stack but they are named.
10701      */
10702     AArch64CallConvImpl parmLocator(GetBecommon());
10703     CCLocInfo pLoc;
10704     uint32 stkSize = 0;
10705     uint32 inReg = 0;
10706     for (uint32 i = 0; i < GetFunction().GetFormalCount(); i++) {
10707         MIRType *ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(GetFunction().GetNthParamTyIdx(i));
10708         CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
10709         parmLocator.LocateNextParm(*ty, pLoc);
10710         if (pLoc.reg0 == kRinvalid) { /* on stack */
10711             stkSize = static_cast<uint32_t>(pLoc.memOffset + pLoc.memSize);
10712         } else {
10713             inReg++;
10714         }
10715     }
10716     if (GetMirModule().GetFlavor() == MIRFlavor::kFlavorLmbc) {
10717         stkSize += (inReg * k8ByteSize);
10718     }
10719     if (CGOptions::IsArm64ilp32()) {
10720         stkSize = static_cast<uint32>(RoundUp(stkSize, k8ByteSize));
10721     } else {
10722         stkSize = static_cast<uint32>(RoundUp(stkSize, GetPointerSize()));
10723     }
10724 
10725     GenCVaStartIntrin(opnd0, stkSize);
10726 
10727     return;
10728 }
10729 
10730 // output
10731 // add_with_overflow/ sub_with_overflow:
10732 //    w1: parm1
10733 //    w2: parm2
10734 //    adds/subs w0, w1, w2
10735 //    cset w3, vs
10736 
10737 // mul_with_overflow:
10738 //    w1: parm1
10739 //    w2: parm2
10740 //    smull x0, w0, w1
10741 //    cmp   x0, w0, sxtw
10742 //    cset  w4, ne
SelectOverFlowCall(const IntrinsiccallNode & intrnNode)10743 void AArch64CGFunc::SelectOverFlowCall(const IntrinsiccallNode &intrnNode)
10744 {
10745     DEBUG_ASSERT(intrnNode.NumOpnds() == 2, "must be 2 operands"); // must be 2 operands
10746     MIRIntrinsicID intrinsic = intrnNode.GetIntrinsic();
10747     PrimType type = intrnNode.Opnd(0)->GetPrimType();
10748     PrimType type2 = intrnNode.Opnd(1)->GetPrimType();
10749     CHECK_FATAL(type == PTY_i32 || type == PTY_u32, "only support i32 or u32 here");
10750     CHECK_FATAL(type2 == PTY_i32 || type == PTY_u32, "only support i32 or u32 here");
10751     // deal with parms
10752     RegOperand &opnd0 = LoadIntoRegister(*HandleExpr(intrnNode, *intrnNode.Opnd(0)),
10753                                          intrnNode.Opnd(0)->GetPrimType()); /* first argument of intrinsic */
10754     RegOperand &opnd1 = LoadIntoRegister(*HandleExpr(intrnNode, *intrnNode.Opnd(1)),
10755                                          intrnNode.Opnd(1)->GetPrimType()); /* first argument of intrinsic */
10756     RegOperand &resReg = CreateRegisterOperandOfType(type);
10757     RegOperand &resReg2 = CreateRegisterOperandOfType(PTY_u8);
10758     Operand &rflag = GetOrCreateRflag();
10759     // arith operation with set flag
10760     if (intrinsic == INTRN_ADD_WITH_OVERFLOW) {
10761         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_waddsrrr, rflag, resReg, opnd0, opnd1));
10762         SelectAArch64CSet(resReg2, GetCondOperand(CC_VS), false);
10763     } else if (intrinsic == INTRN_SUB_WITH_OVERFLOW) {
10764         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wsubsrrr, rflag, resReg, opnd0, opnd1));
10765         SelectAArch64CSet(resReg2, GetCondOperand(CC_VS), false);
10766     } else if (intrinsic == INTRN_MUL_WITH_OVERFLOW) {
10767         // smull
10768         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xsmullrrr, resReg, opnd0, opnd1));
10769         Operand &sxtw = CreateExtendShiftOperand(ExtendShiftOperand::kSXTW, 0, k3BitSize);
10770         Insn &cmpInsn = GetInsnBuilder()->BuildInsn(MOP_xwcmprre, rflag, resReg, resReg, sxtw);
10771         GetCurBB()->AppendInsn(cmpInsn);
10772         SelectAArch64CSet(resReg2, GetCondOperand(CC_NE), false);
10773     } else {
10774         CHECK_FATAL(false, "niy");
10775     }
10776     // store back
10777     auto *retVals = &intrnNode.GetReturnVec();
10778     auto &pair = retVals->at(0);
10779     stIdx2OverflowResult[pair.first] = std::pair<RegOperand*, RegOperand*>(&resReg, &resReg2);
10780     return;
10781 }
10782 
10783 /*
10784  * intrinsiccall C___Atomic_store_N(ptr, val, memorder))
10785  * ====> *ptr = val
10786  * let ptr -> x0
10787  * let val -> x1
10788  * implement to asm: str/stlr x1, [x0]
10789  * a store-release would replace str if memorder is not 0
10790  */
SelectCAtomicStoreN(const IntrinsiccallNode & intrinsiccallNode)10791 void AArch64CGFunc::SelectCAtomicStoreN(const IntrinsiccallNode &intrinsiccallNode)
10792 {
10793     auto primType = intrinsiccallNode.Opnd(1)->GetPrimType();
10794     auto *addr = HandleExpr(intrinsiccallNode, *intrinsiccallNode.Opnd(0));
10795     auto *value = HandleExpr(intrinsiccallNode, *intrinsiccallNode.Opnd(1));
10796     auto *memOrderOpnd = intrinsiccallNode.Opnd(kInsnThirdOpnd);
10797     auto *memOrderConst = static_cast<MIRIntConst *>(static_cast<ConstvalNode *>(memOrderOpnd)->GetConstVal());
10798     auto memOrder = static_cast<std::memory_order>(memOrderConst->GetExtValue());
10799     SelectAtomicStore(*value, *addr, primType, PickMemOrder(memOrder, false));
10800 }
10801 
SelectAtomicStore(Operand & srcOpnd,Operand & addrOpnd,PrimType primType,AArch64isa::MemoryOrdering memOrder)10802 void AArch64CGFunc::SelectAtomicStore(Operand &srcOpnd, Operand &addrOpnd, PrimType primType,
10803                                       AArch64isa::MemoryOrdering memOrder)
10804 {
10805     auto &memOpnd = CreateMemOpnd(LoadIntoRegister(addrOpnd, PTY_a64), 0, k64BitSize);
10806     auto mOp = PickStInsn(GetPrimTypeBitSize(primType), primType, memOrder);
10807     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, LoadIntoRegister(srcOpnd, primType), memOpnd));
10808 }
10809 
SelectAddrofThreadLocal(Operand & result,StImmOperand & stImm)10810 void AArch64CGFunc::SelectAddrofThreadLocal(Operand &result, StImmOperand &stImm)
10811 {
10812     if (CGOptions::IsPIC()) {
10813         SelectCTlsGlobalDesc(result, stImm);
10814     } else {
10815         SelectCTlsLocalDesc(result, stImm);
10816     }
10817     if (stImm.GetOffset() > 0) {
10818         auto &immOpnd = CreateImmOperand(stImm.GetOffset(), result.GetSize(), false);
10819         SelectAdd(result, result, immOpnd, PTY_u64);
10820     }
10821 }
10822 
SelectCTlsLocalDesc(Operand & result,StImmOperand & stImm)10823 void AArch64CGFunc::SelectCTlsLocalDesc(Operand &result, StImmOperand &stImm)
10824 {
10825     auto tpidr = &CreateCommentOperand("tpidr_el0");
10826     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_mrs, result, *tpidr));
10827     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_tls_desc_rel, result, result, stImm));
10828 }
10829 
SelectCTlsGlobalDesc(Operand & result,StImmOperand & stImm)10830 void AArch64CGFunc::SelectCTlsGlobalDesc(Operand &result, StImmOperand &stImm)
10831 {
10832     /* according to AArch64 Machine Directives */
10833     auto &r0opnd = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_u64));
10834     RegOperand *tlsAddr = &CreateRegisterOperandOfType(PTY_u64);
10835     RegOperand *specialFunc = &CreateRegisterOperandOfType(PTY_u64);
10836     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_tls_desc_call, r0opnd, *tlsAddr, stImm));
10837     /* release tls address */
10838     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_pseduo_tls_release, *tlsAddr));
10839     //  mrs xn, tpidr_el0
10840     //  add x0, x0, xn
10841     auto tpidr = &CreateCommentOperand("tpidr_el0");
10842     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_mrs, *specialFunc, *tpidr));
10843     SelectAdd(result, r0opnd, *specialFunc, PTY_u64);
10844 }
10845 
SelectIntrinCall(IntrinsiccallNode & intrinsiccallNode)10846 void AArch64CGFunc::SelectIntrinCall(IntrinsiccallNode &intrinsiccallNode)
10847 {
10848     MIRIntrinsicID intrinsic = intrinsiccallNode.GetIntrinsic();
10849 
10850     if (GetCG()->GenerateVerboseCG()) {
10851         std::string comment = GetIntrinsicName(intrinsic);
10852         GetCurBB()->AppendInsn(CreateCommentInsn(comment));
10853     }
10854 
10855     /*
10856      * At this moment, we eagerly evaluates all argument expressions.  In theory,
10857      * there could be intrinsics that extract meta-information of variables, such as
10858      * their locations, rather than computing their values.  Applications
10859      * include building stack maps that help runtime libraries to find the values
10860      * of local variables (See @stackmap in LLVM), in which case knowing their
10861      * locations will suffice.
10862      */
10863     if (intrinsic == INTRN_MPL_CLINIT_CHECK) { /* special case */
10864         SelectMPLClinitCheck(intrinsiccallNode);
10865         return;
10866     }
10867     if (intrinsic == INTRN_MPL_PROF_COUNTER_INC) { /* special case */
10868         SelectMPLProfCounterInc(intrinsiccallNode);
10869         return;
10870     }
10871     if ((intrinsic == INTRN_MPL_CLEANUP_LOCALREFVARS) || (intrinsic == INTRN_MPL_CLEANUP_LOCALREFVARS_SKIP) ||
10872         (intrinsic == INTRN_MPL_CLEANUP_NORETESCOBJS)) {
10873         return;
10874     }
10875     // js
10876     if (intrinsic == INTRN_ADD_WITH_OVERFLOW || intrinsic == INTRN_SUB_WITH_OVERFLOW ||
10877         intrinsic == INTRN_MUL_WITH_OVERFLOW) {
10878         SelectOverFlowCall(intrinsiccallNode);
10879         return;
10880     }
10881     switch (intrinsic) {
10882         case INTRN_C_va_start:
10883             SelectCVaStart(intrinsiccallNode);
10884             return;
10885         case INTRN_C___sync_lock_release_1:
10886             SelectCSyncLockRelease(intrinsiccallNode, PTY_u8);
10887             return;
10888         case INTRN_C___sync_lock_release_2:
10889             SelectCSyncLockRelease(intrinsiccallNode, PTY_u16);
10890             return;
10891         case INTRN_C___sync_lock_release_4:
10892             SelectCSyncLockRelease(intrinsiccallNode, PTY_u32);
10893             return;
10894         case INTRN_C___sync_lock_release_8:
10895             SelectCSyncLockRelease(intrinsiccallNode, PTY_u64);
10896             return;
10897         case INTRN_C___atomic_store_n:
10898             SelectCAtomicStoreN(intrinsiccallNode);
10899             return;
10900         case INTRN_vector_zip_v8u8:
10901         case INTRN_vector_zip_v8i8:
10902         case INTRN_vector_zip_v4u16:
10903         case INTRN_vector_zip_v4i16:
10904         case INTRN_vector_zip_v2u32:
10905         case INTRN_vector_zip_v2i32:
10906             SelectVectorZip(intrinsiccallNode.Opnd(0)->GetPrimType(),
10907                             HandleExpr(intrinsiccallNode, *intrinsiccallNode.Opnd(0)),
10908                             HandleExpr(intrinsiccallNode, *intrinsiccallNode.Opnd(1)));
10909             return;
10910         case INTRN_C_stack_save:
10911             return;
10912         case INTRN_C_stack_restore:
10913             return;
10914         default:
10915             break;
10916     }
10917     std::vector<Operand *> operands; /* Temporary.  Deallocated on return. */
10918     ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
10919     for (size_t i = 0; i < intrinsiccallNode.NumOpnds(); i++) {
10920         BaseNode *argExpr = intrinsiccallNode.Opnd(i);
10921         Operand *opnd = HandleExpr(intrinsiccallNode, *argExpr);
10922         operands.emplace_back(opnd);
10923         if (!opnd->IsRegister()) {
10924             opnd = &LoadIntoRegister(*opnd, argExpr->GetPrimType());
10925         }
10926         RegOperand *expRegOpnd = static_cast<RegOperand *>(opnd);
10927         srcOpnds->PushOpnd(*expRegOpnd);
10928     }
10929     CallReturnVector *retVals = &intrinsiccallNode.GetReturnVec();
10930 
10931     switch (intrinsic) {
10932         case INTRN_MPL_ATOMIC_EXCHANGE_PTR: {
10933             BB *origFtBB = GetCurBB()->GetNext();
10934             Operand *loc = operands[kInsnFirstOpnd];
10935             Operand *newVal = operands[kInsnSecondOpnd];
10936             Operand *memOrd = operands[kInsnThirdOpnd];
10937 
10938             MemOrd ord = OperandToMemOrd(*memOrd);
10939             bool isAcquire = MemOrdIsAcquire(ord);
10940             bool isRelease = MemOrdIsRelease(ord);
10941 
10942             const PrimType kValPrimType = PTY_a64;
10943 
10944             RegOperand &locReg = LoadIntoRegister(*loc, PTY_a64);
10945             /* Because there is no live analysis when -O1 */
10946             if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) {
10947                 locReg.SetRegNotBBLocal();
10948             }
10949             MemOperand &locMem = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, &locReg, nullptr,
10950                                                     &GetOrCreateOfstOpnd(0, k32BitSize), nullptr);
10951             RegOperand &newValReg = LoadIntoRegister(*newVal, PTY_a64);
10952             if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) {
10953                 newValReg.SetRegNotBBLocal();
10954             }
10955             GetCurBB()->SetKind(BB::kBBFallthru);
10956 
10957             LabelIdx retryLabIdx = CreateLabeledBB(intrinsiccallNode);
10958 
10959             RegOperand *oldVal = SelectLoadExcl(kValPrimType, locMem, isAcquire);
10960             if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) {
10961                 oldVal->SetRegNotBBLocal();
10962             }
10963             RegOperand *succ = SelectStoreExcl(kValPrimType, locMem, newValReg, isRelease);
10964             if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) {
10965                 succ->SetRegNotBBLocal();
10966             }
10967 
10968             GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wcbnz, *succ, GetOrCreateLabelOperand(retryLabIdx)));
10969             GetCurBB()->SetKind(BB::kBBIntrinsic);
10970             GetCurBB()->SetNext(origFtBB);
10971 
10972             SaveReturnValueInLocal(*retVals, 0, kValPrimType, *oldVal, intrinsiccallNode);
10973             break;
10974         }
10975         case INTRN_GET_AND_ADDI: {
10976             IntrinsifyGetAndAddInt(*srcOpnds, PTY_i32);
10977             break;
10978         }
10979         case INTRN_GET_AND_ADDL: {
10980             IntrinsifyGetAndAddInt(*srcOpnds, PTY_i64);
10981             break;
10982         }
10983         case INTRN_GET_AND_SETI: {
10984             IntrinsifyGetAndSetInt(*srcOpnds, PTY_i32);
10985             break;
10986         }
10987         case INTRN_GET_AND_SETL: {
10988             IntrinsifyGetAndSetInt(*srcOpnds, PTY_i64);
10989             break;
10990         }
10991         case INTRN_COMP_AND_SWAPI: {
10992             IntrinsifyCompareAndSwapInt(*srcOpnds, PTY_i32);
10993             break;
10994         }
10995         case INTRN_COMP_AND_SWAPL: {
10996             IntrinsifyCompareAndSwapInt(*srcOpnds, PTY_i64);
10997             break;
10998         }
10999         default: {
11000             CHECK_FATAL(false, "Intrinsic %d: %s not implemented by the AArch64 CG.", intrinsic,
11001                         GetIntrinsicName(intrinsic));
11002             break;
11003         }
11004     }
11005 }
11006 
SelectCclz(IntrinsicopNode & intrnNode)11007 Operand *AArch64CGFunc::SelectCclz(IntrinsicopNode &intrnNode)
11008 {
11009     BaseNode *argexpr = intrnNode.Opnd(0);
11010     PrimType ptype = argexpr->GetPrimType();
11011     Operand *opnd = HandleExpr(intrnNode, *argexpr);
11012     MOperator mop;
11013 
11014     RegOperand &ldDest = CreateRegisterOperandOfType(ptype);
11015     if (opnd->IsMemoryAccessOperand()) {
11016         Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype), ptype), ldDest, *opnd);
11017         GetCurBB()->AppendInsn(insn);
11018         opnd = &ldDest;
11019     } else if (opnd->IsImmediate()) {
11020         SelectCopyImm(ldDest, *static_cast<ImmOperand *>(opnd), ptype);
11021         opnd = &ldDest;
11022     }
11023 
11024     if (GetPrimTypeSize(ptype) == k4ByteSize) {
11025         mop = MOP_wclz;
11026     } else {
11027         mop = MOP_xclz;
11028     }
11029     RegOperand &dst = CreateRegisterOperandOfType(ptype);
11030     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mop, dst, *opnd));
11031     return &dst;
11032 }
11033 
SelectCctz(IntrinsicopNode & intrnNode)11034 Operand *AArch64CGFunc::SelectCctz(IntrinsicopNode &intrnNode)
11035 {
11036     BaseNode *argexpr = intrnNode.Opnd(0);
11037     PrimType ptype = argexpr->GetPrimType();
11038     Operand *opnd = HandleExpr(intrnNode, *argexpr);
11039 
11040     RegOperand &ldDest = CreateRegisterOperandOfType(ptype);
11041     if (opnd->IsMemoryAccessOperand()) {
11042         Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype), ptype), ldDest, *opnd);
11043         GetCurBB()->AppendInsn(insn);
11044         opnd = &ldDest;
11045     } else if (opnd->IsImmediate()) {
11046         SelectCopyImm(ldDest, *static_cast<ImmOperand *>(opnd), ptype);
11047         opnd = &ldDest;
11048     }
11049 
11050     MOperator clzmop;
11051     MOperator rbitmop;
11052     if (GetPrimTypeSize(ptype) == k4ByteSize) {
11053         clzmop = MOP_wclz;
11054         rbitmop = MOP_wrbit;
11055     } else {
11056         clzmop = MOP_xclz;
11057         rbitmop = MOP_xrbit;
11058     }
11059     RegOperand &dst1 = CreateRegisterOperandOfType(ptype);
11060     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(rbitmop, dst1, *opnd));
11061     RegOperand &dst2 = CreateRegisterOperandOfType(ptype);
11062     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(clzmop, dst2, dst1));
11063     return &dst2;
11064 }
11065 
SelectCpopcount(IntrinsicopNode & intrnNode)11066 Operand *AArch64CGFunc::SelectCpopcount(IntrinsicopNode &intrnNode)
11067 {
11068     CHECK_FATAL(false, "%s NIY", intrnNode.GetIntrinDesc().name);
11069     return nullptr;
11070 }
11071 
SelectCparity(IntrinsicopNode & intrnNode)11072 Operand *AArch64CGFunc::SelectCparity(IntrinsicopNode &intrnNode)
11073 {
11074     CHECK_FATAL(false, "%s NIY", intrnNode.GetIntrinDesc().name);
11075     return nullptr;
11076 }
11077 
SelectCclrsb(IntrinsicopNode & intrnNode)11078 Operand *AArch64CGFunc::SelectCclrsb(IntrinsicopNode &intrnNode)
11079 {
11080     BaseNode *argexpr = intrnNode.Opnd(0);
11081     PrimType ptype = argexpr->GetPrimType();
11082     Operand *opnd = HandleExpr(intrnNode, *argexpr);
11083 
11084     RegOperand &ldDest = CreateRegisterOperandOfType(ptype);
11085     if (opnd->IsMemoryAccessOperand()) {
11086         Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype), ptype), ldDest, *opnd);
11087         GetCurBB()->AppendInsn(insn);
11088         opnd = &ldDest;
11089     } else if (opnd->IsImmediate()) {
11090         SelectCopyImm(ldDest, *static_cast<ImmOperand *>(opnd), ptype);
11091         opnd = &ldDest;
11092     }
11093 
11094     bool is32Bit = (GetPrimTypeSize(ptype) == k4ByteSize);
11095     RegOperand &res = CreateRegisterOperandOfType(ptype);
11096     SelectMvn(res, *opnd, ptype);
11097     SelectAArch64Cmp(*opnd, GetZeroOpnd(is32Bit ? k32BitSize : k64BitSize), true, is32Bit ? k32BitSize : k64BitSize);
11098     SelectAArch64Select(*opnd, res, *opnd, GetCondOperand(CC_LT), true, is32Bit ? k32BitSize : k64BitSize);
11099     MOperator clzmop = (is32Bit ? MOP_wclz : MOP_xclz);
11100     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(clzmop, *opnd, *opnd));
11101     SelectSub(*opnd, *opnd, CreateImmOperand(1, is32Bit ? k32BitSize : k64BitSize, true), ptype);
11102     return opnd;
11103 }
11104 
SelectCisaligned(IntrinsicopNode & intrnNode)11105 Operand *AArch64CGFunc::SelectCisaligned(IntrinsicopNode &intrnNode)
11106 {
11107     BaseNode *argexpr0 = intrnNode.Opnd(0);
11108     PrimType ptype0 = argexpr0->GetPrimType();
11109     Operand *opnd0 = HandleExpr(intrnNode, *argexpr0);
11110 
11111     RegOperand &ldDest0 = CreateRegisterOperandOfType(ptype0);
11112     if (opnd0->IsMemoryAccessOperand()) {
11113         GetCurBB()->AppendInsn(
11114             GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype0), ptype0), ldDest0, *opnd0));
11115         opnd0 = &ldDest0;
11116     } else if (opnd0->IsImmediate()) {
11117         SelectCopyImm(ldDest0, *static_cast<ImmOperand *>(opnd0), ptype0);
11118         opnd0 = &ldDest0;
11119     }
11120 
11121     BaseNode *argexpr1 = intrnNode.Opnd(1);
11122     PrimType ptype1 = argexpr1->GetPrimType();
11123     Operand *opnd1 = HandleExpr(intrnNode, *argexpr1);
11124 
11125     RegOperand &ldDest1 = CreateRegisterOperandOfType(ptype1);
11126     if (opnd1->IsMemoryAccessOperand()) {
11127         GetCurBB()->AppendInsn(
11128             GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype1), ptype1), ldDest1, *opnd1));
11129         opnd1 = &ldDest1;
11130     } else if (opnd1->IsImmediate()) {
11131         SelectCopyImm(ldDest1, *static_cast<ImmOperand *>(opnd1), ptype1);
11132         opnd1 = &ldDest1;
11133     }
11134     // mov w4, #1
11135     RegOperand &reg0 = CreateRegisterOperandOfType(PTY_i32);
11136     SelectCopyImm(reg0, CreateImmOperand(1, k32BitSize, true), PTY_i32);
11137     // sxtw x4, w4
11138     MOperator mOp = MOP_xsxtw64;
11139     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, reg0, reg0));
11140     // sub x3, x3, x4
11141     SelectSub(*opnd1, *opnd1, reg0, ptype1);
11142     // and x2, x2, x3
11143     SelectBand(*opnd0, *opnd0, *opnd1, ptype1);
11144     // mov w3, #0
11145     // sxtw x3, w3
11146     // cmp x2, x3
11147     SelectAArch64Cmp(*opnd0, GetZeroOpnd(k64BitSize), true, k64BitSize);
11148     // cset w2, EQ
11149     SelectAArch64CSet(*opnd0, GetCondOperand(CC_EQ), false);
11150     return opnd0;
11151 }
11152 
SelectArithmeticAndLogical(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType,Opcode op)11153 void AArch64CGFunc::SelectArithmeticAndLogical(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType,
11154                                                Opcode op)
11155 {
11156     switch (op) {
11157         case OP_add:
11158             SelectAdd(resOpnd, opnd0, opnd1, primType);
11159             break;
11160         case OP_sub:
11161             SelectSub(resOpnd, opnd0, opnd1, primType);
11162             break;
11163         case OP_band:
11164             SelectBand(resOpnd, opnd0, opnd1, primType);
11165             break;
11166         case OP_bior:
11167             SelectBior(resOpnd, opnd0, opnd1, primType);
11168             break;
11169         case OP_bxor:
11170             SelectBxor(resOpnd, opnd0, opnd1, primType);
11171             break;
11172         default:
11173             CHECK_FATAL(false, "unconcerned opcode for arithmetical and logical insns");
11174             break;
11175     }
11176 }
11177 
SelectAArch64CSyncFetch(const IntrinsicopNode & intrinopNode,Opcode op,bool fetchBefore)11178 Operand *AArch64CGFunc::SelectAArch64CSyncFetch(const IntrinsicopNode &intrinopNode, Opcode op, bool fetchBefore)
11179 {
11180     auto primType = intrinopNode.GetPrimType();
11181     /* Create BB which includes atomic built_in function */
11182     LabelIdx atomicBBLabIdx = CreateLabel();
11183     BB *atomicBB = CreateNewBB();
11184     atomicBB->SetKind(BB::kBBIf);
11185     atomicBB->SetAtomicBuiltIn();
11186     atomicBB->AddLabel(atomicBBLabIdx);
11187     SetLab2BBMap(static_cast<int32>(atomicBBLabIdx), *atomicBB);
11188     GetCurBB()->AppendBB(*atomicBB);
11189     /* keep variables inside same BB */
11190     if (GetCG()->GetOptimizeLevel() == CGOptions::kLevel0) {
11191         SetCurBB(*atomicBB);
11192     }
11193     /* handle built_in args */
11194     Operand *addrOpnd = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnFirstOpnd));
11195     Operand *valueOpnd = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnSecondOpnd));
11196     addrOpnd = &LoadIntoRegister(*addrOpnd, intrinopNode.GetNopndAt(kInsnFirstOpnd)->GetPrimType());
11197     valueOpnd = &LoadIntoRegister(*valueOpnd, intrinopNode.GetNopndAt(kInsnSecondOpnd)->GetPrimType());
11198     if (GetCG()->GetOptimizeLevel() != CGOptions::kLevel0) {
11199         SetCurBB(*atomicBB);
11200     }
11201     /* load from pointed address */
11202     auto primTypeP2Size = GetPrimTypeP2Size(primType);
11203     auto *regLoaded = &CreateRegisterOperandOfType(primType);
11204     auto &memOpnd = CreateMemOpnd(*static_cast<RegOperand *>(addrOpnd), 0, GetPrimTypeBitSize(primType));
11205     auto mOpLoad = PickLoadStoreExclInsn(primTypeP2Size, false, false);
11206     atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(mOpLoad, *regLoaded, memOpnd));
11207     /* update loaded value */
11208     auto *regOperated = &CreateRegisterOperandOfType(primType);
11209     SelectArithmeticAndLogical(*regOperated, *regLoaded, *valueOpnd, primType, op);
11210     /* store to pointed address */
11211     auto *accessStatus = &CreateRegisterOperandOfType(PTY_u32);
11212     auto mOpStore = PickLoadStoreExclInsn(primTypeP2Size, true, true);
11213     atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(mOpStore, *accessStatus, *regOperated, memOpnd));
11214     /* check the exclusive accsess status */
11215     auto &atomicBBOpnd = GetOrCreateLabelOperand(*atomicBB);
11216     atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wcbnz, *accessStatus, atomicBBOpnd));
11217 
11218     /* Data Memory Barrier */
11219     BB *nextBB = CreateNewBB();
11220     atomicBB->AppendBB(*nextBB);
11221     SetCurBB(*nextBB);
11222     nextBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_dmb_ish, AArch64CG::kMd[MOP_dmb_ish]));
11223     return fetchBefore ? regLoaded : regOperated;
11224 }
11225 
SelectCSyncCmpSwap(const IntrinsicopNode & intrinopNode,bool retBool)11226 Operand *AArch64CGFunc::SelectCSyncCmpSwap(const IntrinsicopNode &intrinopNode, bool retBool)
11227 {
11228     PrimType primType = intrinopNode.GetNopndAt(kInsnSecondOpnd)->GetPrimType();
11229     DEBUG_ASSERT(primType == intrinopNode.GetNopndAt(kInsnThirdOpnd)->GetPrimType(), "gcc built_in rule");
11230     LabelIdx atomicBBLabIdx = CreateLabel();
11231     BB *atomicBB = CreateNewBB();
11232     atomicBB->SetKind(BB::kBBIf);
11233     atomicBB->SetAtomicBuiltIn();
11234     atomicBB->AddLabel(atomicBBLabIdx);
11235     SetLab2BBMap(static_cast<int32>(atomicBBLabIdx), *atomicBB);
11236     GetCurBB()->AppendBB(*atomicBB);
11237     if (GetCG()->GetOptimizeLevel() == CGOptions::kLevel0) {
11238         SetCurBB(*atomicBB);
11239     }
11240     /* handle built_in args */
11241     Operand *addrOpnd = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnFirstOpnd));
11242     Operand *oldVal = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnSecondOpnd));
11243     Operand *newVal = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnThirdOpnd));
11244     if (GetCG()->GetOptimizeLevel() != CGOptions::kLevel0) {
11245         SetCurBB(*atomicBB);
11246     }
11247 
11248     uint32 primTypeP2Size = GetPrimTypeP2Size(primType);
11249     /* ldxr */
11250     auto *regLoaded = &CreateRegisterOperandOfType(primType);
11251     auto &memOpnd = CreateMemOpnd(LoadIntoRegister(*addrOpnd, primType), 0, GetPrimTypeBitSize(primType));
11252     auto mOpLoad = PickLoadStoreExclInsn(primTypeP2Size, false, false);
11253     atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(mOpLoad, *regLoaded, memOpnd));
11254     Operand *regExtend = &CreateRegisterOperandOfType(primType);
11255     PrimType targetType = (oldVal->GetSize() <= k32BitSize) ? (IsSignedInteger(primType) ? PTY_i32 : PTY_u32)
11256                                                             : (IsSignedInteger(primType) ? PTY_i64 : PTY_u64);
11257     SelectCvtInt2Int(nullptr, regExtend, regLoaded, primType, targetType);
11258     /* cmp */
11259     SelectAArch64Cmp(*regExtend, *oldVal, true, oldVal->GetSize());
11260     /* bne */
11261     Operand &rflag = GetOrCreateRflag();
11262     LabelIdx nextBBLableIdx = CreateLabel();
11263     LabelOperand &targetOpnd = GetOrCreateLabelOperand(nextBBLableIdx);
11264     atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_bne, rflag, targetOpnd));
11265     /* stlxr */
11266     BB *stlxrBB = CreateNewBB();
11267     stlxrBB->SetKind(BB::kBBIf);
11268     atomicBB->AppendBB(*stlxrBB);
11269     SetCurBB(*stlxrBB);
11270     auto *accessStatus = &CreateRegisterOperandOfType(PTY_u32);
11271     auto &newRegVal = LoadIntoRegister(*newVal, primType);
11272     auto mOpStore = PickLoadStoreExclInsn(primTypeP2Size, true, true);
11273     stlxrBB->AppendInsn(GetInsnBuilder()->BuildInsn(mOpStore, *accessStatus, newRegVal, memOpnd));
11274     /* cbnz ==> check the exclusive accsess status */
11275     auto &atomicBBOpnd = GetOrCreateLabelOperand(*atomicBB);
11276     stlxrBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wcbnz, *accessStatus, atomicBBOpnd));
11277     /* Data Memory Barrier */
11278     BB *nextBB = CreateNewBB();
11279     nextBB->AddLabel(nextBBLableIdx);
11280     nextBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_dmb_ish, AArch64CG::kMd[MOP_dmb_ish]));
11281     SetLab2BBMap(static_cast<int32>(nextBBLableIdx), *nextBB);
11282     stlxrBB->AppendBB(*nextBB);
11283     SetCurBB(*nextBB);
11284     /* bool version return true if the comparison is successful and newval is written */
11285     if (retBool) {
11286         auto *retOpnd = &CreateRegisterOperandOfType(PTY_u32);
11287         SelectAArch64CSet(*retOpnd, GetCondOperand(CC_EQ), false);
11288         return retOpnd;
11289     }
11290     /* type version return the contents of *addrOpnd before the operation */
11291     return regLoaded;
11292 }
11293 
SelectCSyncFetch(IntrinsicopNode & intrinopNode,Opcode op,bool fetchBefore)11294 Operand *AArch64CGFunc::SelectCSyncFetch(IntrinsicopNode &intrinopNode, Opcode op, bool fetchBefore)
11295 {
11296     return SelectAArch64CSyncFetch(intrinopNode, op, fetchBefore);
11297 }
11298 
SelectCSyncBoolCmpSwap(IntrinsicopNode & intrinopNode)11299 Operand *AArch64CGFunc::SelectCSyncBoolCmpSwap(IntrinsicopNode &intrinopNode)
11300 {
11301     return SelectCSyncCmpSwap(intrinopNode, true);
11302 }
11303 
SelectCSyncValCmpSwap(IntrinsicopNode & intrinopNode)11304 Operand *AArch64CGFunc::SelectCSyncValCmpSwap(IntrinsicopNode &intrinopNode)
11305 {
11306     return SelectCSyncCmpSwap(intrinopNode);
11307 }
11308 
SelectCSyncLockTestSet(IntrinsicopNode & intrinopNode,PrimType pty)11309 Operand *AArch64CGFunc::SelectCSyncLockTestSet(IntrinsicopNode &intrinopNode, PrimType pty)
11310 {
11311     auto primType = intrinopNode.GetPrimType();
11312     Operand *addrOpnd = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnFirstOpnd));
11313     Operand *valueOpnd = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnSecondOpnd));
11314     addrOpnd = &LoadIntoRegister(*addrOpnd, intrinopNode.GetNopndAt(kInsnFirstOpnd)->GetPrimType());
11315     valueOpnd = &LoadIntoRegister(*valueOpnd, intrinopNode.GetNopndAt(kInsnSecondOpnd)->GetPrimType());
11316 
11317     /* Create BB which includes atomic built_in function */
11318     LabelIdx atomicBBLabIdx = CreateLabel();
11319     BB *atomicBB = CreateNewBB();
11320     atomicBB->SetKind(BB::kBBIf);
11321     atomicBB->SetAtomicBuiltIn();
11322     atomicBB->AddLabel(atomicBBLabIdx);
11323     SetLab2BBMap(static_cast<int32>(atomicBBLabIdx), *atomicBB);
11324     GetCurBB()->AppendBB(*atomicBB);
11325     SetCurBB(*atomicBB);
11326     /* load from pointed address */
11327     auto primTypeP2Size = GetPrimTypeP2Size(primType);
11328     auto *regLoaded = &CreateRegisterOperandOfType(primType);
11329     auto &memOpnd = CreateMemOpnd(*static_cast<RegOperand *>(addrOpnd), 0, GetPrimTypeBitSize(primType));
11330     auto mOpLoad = PickLoadStoreExclInsn(primTypeP2Size, false, false);
11331     atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(mOpLoad, *regLoaded, memOpnd));
11332     /* store to pointed address */
11333     auto *accessStatus = &CreateRegisterOperandOfType(PTY_u32);
11334     auto mOpStore = PickLoadStoreExclInsn(primTypeP2Size, true, false);
11335     atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(mOpStore, *accessStatus, *valueOpnd, memOpnd));
11336     /* check the exclusive accsess status */
11337     auto &atomicBBOpnd = GetOrCreateLabelOperand(*atomicBB);
11338     atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wcbnz, *accessStatus, atomicBBOpnd));
11339 
11340     /* Data Memory Barrier */
11341     BB *nextBB = CreateNewBB();
11342     atomicBB->AppendBB(*nextBB);
11343     SetCurBB(*nextBB);
11344     nextBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_dmb_ish, AArch64CG::kMd[MOP_dmb_ish]));
11345     return regLoaded;
11346 }
11347 
SelectCSyncLockRelease(const IntrinsiccallNode & intrinsiccall,PrimType primType)11348 void AArch64CGFunc::SelectCSyncLockRelease(const IntrinsiccallNode &intrinsiccall, PrimType primType)
11349 {
11350     auto *addrOpnd = HandleExpr(intrinsiccall, *intrinsiccall.GetNopndAt(kInsnFirstOpnd));
11351     auto primTypeBitSize = GetPrimTypeBitSize(primType);
11352     auto mOp = PickStInsn(primTypeBitSize, primType, AArch64isa::kMoRelease);
11353     auto &zero = GetZeroOpnd(primTypeBitSize);
11354     auto &memOpnd = CreateMemOpnd(LoadIntoRegister(*addrOpnd, primType), 0, primTypeBitSize);
11355     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, zero, memOpnd));
11356 }
11357 
SelectCSyncSynchronize(IntrinsicopNode & intrinopNode)11358 Operand *AArch64CGFunc::SelectCSyncSynchronize(IntrinsicopNode &intrinopNode)
11359 {
11360     (void)intrinopNode;
11361     CHECK_FATAL(false, "have not implement SelectCSyncSynchronize yet");
11362     return nullptr;
11363 }
11364 
PickMemOrder(std::memory_order memOrder,bool isLdr) const11365 AArch64isa::MemoryOrdering AArch64CGFunc::PickMemOrder(std::memory_order memOrder, bool isLdr) const
11366 {
11367     switch (memOrder) {
11368         case std::memory_order_relaxed:
11369             return AArch64isa::kMoNone;
11370         case std::memory_order_consume:
11371         case std::memory_order_acquire:
11372             return isLdr ? AArch64isa::kMoAcquire : AArch64isa::kMoNone;
11373         case std::memory_order_release:
11374             return isLdr ? AArch64isa::kMoNone : AArch64isa::kMoRelease;
11375         case std::memory_order_acq_rel:
11376         case std::memory_order_seq_cst:
11377             return isLdr ? AArch64isa::kMoAcquire : AArch64isa::kMoRelease;
11378         default:
11379             CHECK_FATAL(false, "unexpected memorder");
11380             return AArch64isa::kMoNone;
11381     }
11382 }
11383 
11384 /*
11385  * regassign %1 (intrinsicop C___Atomic_Load_N(ptr, memorder))
11386  * ====> %1 = *ptr
11387  * let %1 -> x0
11388  * let ptr -> x1
11389  * implement to asm: ldr/ldar x0, [x1]
11390  * a load-acquire would replace ldr if memorder is not 0
11391  */
SelectCAtomicLoadN(IntrinsicopNode & intrinsicopNode)11392 Operand *AArch64CGFunc::SelectCAtomicLoadN(IntrinsicopNode &intrinsicopNode)
11393 {
11394     auto *addrOpnd = HandleExpr(intrinsicopNode, *intrinsicopNode.Opnd(0));
11395     auto *memOrderOpnd = intrinsicopNode.Opnd(1);
11396     auto primType = intrinsicopNode.GetPrimType();
11397     auto *memOrderConst = static_cast<MIRIntConst *>(static_cast<ConstvalNode *>(memOrderOpnd)->GetConstVal());
11398     auto memOrder = static_cast<std::memory_order>(memOrderConst->GetExtValue());
11399     return SelectAtomicLoad(*addrOpnd, primType, PickMemOrder(memOrder, true));
11400 }
11401 
11402 /*
11403  * regassign %1 (intrinsicop C___Atomic_exchange_n(ptr, val, memorder))
11404  * ====> %1 = *ptr; *ptr = val;
11405  * let %1 -> x0
11406  * let ptr -> x1
11407  * let val -> x2
11408  * implement to asm:
11409  * ldr/ldar x0, [x1]
11410  * str/stlr x2, [x1]
11411  * a load-acquire would replace ldr if acquire needed
11412  * a store-relase would replace str if release needed
11413  */
SelectCAtomicExchangeN(IntrinsicopNode & intrinsicopNode)11414 Operand *AArch64CGFunc::SelectCAtomicExchangeN(IntrinsicopNode &intrinsicopNode)
11415 {
11416     auto primType = intrinsicopNode.GetPrimType();
11417     auto *addrOpnd = HandleExpr(intrinsicopNode, *intrinsicopNode.Opnd(0));
11418     auto *valueOpnd = HandleExpr(intrinsicopNode, *intrinsicopNode.Opnd(1));
11419     auto *memOrderOpnd = intrinsicopNode.Opnd(kInsnThirdOpnd);
11420     auto *memOrderConst = static_cast<MIRIntConst *>(static_cast<ConstvalNode *>(memOrderOpnd)->GetConstVal());
11421     auto memOrder = static_cast<std::memory_order>(memOrderConst->GetExtValue());
11422     auto *result = SelectAtomicLoad(*addrOpnd, primType, PickMemOrder(memOrder, true));
11423     SelectAtomicStore(*valueOpnd, *addrOpnd, primType, PickMemOrder(memOrder, false));
11424     return result;
11425 }
11426 
SelectAtomicLoad(Operand & addrOpnd,PrimType primType,AArch64isa::MemoryOrdering memOrder)11427 Operand *AArch64CGFunc::SelectAtomicLoad(Operand &addrOpnd, PrimType primType, AArch64isa::MemoryOrdering memOrder)
11428 {
11429     auto mOp = PickLdInsn(GetPrimTypeBitSize(primType), primType, memOrder);
11430     auto &memOpnd = CreateMemOpnd(LoadIntoRegister(addrOpnd, PTY_a64), 0, k64BitSize);
11431     auto *resultOpnd = &CreateRegisterOperandOfType(primType);
11432     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, *resultOpnd, memOpnd));
11433     return resultOpnd;
11434 }
11435 
SelectCReturnAddress(IntrinsicopNode & intrinopNode)11436 Operand *AArch64CGFunc::SelectCReturnAddress(IntrinsicopNode &intrinopNode)
11437 {
11438     if (intrinopNode.GetIntrinsic() == INTRN_C__builtin_extract_return_addr) {
11439         DEBUG_ASSERT(intrinopNode.GetNumOpnds() == 1, "expect one parameter");
11440         Operand *addrOpnd = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnFirstOpnd));
11441         return &LoadIntoRegister(*addrOpnd, PTY_a64);
11442     } else if (intrinopNode.GetIntrinsic() == INTRN_C__builtin_return_address) {
11443         BaseNode *argexpr0 = intrinopNode.Opnd(0);
11444         while (!argexpr0->IsLeaf()) {
11445             argexpr0 = argexpr0->Opnd(0);
11446         }
11447         CHECK_FATAL(argexpr0->IsConstval(), "Invalid argument of __builtin_return_address");
11448         auto &constNode = static_cast<ConstvalNode &>(*argexpr0);
11449         DEBUG_ASSERT(constNode.GetConstVal()->GetKind() == kConstInt, "expect MIRIntConst does not support float yet");
11450         MIRIntConst *mirIntConst = safe_cast<MIRIntConst>(constNode.GetConstVal());
11451         DEBUG_ASSERT(mirIntConst != nullptr, "nullptr checking");
11452         int64 scale = mirIntConst->GetExtValue();
11453         /*
11454          * Do not support getting return address with a nonzero argument
11455          * inline / tail call opt will destory this behavior
11456          */
11457         CHECK_FATAL(scale == 0, "Do not support recursion");
11458         Operand *resReg = &static_cast<Operand &>(CreateRegisterOperandOfType(PTY_i64));
11459         SelectCopy(*resReg, PTY_i64, GetOrCreatePhysicalRegisterOperand(RLR, k64BitSize, kRegTyInt), PTY_i64);
11460         return resReg;
11461     }
11462     return nullptr;
11463 }
11464 
SelectCalignup(IntrinsicopNode & intrnNode)11465 Operand *AArch64CGFunc::SelectCalignup(IntrinsicopNode &intrnNode)
11466 {
11467     return SelectAArch64align(intrnNode, true);
11468 }
11469 
SelectCaligndown(IntrinsicopNode & intrnNode)11470 Operand *AArch64CGFunc::SelectCaligndown(IntrinsicopNode &intrnNode)
11471 {
11472     return SelectAArch64align(intrnNode, false);
11473 }
11474 
SelectAArch64align(const IntrinsicopNode & intrnNode,bool isUp)11475 Operand *AArch64CGFunc::SelectAArch64align(const IntrinsicopNode &intrnNode, bool isUp)
11476 {
11477     /* Handle Two args */
11478     BaseNode *argexpr0 = intrnNode.Opnd(0);
11479     PrimType ptype0 = argexpr0->GetPrimType();
11480     Operand *opnd0 = HandleExpr(intrnNode, *argexpr0);
11481     PrimType resultPtype = intrnNode.GetPrimType();
11482     RegOperand &ldDest0 = LoadIntoRegister(*opnd0, ptype0);
11483 
11484     BaseNode *argexpr1 = intrnNode.Opnd(1);
11485     PrimType ptype1 = argexpr1->GetPrimType();
11486     Operand *opnd1 = HandleExpr(intrnNode, *argexpr1);
11487     RegOperand &arg1 = LoadIntoRegister(*opnd1, ptype1);
11488     DEBUG_ASSERT(IsPrimitiveInteger(ptype0) && IsPrimitiveInteger(ptype1), "align integer type only");
11489     Operand *ldDest1 = &static_cast<Operand &>(CreateRegisterOperandOfType(ptype0));
11490     SelectCvtInt2Int(nullptr, ldDest1, &arg1, ptype1, ptype0);
11491 
11492     Operand *resultReg = &static_cast<Operand &>(CreateRegisterOperandOfType(ptype0));
11493     Operand &immReg = CreateImmOperand(1, GetPrimTypeBitSize(ptype0), true);
11494     /* Do alignment  x0 -- value to be aligned   x1 -- alignment */
11495     if (isUp) {
11496         /* add res, x0, x1 */
11497         SelectAdd(*resultReg, ldDest0, *ldDest1, ptype0);
11498         /* sub res, res, 1 */
11499         SelectSub(*resultReg, *resultReg, immReg, ptype0);
11500     }
11501     Operand *tempReg = &static_cast<Operand &>(CreateRegisterOperandOfType(ptype0));
11502     /* sub temp, x1, 1 */
11503     SelectSub(*tempReg, *ldDest1, immReg, ptype0);
11504     /* mvn temp, temp */
11505     SelectMvn(*tempReg, *tempReg, ptype0);
11506     /* and res, res, temp */
11507     if (isUp) {
11508         SelectBand(*resultReg, *resultReg, *tempReg, ptype0);
11509     } else {
11510         SelectBand(*resultReg, ldDest0, *tempReg, ptype0);
11511     }
11512     if (resultPtype != ptype0) {
11513         SelectCvtInt2Int(&intrnNode, resultReg, resultReg, ptype0, resultPtype);
11514     }
11515     return resultReg;
11516 }
11517 
11518 /*
11519  * NOTE: consider moving the following things into aarch64_cg.cpp  They may
11520  * serve not only inrinsics, but other MapleIR instructions as well.
11521  * Do it as if we are adding a label in straight-line assembly code.
11522  */
CreateLabeledBB(StmtNode & stmt)11523 LabelIdx AArch64CGFunc::CreateLabeledBB(StmtNode &stmt)
11524 {
11525     LabelIdx labIdx = CreateLabel();
11526     BB *newBB = StartNewBBImpl(false, stmt);
11527     newBB->AddLabel(labIdx);
11528     SetLab2BBMap(labIdx, *newBB);
11529     SetCurBB(*newBB);
11530     return labIdx;
11531 }
11532 
11533 /* Save value into the local variable for the index-th return value; */
SaveReturnValueInLocal(CallReturnVector & retVals,size_t index,PrimType primType,Operand & value,StmtNode & parentStmt)11534 void AArch64CGFunc::SaveReturnValueInLocal(CallReturnVector &retVals, size_t index, PrimType primType, Operand &value,
11535                                            StmtNode &parentStmt)
11536 {
11537     CallReturnPair &pair = retVals.at(index);
11538     BB tempBB(static_cast<uint32>(-1), *GetFuncScopeAllocator());
11539     BB *realCurBB = GetCurBB();
11540     CHECK_FATAL(!pair.second.IsReg(), "NYI");
11541     Operand *destOpnd = &value;
11542     /* for O0 ,corss-BB var is not support, do extra store/load but why new BB */
11543     if (GetCG()->GetOptimizeLevel() == CGOptions::kLevel0) {
11544         MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(pair.first);
11545         MIRType *sPty = symbol->GetType();
11546         PrimType ty = symbol->GetType()->GetPrimType();
11547         if (sPty->GetKind() == kTypeStruct || sPty->GetKind() == kTypeUnion) {
11548             MIRStructType *structType = static_cast<MIRStructType *>(sPty);
11549             ty = structType->GetFieldType(pair.second.GetFieldID())->GetPrimType();
11550         } else if (sPty->GetKind() == kTypeClass) {
11551             CHECK_FATAL(false, "unsuppotr type for inlineasm / intrinsic");
11552         }
11553         RegOperand &tempReg = CreateVirtualRegisterOperand(NewVReg(GetRegTyFromPrimTy(ty), GetPrimTypeSize(ty)));
11554         SelectCopy(tempReg, ty, value, ty);
11555         destOpnd = &tempReg;
11556     }
11557     SetCurBB(tempBB);
11558     SelectDassign(pair.first, pair.second.GetFieldID(), primType, *destOpnd);
11559 
11560     CHECK_FATAL(realCurBB->GetNext() == nullptr, "current BB must has not nextBB");
11561     realCurBB->SetLastStmt(parentStmt);
11562     realCurBB->SetNext(StartNewBBImpl(true, parentStmt));
11563     realCurBB->GetNext()->SetKind(BB::kBBFallthru);
11564     realCurBB->GetNext()->SetPrev(realCurBB);
11565 
11566     realCurBB->GetNext()->InsertAtBeginning(*GetCurBB());
11567     /* restore it */
11568     SetCurBB(*realCurBB->GetNext());
11569 }
11570 
11571 /* The following are translation of LL/SC and atomic RMW operations */
OperandToMemOrd(Operand & opnd) const11572 MemOrd AArch64CGFunc::OperandToMemOrd(Operand &opnd) const
11573 {
11574     CHECK_FATAL(opnd.IsImmediate(), "Memory order must be an int constant.");
11575     auto immOpnd = static_cast<ImmOperand *>(&opnd);
11576     int32 val = immOpnd->GetValue();
11577     CHECK_FATAL(val >= 0, "val must be non-negtive");
11578     return MemOrdFromU32(static_cast<uint32>(val));
11579 }
11580 
11581 /*
11582  * Generate ldxr or ldaxr instruction.
11583  * byte_p2x: power-of-2 size of operand in bytes (0: 1B, 1: 2B, 2: 4B, 3: 8B).
11584  */
PickLoadStoreExclInsn(uint32 byteP2Size,bool store,bool acqRel) const11585 MOperator AArch64CGFunc::PickLoadStoreExclInsn(uint32 byteP2Size, bool store, bool acqRel) const
11586 {
11587     CHECK_FATAL(byteP2Size < kIntByteSizeDimension, "Illegal argument p2size: %d", byteP2Size);
11588 
11589     static MOperator operators[4][2][2] = {{{MOP_wldxrb, MOP_wldaxrb}, {MOP_wstxrb, MOP_wstlxrb}},
11590                                            {{MOP_wldxrh, MOP_wldaxrh}, {MOP_wstxrh, MOP_wstlxrh}},
11591                                            {{MOP_wldxr, MOP_wldaxr}, {MOP_wstxr, MOP_wstlxr}},
11592                                            {{MOP_xldxr, MOP_xldaxr}, {MOP_xstxr, MOP_xstlxr}}};
11593 
11594     MOperator optr = operators[byteP2Size][store][acqRel];
11595     CHECK_FATAL(optr != MOP_undef, "Unsupported type p2size: %d", byteP2Size);
11596 
11597     return optr;
11598 }
11599 
SelectLoadExcl(PrimType valPrimType,MemOperand & loc,bool acquire)11600 RegOperand *AArch64CGFunc::SelectLoadExcl(PrimType valPrimType, MemOperand &loc, bool acquire)
11601 {
11602     uint32 p2size = GetPrimTypeP2Size(valPrimType);
11603 
11604     RegOperand &result = CreateRegisterOperandOfType(valPrimType);
11605     MOperator mOp = PickLoadStoreExclInsn(p2size, false, acquire);
11606     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, loc));
11607 
11608     return &result;
11609 }
11610 
SelectStoreExcl(PrimType valPty,MemOperand & loc,RegOperand & newVal,bool release)11611 RegOperand *AArch64CGFunc::SelectStoreExcl(PrimType valPty, MemOperand &loc, RegOperand &newVal, bool release)
11612 {
11613     uint32 p2size = GetPrimTypeP2Size(valPty);
11614 
11615     /* the result (success/fail) is to be stored in a 32-bit register */
11616     RegOperand &result = CreateRegisterOperandOfType(PTY_u32);
11617 
11618     MOperator mOp = PickLoadStoreExclInsn(p2size, true, release);
11619     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, newVal, loc));
11620 
11621     return &result;
11622 }
11623 
GetRegisterType(regno_t reg) const11624 RegType AArch64CGFunc::GetRegisterType(regno_t reg) const
11625 {
11626     if (AArch64isa::IsPhysicalRegister(reg)) {
11627         return AArch64isa::GetRegType(static_cast<AArch64reg>(reg));
11628     } else if (reg == kRFLAG) {
11629         return kRegTyCc;
11630     } else {
11631         return CGFunc::GetRegisterType(reg);
11632     }
11633 }
11634 
LoadStructCopyBase(const MIRSymbol & symbol,int64 offset,int dataSize)11635 MemOperand &AArch64CGFunc::LoadStructCopyBase(const MIRSymbol &symbol, int64 offset, int dataSize)
11636 {
11637     /* For struct formals > 16 bytes, this is the pointer to the struct copy. */
11638     /* Load the base pointer first. */
11639     RegOperand *vreg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
11640     MemOperand *baseMemOpnd = &GetOrCreateMemOpnd(symbol, 0, k64BitSize);
11641     GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_i64), *vreg, *baseMemOpnd));
11642     /* Create the indirect load mem opnd from the base pointer. */
11643     return CreateMemOpnd(*vreg, offset, static_cast<uint32>(dataSize));
11644 }
11645 
11646 /* For long branch, insert an unconditional branch.
11647  * From                      To
11648  *   cond_br targe_label       reverse_cond_br fallthru_label
11649  *   fallthruBB                unconditional br target_label
11650  *                             fallthru_label:
11651  *                             fallthruBB
11652  */
InsertJumpPad(Insn * insn)11653 void AArch64CGFunc::InsertJumpPad(Insn *insn)
11654 {
11655     BB *bb = insn->GetBB();
11656     DEBUG_ASSERT(bb, "instruction has no bb");
11657     DEBUG_ASSERT(bb->GetKind() == BB::kBBIf || bb->GetKind() == BB::kBBGoto,
11658                  "instruction is in neither if bb nor goto bb");
11659     if (bb->GetKind() == BB::kBBGoto) {
11660         return;
11661     }
11662     DEBUG_ASSERT(bb->NumSuccs() == k2ByteSize, "if bb should have 2 successors");
11663 
11664     BB *longBrBB = CreateNewBB();
11665 
11666     BB *fallthruBB = bb->GetNext();
11667     LabelIdx fallthruLBL = fallthruBB->GetLabIdx();
11668     if (fallthruLBL == 0) {
11669         fallthruLBL = CreateLabel();
11670         SetLab2BBMap(static_cast<int32>(fallthruLBL), *fallthruBB);
11671         fallthruBB->AddLabel(fallthruLBL);
11672     }
11673 
11674     BB *targetBB;
11675     if (bb->GetSuccs().front() == fallthruBB) {
11676         targetBB = bb->GetSuccs().back();
11677     } else {
11678         targetBB = bb->GetSuccs().front();
11679     }
11680     LabelIdx targetLBL = targetBB->GetLabIdx();
11681     if (targetLBL == 0) {
11682         targetLBL = CreateLabel();
11683         SetLab2BBMap(static_cast<int32>(targetLBL), *targetBB);
11684         targetBB->AddLabel(targetLBL);
11685     }
11686 
11687     // Adjustment on br and CFG
11688     bb->RemoveSuccs(*targetBB);
11689     bb->PushBackSuccs(*longBrBB);
11690     bb->SetNext(longBrBB);
11691     // reverse cond br targeting fallthruBB
11692     uint32 targetIdx = AArch64isa::GetJumpTargetIdx(*insn);
11693     MOperator mOp = AArch64isa::FlipConditionOp(insn->GetMachineOpcode());
11694     insn->SetMOP(AArch64CG::kMd[mOp]);
11695     LabelOperand &fallthruBBLBLOpnd = GetOrCreateLabelOperand(fallthruLBL);
11696     insn->SetOperand(targetIdx, fallthruBBLBLOpnd);
11697 
11698     longBrBB->PushBackPreds(*bb);
11699     longBrBB->PushBackSuccs(*targetBB);
11700     LabelOperand &targetLBLOpnd = GetOrCreateLabelOperand(targetLBL);
11701     longBrBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xuncond, targetLBLOpnd));
11702     longBrBB->SetPrev(bb);
11703     longBrBB->SetNext(fallthruBB);
11704     longBrBB->SetKind(BB::kBBGoto);
11705 
11706     fallthruBB->SetPrev(longBrBB);
11707 
11708     targetBB->RemovePreds(*bb);
11709     targetBB->PushBackPreds(*longBrBB);
11710 }
11711 
AdjustOneElementVectorOperand(PrimType oType,RegOperand * opnd)11712 RegOperand *AArch64CGFunc::AdjustOneElementVectorOperand(PrimType oType, RegOperand *opnd)
11713 {
11714     RegOperand *resCvt = &CreateRegisterOperandOfType(oType);
11715     Insn *insnCvt = &GetInsnBuilder()->BuildInsn(MOP_xvmovrd, *resCvt, *opnd);
11716     GetCurBB()->AppendInsn(*insnCvt);
11717     return resCvt;
11718 }
11719 
SelectOneElementVectorCopy(Operand * src,PrimType sType)11720 RegOperand *AArch64CGFunc::SelectOneElementVectorCopy(Operand *src, PrimType sType)
11721 {
11722     RegOperand *res = &CreateRegisterOperandOfType(PTY_f64);
11723     SelectCopy(*res, PTY_f64, *src, sType);
11724     static_cast<RegOperand *>(res)->SetIF64Vec();
11725     return res;
11726 }
11727 
SelectVectorAbs(PrimType rType,Operand * o1)11728 RegOperand *AArch64CGFunc::SelectVectorAbs(PrimType rType, Operand *o1)
11729 {
11730     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
11731     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
11732     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
11733 
11734     MOperator mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vabsvv : MOP_vabsuu;
11735     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
11736     vInsn.AddOpndChain(*res).AddOpndChain(*o1);
11737     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1);
11738     GetCurBB()->AppendInsn(vInsn);
11739     return res;
11740 }
11741 
SelectVectorAddLong(PrimType rType,Operand * o1,Operand * o2,PrimType otyp,bool isLow)11742 RegOperand *AArch64CGFunc::SelectVectorAddLong(PrimType rType, Operand *o1, Operand *o2, PrimType otyp, bool isLow)
11743 {
11744     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result type */
11745     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
11746     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(otyp); /* vector operand 1 */
11747     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(otyp); /* vector operand 2 */
11748     MOperator mOp;
11749     if (isLow) {
11750         mOp = IsUnsignedInteger(rType) ? MOP_vuaddlvuu : MOP_vsaddlvuu;
11751     } else {
11752         mOp = IsUnsignedInteger(rType) ? MOP_vuaddl2vvv : MOP_vsaddl2vvv;
11753     }
11754     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
11755     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
11756     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
11757     GetCurBB()->AppendInsn(vInsn);
11758     return res;
11759 }
11760 
SelectVectorAddWiden(Operand * o1,PrimType otyp1,Operand * o2,PrimType otyp2,bool isLow)11761 RegOperand *AArch64CGFunc::SelectVectorAddWiden(Operand *o1, PrimType otyp1, Operand *o2, PrimType otyp2, bool isLow)
11762 {
11763     RegOperand *res = &CreateRegisterOperandOfType(otyp1); /* restype is same as o1 */
11764     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(otyp1);
11765     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(otyp1); /* vector operand 1 */
11766     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(otyp2); /* vector operand 2 */
11767 
11768     MOperator mOp;
11769     if (isLow) {
11770         mOp = IsUnsignedInteger(otyp1) ? MOP_vuaddwvvu : MOP_vsaddwvvu;
11771     } else {
11772         mOp = IsUnsignedInteger(otyp1) ? MOP_vuaddw2vvv : MOP_vsaddw2vvv;
11773     }
11774     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
11775     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
11776     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
11777     GetCurBB()->AppendInsn(vInsn);
11778     return res;
11779 }
11780 
SelectVectorImmMov(PrimType rType,Operand * src,PrimType sType)11781 RegOperand *AArch64CGFunc::SelectVectorImmMov(PrimType rType, Operand *src, PrimType sType)
11782 {
11783     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
11784     VectorRegSpec *vecSpec = GetMemoryPool()->New<VectorRegSpec>(rType);
11785     int64 val = static_cast<ImmOperand *>(src)->GetValue();
11786     /* copy the src imm operand to a reg if out of range */
11787     if ((GetVecEleSize(rType) >= k64BitSize) || (GetPrimTypeSize(sType) > k4ByteSize && val != 0) ||
11788         (val < kMinImmVal || val > kMaxImmVal)) {
11789         Operand *reg = &CreateRegisterOperandOfType(sType);
11790         SelectCopy(*reg, sType, *src, sType);
11791         return SelectVectorRegMov(rType, reg, sType);
11792     }
11793 
11794     MOperator mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vmovvi : MOP_vmovui;
11795     if (GetVecEleSize(rType) == k8BitSize && val < 0) {
11796         src = &CreateImmOperand(static_cast<uint8>(val), k8BitSize, true);
11797     } else if (val < 0) {
11798         src = &CreateImmOperand(-(val + 1), k8BitSize, true);
11799         mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vnotvi : MOP_vnotui;
11800     }
11801 
11802     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
11803     vInsn.AddOpndChain(*res).AddOpndChain(*src);
11804     vInsn.PushRegSpecEntry(vecSpec);
11805     GetCurBB()->AppendInsn(vInsn);
11806     return res;
11807 }
11808 
SelectVectorRegMov(PrimType rType,Operand * src,PrimType sType)11809 RegOperand *AArch64CGFunc::SelectVectorRegMov(PrimType rType, Operand *src, PrimType sType)
11810 {
11811     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
11812     VectorRegSpec *vecSpec = GetMemoryPool()->New<VectorRegSpec>(rType);
11813 
11814     MOperator mOp;
11815     if (GetPrimTypeSize(sType) > k4ByteSize) {
11816         mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vxdupvr : MOP_vxdupur;
11817     } else {
11818         mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vwdupvr : MOP_vwdupur;
11819     }
11820 
11821     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
11822     vInsn.AddOpndChain(*res).AddOpndChain(*src);
11823     vInsn.PushRegSpecEntry(vecSpec);
11824     GetCurBB()->AppendInsn(vInsn);
11825     return res;
11826 }
11827 
SelectVectorFromScalar(PrimType rType,Operand * src,PrimType sType)11828 RegOperand *AArch64CGFunc::SelectVectorFromScalar(PrimType rType, Operand *src, PrimType sType)
11829 {
11830     if (!IsPrimitiveVector(rType)) {
11831         return SelectOneElementVectorCopy(src, sType);
11832     } else if (src->IsConstImmediate()) {
11833         return SelectVectorImmMov(rType, src, sType);
11834     } else {
11835         return SelectVectorRegMov(rType, src, sType);
11836     }
11837 }
11838 
SelectVectorDup(PrimType rType,Operand * src,bool getLow)11839 RegOperand *AArch64CGFunc::SelectVectorDup(PrimType rType, Operand *src, bool getLow)
11840 {
11841     PrimType oType = rType;
11842     rType = FilterOneElementVectorType(oType);
11843     RegOperand *res = &CreateRegisterOperandOfType(rType);
11844     VectorRegSpec *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(k2ByteSize, k64BitSize, getLow ? 0 : 1);
11845 
11846     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(MOP_vduprv, AArch64CG::kMd[MOP_vduprv]);
11847     vInsn.AddOpndChain(*res).AddOpndChain(*src);
11848     vInsn.PushRegSpecEntry(vecSpecSrc);
11849     GetCurBB()->AppendInsn(vInsn);
11850     if (oType != rType) {
11851         res = AdjustOneElementVectorOperand(oType, res);
11852         static_cast<RegOperand *>(res)->SetIF64Vec();
11853     }
11854     return res;
11855 }
11856 
SelectVectorGetElement(PrimType rType,Operand * src,PrimType sType,int32 lane)11857 RegOperand *AArch64CGFunc::SelectVectorGetElement(PrimType rType, Operand *src, PrimType sType, int32 lane)
11858 {
11859     RegOperand *res = &CreateRegisterOperandOfType(rType);                        /* result operand */
11860     VectorRegSpec *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(sType, lane); /* vector operand */
11861 
11862     MOperator mop;
11863     if (!IsPrimitiveVector(sType)) {
11864         mop = MOP_xmovrr;
11865     } else if (GetPrimTypeBitSize(rType) >= k64BitSize) {
11866         mop = MOP_vxmovrv;
11867     } else {
11868         mop = (GetPrimTypeBitSize(sType) > k64BitSize) ? MOP_vwmovrv : MOP_vwmovru;
11869     }
11870 
11871     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
11872     vInsn.AddOpndChain(*res).AddOpndChain(*src);
11873     vInsn.PushRegSpecEntry(vecSpecSrc);
11874     GetCurBB()->AppendInsn(vInsn);
11875     return res;
11876 }
11877 
11878 /* adalp o1, o2 instruction accumulates into o1, overwriting the original operand.
11879    Hence we perform c = vadalp(a,b) as
11880        T tmp = a;
11881        return tmp+b;
11882    The return value of vadalp is then assigned to c, leaving value of a intact.
11883  */
SelectVectorPairwiseAdalp(Operand * src1,PrimType sty1,Operand * src2,PrimType sty2)11884 RegOperand *AArch64CGFunc::SelectVectorPairwiseAdalp(Operand *src1, PrimType sty1, Operand *src2, PrimType sty2)
11885 {
11886     VectorRegSpec *vecSpecDest;
11887     RegOperand *res;
11888 
11889     if (!IsPrimitiveVector(sty1)) {
11890         RegOperand *resF = SelectOneElementVectorCopy(src1, sty1);
11891         res = &CreateRegisterOperandOfType(PTY_f64);
11892         SelectCopy(*res, PTY_f64, *resF, PTY_f64);
11893         vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(k1ByteSize, k64BitSize);
11894     } else {
11895         res = &CreateRegisterOperandOfType(sty1); /* result type same as sty1 */
11896         SelectCopy(*res, sty1, *src1, sty1);
11897         vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(sty1);
11898     }
11899     VectorRegSpec *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(sty2);
11900 
11901     MOperator mop;
11902     if (IsUnsignedInteger(sty1)) {
11903         mop = GetPrimTypeSize(sty1) > k8ByteSize ? MOP_vupadalvv : MOP_vupadaluu;
11904     } else {
11905         mop = GetPrimTypeSize(sty1) > k8ByteSize ? MOP_vspadalvv : MOP_vspadaluu;
11906     }
11907 
11908     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
11909     vInsn.AddOpndChain(*res).AddOpndChain(*src2);
11910     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpecSrc);
11911     GetCurBB()->AppendInsn(vInsn);
11912     if (!IsPrimitiveVector(sty1)) {
11913         res = AdjustOneElementVectorOperand(sty1, res);
11914     }
11915     return res;
11916 }
11917 
SelectVectorPairwiseAdd(PrimType rType,Operand * src,PrimType sType)11918 RegOperand *AArch64CGFunc::SelectVectorPairwiseAdd(PrimType rType, Operand *src, PrimType sType)
11919 {
11920     PrimType oType = rType;
11921     rType = FilterOneElementVectorType(oType);
11922     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
11923     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
11924     VectorRegSpec *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(sType); /* source operand */
11925 
11926     if (rType == PTY_f64) {
11927         vecSpecDest->vecLaneMax = 1;
11928     }
11929 
11930     MOperator mop;
11931     if (IsUnsignedInteger(sType)) {
11932         mop = GetPrimTypeSize(sType) > k8ByteSize ? MOP_vupaddvv : MOP_vupadduu;
11933     } else {
11934         mop = GetPrimTypeSize(sType) > k8ByteSize ? MOP_vspaddvv : MOP_vspadduu;
11935     }
11936 
11937     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
11938     vInsn.AddOpndChain(*res).AddOpndChain(*src);
11939     /* dest pushed first, popped first */
11940     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpecSrc);
11941     GetCurBB()->AppendInsn(vInsn);
11942     if (oType != rType) {
11943         res = AdjustOneElementVectorOperand(oType, res);
11944     }
11945     return res;
11946 }
11947 
SelectVectorSetElement(Operand * eOpnd,PrimType eType,Operand * vOpnd,PrimType vType,int32 lane)11948 RegOperand *AArch64CGFunc::SelectVectorSetElement(Operand *eOpnd, PrimType eType, Operand *vOpnd, PrimType vType,
11949                                                   int32 lane)
11950 {
11951     if (!IsPrimitiveVector(vType)) {
11952         return SelectOneElementVectorCopy(eOpnd, eType);
11953     }
11954     RegOperand *reg = &CreateRegisterOperandOfType(eType); /* vector element type */
11955     SelectCopy(*reg, eType, *eOpnd, eType);
11956     VectorRegSpec *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(vType, lane); /* vector operand == result */
11957 
11958     MOperator mOp;
11959     if (GetPrimTypeSize(eType) > k4ByteSize) {
11960         mOp = GetPrimTypeSize(vType) > k8ByteSize ? MOP_vxinsvr : MOP_vxinsur;
11961     } else {
11962         mOp = GetPrimTypeSize(vType) > k8ByteSize ? MOP_vwinsvr : MOP_vwinsur;
11963     }
11964 
11965     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
11966     vInsn.AddOpndChain(*vOpnd).AddOpndChain(*reg);
11967     vInsn.PushRegSpecEntry(vecSpecSrc);
11968     GetCurBB()->AppendInsn(vInsn);
11969     return static_cast<RegOperand *>(vOpnd);
11970 }
11971 
SelectVectorAbsSubL(PrimType rType,Operand * o1,Operand * o2,PrimType oTy,bool isLow)11972 RegOperand *AArch64CGFunc::SelectVectorAbsSubL(PrimType rType, Operand *o1, Operand *o2, PrimType oTy, bool isLow)
11973 {
11974     RegOperand *res = &CreateRegisterOperandOfType(rType);
11975     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
11976     VectorRegSpec *vecSpecOpd1 = GetMemoryPool()->New<VectorRegSpec>(oTy);
11977     VectorRegSpec *vecSpecOpd2 = GetMemoryPool()->New<VectorRegSpec>(oTy); /* same opnd types */
11978 
11979     MOperator mop;
11980     if (isLow) {
11981         mop = IsPrimitiveUnSignedVector(rType) ? MOP_vuabdlvuu : MOP_vsabdlvuu;
11982     } else {
11983         mop = IsPrimitiveUnSignedVector(rType) ? MOP_vuabdl2vvv : MOP_vsabdl2vvv;
11984     }
11985     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
11986     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
11987     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpecOpd1).PushRegSpecEntry(vecSpecOpd2);
11988     GetCurBB()->AppendInsn(vInsn);
11989     return res;
11990 }
11991 
SelectVectorMerge(PrimType rType,Operand * o1,Operand * o2,int32 index)11992 RegOperand *AArch64CGFunc::SelectVectorMerge(PrimType rType, Operand *o1, Operand *o2, int32 index)
11993 {
11994     if (!IsPrimitiveVector(rType)) {
11995         static_cast<RegOperand *>(o1)->SetIF64Vec();
11996         return static_cast<RegOperand *>(o1); /* 64x1_t, index equals 0 */
11997     }
11998     RegOperand *res = &CreateRegisterOperandOfType(rType);
11999     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12000     VectorRegSpec *vecSpecOpd1 = GetMemoryPool()->New<VectorRegSpec>(rType);
12001     VectorRegSpec *vecSpecOpd2 = GetMemoryPool()->New<VectorRegSpec>(rType);
12002 
12003     ImmOperand *imm = &CreateImmOperand(index, k8BitSize, true);
12004 
12005     MOperator mOp = (GetPrimTypeSize(rType) > k8ByteSize) ? MOP_vextvvvi : MOP_vextuuui;
12006     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12007     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2).AddOpndChain(*imm);
12008     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpecOpd1).PushRegSpecEntry(vecSpecOpd2);
12009     GetCurBB()->AppendInsn(vInsn);
12010     return res;
12011 }
12012 
SelectVectorReverse(PrimType rType,Operand * src,PrimType sType,uint32 size)12013 RegOperand *AArch64CGFunc::SelectVectorReverse(PrimType rType, Operand *src, PrimType sType, uint32 size)
12014 {
12015     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12016     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12017     VectorRegSpec *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(sType); /* vector operand */
12018 
12019     MOperator mOp;
12020     if (GetPrimTypeBitSize(rType) == k128BitSize) {
12021         mOp = size >= k64BitSize ? MOP_vrev64qq : (size >= k32BitSize ? MOP_vrev32qq : MOP_vrev16qq);
12022     } else if (GetPrimTypeBitSize(rType) == k64BitSize) {
12023         mOp = size >= k64BitSize ? MOP_vrev64dd : (size >= k32BitSize ? MOP_vrev32dd : MOP_vrev16dd);
12024     } else {
12025         CHECK_FATAL(false, "should not be here");
12026     }
12027     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12028     vInsn.AddOpndChain(*res).AddOpndChain(*src);
12029     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpecSrc);
12030     GetCurBB()->AppendInsn(vInsn);
12031     return res;
12032 }
12033 
SelectVectorSum(PrimType rType,Operand * o1,PrimType oType)12034 RegOperand *AArch64CGFunc::SelectVectorSum(PrimType rType, Operand *o1, PrimType oType)
12035 {
12036     RegOperand *res = &CreateRegisterOperandOfType(rType); /* uint32_t result */
12037     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oType);
12038     RegOperand *iOpnd = &CreateRegisterOperandOfType(oType); /* float intermediate result */
12039     uint32 eSize = GetVecEleSize(oType);                     /* vector opd in bits */
12040     bool is16ByteVec = GetPrimTypeSize(oType) >= k16ByteSize;
12041     MOperator mOp;
12042     if (is16ByteVec) {
12043         mOp = eSize <= k8BitSize
12044                   ? MOP_vbaddvrv
12045                   : (eSize <= k16BitSize ? MOP_vhaddvrv : (eSize <= k32BitSize ? MOP_vsaddvrv : MOP_vdaddvrv));
12046     } else {
12047         mOp = eSize <= k8BitSize ? MOP_vbaddvru : (eSize <= k16BitSize ? MOP_vhaddvru : MOP_vsaddvru);
12048     }
12049     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12050     vInsn.AddOpndChain(*iOpnd).AddOpndChain(*o1);
12051     vInsn.PushRegSpecEntry(vecSpec1);
12052     GetCurBB()->AppendInsn(vInsn);
12053 
12054     mOp = eSize > k32BitSize ? MOP_vxmovrv : MOP_vwmovrv;
12055     VectorInsn &vInsn2 = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12056     auto *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(oType);
12057     vInsn2.AddOpndChain(*res).AddOpndChain(*iOpnd);
12058     vecSpec2->vecLane = 0;
12059     vInsn2.PushRegSpecEntry(vecSpec2);
12060     GetCurBB()->AppendInsn(vInsn2);
12061     return res;
12062 }
12063 
PrepareVectorOperands(Operand ** o1,PrimType & oty1,Operand ** o2,PrimType & oty2)12064 void AArch64CGFunc::PrepareVectorOperands(Operand **o1, PrimType &oty1, Operand **o2, PrimType &oty2)
12065 {
12066     /* Only 1 operand can be non vector, otherwise it's a scalar operation, wouldn't come here */
12067     if (IsPrimitiveVector(oty1) == IsPrimitiveVector(oty2)) {
12068         return;
12069     }
12070     PrimType origTyp = !IsPrimitiveVector(oty2) ? oty2 : oty1;
12071     Operand *opd = !IsPrimitiveVector(oty2) ? *o2 : *o1;
12072     PrimType rType = !IsPrimitiveVector(oty2) ? oty1 : oty2; /* Type to dup into */
12073     RegOperand *res = &CreateRegisterOperandOfType(rType);
12074     VectorRegSpec *vecSpec = GetMemoryPool()->New<VectorRegSpec>(rType);
12075 
12076     bool immOpnd = false;
12077     if (opd->IsConstImmediate()) {
12078         int64 val = static_cast<ImmOperand *>(opd)->GetValue();
12079         if (val >= kMinImmVal && val <= kMaxImmVal && GetVecEleSize(rType) < k64BitSize) {
12080             immOpnd = true;
12081         } else {
12082             RegOperand *regOpd = &CreateRegisterOperandOfType(origTyp);
12083             SelectCopyImm(*regOpd, origTyp, static_cast<ImmOperand &>(*opd), origTyp);
12084             opd = static_cast<Operand *>(regOpd);
12085         }
12086     }
12087 
12088     /* need dup to vector operand */
12089     MOperator mOp;
12090     if (immOpnd) {
12091         mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vmovvi : MOP_vmovui; /* a const */
12092     } else {
12093         if (GetPrimTypeSize(origTyp) > k4ByteSize) {
12094             mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vxdupvr : MOP_vxdupur;
12095         } else {
12096             mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vwdupvr : MOP_vwdupur; /* a scalar var */
12097         }
12098     }
12099     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12100     vInsn.AddOpndChain(*res).AddOpndChain(*opd);
12101     vInsn.PushRegSpecEntry(vecSpec);
12102     GetCurBB()->AppendInsn(vInsn);
12103     if (!IsPrimitiveVector(oty2)) {
12104         *o2 = static_cast<Operand *>(res);
12105         oty2 = rType;
12106     } else {
12107         *o1 = static_cast<Operand *>(res);
12108         oty1 = rType;
12109     }
12110 }
12111 
SelectVectorCvt(Operand * res,PrimType rType,Operand * o1,PrimType oType)12112 void AArch64CGFunc::SelectVectorCvt(Operand *res, PrimType rType, Operand *o1, PrimType oType)
12113 {
12114     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12115     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oType); /* vector operand 1 */
12116 
12117     MOperator mOp;
12118     VectorInsn *insn;
12119     if (GetPrimTypeSize(rType) > GetPrimTypeSize(oType)) {
12120         /* expand, similar to vmov_XX() intrinsics */
12121         mOp = IsUnsignedInteger(rType) ? MOP_vushllvvi : MOP_vshllvvi;
12122         ImmOperand *imm = &CreateImmOperand(0, k8BitSize, true);
12123         insn = &GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12124         insn->AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*imm);
12125     } else if (GetPrimTypeSize(rType) < GetPrimTypeSize(oType)) {
12126         /* extract, similar to vqmovn_XX() intrinsics */
12127         insn = &GetInsnBuilder()->BuildVectorInsn(MOP_vxtnuv, AArch64CG::kMd[MOP_vxtnuv]);
12128         insn->AddOpndChain(*res).AddOpndChain(*o1);
12129     } else {
12130         CHECK_FATAL(0, "Invalid cvt between 2 operands of the same size");
12131     }
12132     insn->PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1);
12133     GetCurBB()->AppendInsn(*insn);
12134 }
12135 
SelectVectorCompareZero(Operand * o1,PrimType oty1,Operand * o2,Opcode opc)12136 RegOperand *AArch64CGFunc::SelectVectorCompareZero(Operand *o1, PrimType oty1, Operand *o2, Opcode opc)
12137 {
12138     if (IsUnsignedInteger(oty1) && (opc != OP_eq && opc != OP_ne)) {
12139         return nullptr; /* no unsigned instr for zero */
12140     }
12141     RegOperand *res = &CreateRegisterOperandOfType(oty1); /* result operand */
12142     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(oty1);
12143     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oty1); /* vector operand 1 */
12144 
12145     MOperator mOp;
12146     switch (opc) {
12147         case OP_eq:
12148         case OP_ne:
12149             mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vzcmeqvv : MOP_vzcmequu;
12150             break;
12151         case OP_gt:
12152             mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vzcmgtvv : MOP_vzcmgtuu;
12153             break;
12154         case OP_ge:
12155             mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vzcmgevv : MOP_vzcmgeuu;
12156             break;
12157         case OP_lt:
12158             mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vzcmltvv : MOP_vzcmltuu;
12159             break;
12160         case OP_le:
12161             mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vzcmlevv : MOP_vzcmleuu;
12162             break;
12163         default:
12164             CHECK_FATAL(0, "Invalid cc in vector compare");
12165     }
12166     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12167     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12168     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1);
12169     GetCurBB()->AppendInsn(vInsn);
12170     if (opc == OP_ne) {
12171         res = SelectVectorNot(oty1, res);
12172     }
12173     return res;
12174 }
12175 
12176 /* Neon compare intrinsics always return unsigned vector, MapleIR for comparison always return
12177    signed.  Using type of 1st operand for operation here */
SelectVectorCompare(Operand * o1,PrimType oty1,Operand * o2,PrimType oty2,Opcode opc)12178 RegOperand *AArch64CGFunc::SelectVectorCompare(Operand *o1, PrimType oty1, Operand *o2, PrimType oty2, Opcode opc)
12179 {
12180     if (o2->IsConstImmediate() && static_cast<ImmOperand *>(o2)->GetValue() == 0) {
12181         RegOperand *zeroCmp = SelectVectorCompareZero(o1, oty1, o2, opc);
12182         if (zeroCmp != nullptr) {
12183             return zeroCmp;
12184         }
12185     }
12186     PrepareVectorOperands(&o1, oty1, &o2, oty2);
12187     DEBUG_ASSERT(oty1 == oty2, "vector operand type mismatch");
12188 
12189     RegOperand *res = &CreateRegisterOperandOfType(oty1); /* result operand */
12190     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(oty1);
12191     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oty1); /* vector operand 1 */
12192     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(oty2); /* vector operand 2 */
12193 
12194     MOperator mOp;
12195     switch (opc) {
12196         case OP_eq:
12197         case OP_ne:
12198             mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vcmeqvvv : MOP_vcmequuu;
12199             break;
12200         case OP_lt:
12201         case OP_gt:
12202             if (IsUnsignedInteger(oty1)) {
12203                 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vcmhivvv : MOP_vcmhiuuu;
12204             } else {
12205                 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vcmgtvvv : MOP_vcmgtuuu;
12206             }
12207             break;
12208         case OP_le:
12209         case OP_ge:
12210             if (IsUnsignedInteger(oty1)) {
12211                 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vcmhsvvv : MOP_vcmhsuuu;
12212             } else {
12213                 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vcmgevvv : MOP_vcmgeuuu;
12214             }
12215             break;
12216         default:
12217             CHECK_FATAL(0, "Invalid cc in vector compare");
12218     }
12219     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12220     if (opc == OP_lt || opc == OP_le) {
12221         vInsn.AddOpndChain(*res).AddOpndChain(*o2).AddOpndChain(*o1);
12222     } else {
12223         vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12224     }
12225     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
12226     GetCurBB()->AppendInsn(vInsn);
12227     if (opc == OP_ne) {
12228         res = SelectVectorNot(oty1, res);
12229     }
12230     return res;
12231 }
12232 
SelectVectorShift(PrimType rType,Operand * o1,PrimType oty1,Operand * o2,PrimType oty2,Opcode opc)12233 RegOperand *AArch64CGFunc::SelectVectorShift(PrimType rType, Operand *o1, PrimType oty1, Operand *o2, PrimType oty2,
12234                                              Opcode opc)
12235 {
12236     PrepareVectorOperands(&o1, oty1, &o2, oty2);
12237     PrimType resultType = rType;
12238     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12239     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12240     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 2 */
12241 
12242     if (!IsPrimitiveVector(rType)) {
12243         o1 = &SelectCopy(*o1, rType, PTY_f64);
12244         o2 = &SelectCopy(*o2, rType, PTY_f64);
12245         resultType = PTY_f64;
12246     }
12247     RegOperand *res = &CreateRegisterOperandOfType(resultType); /* result operand */
12248 
12249     /* signed and unsigned shl(v,v) both use sshl or ushl, they are the same */
12250     MOperator mOp;
12251     if (IsPrimitiveUnsigned(rType)) {
12252         mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vushlvvv : MOP_vushluuu;
12253     } else {
12254         mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vshlvvv : MOP_vshluuu;
12255     }
12256 
12257     if (opc != OP_shl) {
12258         o2 = SelectVectorNeg(rType, o2);
12259     }
12260     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12261     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12262     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
12263     GetCurBB()->AppendInsn(vInsn);
12264     return res;
12265 }
12266 
ValidShiftConst(PrimType rType)12267 uint32 ValidShiftConst(PrimType rType)
12268 {
12269     switch (rType) {
12270         case PTY_v8u8:
12271         case PTY_v8i8:
12272         case PTY_v16u8:
12273         case PTY_v16i8:
12274             return k8BitSize;
12275         case PTY_v4u16:
12276         case PTY_v4i16:
12277         case PTY_v8u16:
12278         case PTY_v8i16:
12279             return k16BitSize;
12280         case PTY_v2u32:
12281         case PTY_v2i32:
12282         case PTY_v4u32:
12283         case PTY_v4i32:
12284             return k32BitSize;
12285         case PTY_v2u64:
12286         case PTY_v2i64:
12287             return k64BitSize;
12288         default:
12289             CHECK_FATAL(0, "Invalid Shift operand type");
12290     }
12291     return 0;
12292 }
12293 
SelectVectorShiftImm(PrimType rType,Operand * o1,Operand * imm,int32 sVal,Opcode opc)12294 RegOperand *AArch64CGFunc::SelectVectorShiftImm(PrimType rType, Operand *o1, Operand *imm, int32 sVal, Opcode opc)
12295 {
12296     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12297     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12298     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12299 
12300     if (!imm->IsConstImmediate()) {
12301         CHECK_FATAL(0, "VectorUShiftImm has invalid shift const");
12302     }
12303     uint32 shift = static_cast<uint32>(ValidShiftConst(rType));
12304     bool needDup = false;
12305     if (opc == OP_shl) {
12306         if ((shift == k8BitSize && (sVal < 0 || static_cast<uint32>(sVal) >= shift)) ||
12307             (shift == k16BitSize && (sVal < 0 || static_cast<uint32>(sVal) >= shift)) ||
12308             (shift == k32BitSize && (sVal < 0 || static_cast<uint32>(sVal) >= shift)) ||
12309             (shift == k64BitSize && (sVal < 0 || static_cast<uint32>(sVal) >= shift))) {
12310             needDup = true;
12311         }
12312     } else {
12313         if ((shift == k8BitSize && (sVal < 1 || static_cast<uint32>(sVal) > shift)) ||
12314             (shift == k16BitSize && (sVal < 1 || static_cast<uint32>(sVal) > shift)) ||
12315             (shift == k32BitSize && (sVal < 1 || static_cast<uint32>(sVal) > shift)) ||
12316             (shift == k64BitSize && (sVal < 1 || static_cast<uint32>(sVal) > shift))) {
12317             needDup = true;
12318         }
12319     }
12320     if (needDup) {
12321         /* Dup constant to vector reg */
12322         MOperator mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vmovvi : MOP_vmovui;
12323         VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12324         vInsn.AddOpndChain(*res).AddOpndChain(*imm);
12325         vInsn.PushRegSpecEntry(vecSpecDest);
12326         GetCurBB()->AppendInsn(vInsn);
12327         res = SelectVectorShift(rType, o1, rType, res, rType, opc);
12328         return res;
12329     }
12330     MOperator mOp;
12331     if (GetPrimTypeSize(rType) > k8ByteSize) {
12332         if (IsUnsignedInteger(rType)) {
12333             mOp = opc == OP_shl ? MOP_vushlvvi : MOP_vushrvvi;
12334         } else {
12335             mOp = opc == OP_shl ? MOP_vushlvvi : MOP_vshrvvi;
12336         }
12337     } else {
12338         if (IsUnsignedInteger(rType)) {
12339             mOp = opc == OP_shl ? MOP_vushluui : MOP_vushruui;
12340         } else {
12341             mOp = opc == OP_shl ? MOP_vushluui : MOP_vshruui;
12342         }
12343     }
12344     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12345     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*imm);
12346     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1);
12347     GetCurBB()->AppendInsn(vInsn);
12348     return res;
12349 }
12350 
SelectVectorTableLookup(PrimType rType,Operand * o1,Operand * o2)12351 RegOperand *AArch64CGFunc::SelectVectorTableLookup(PrimType rType, Operand *o1, Operand *o2)
12352 {
12353     RegOperand *res = &CreateRegisterOperandOfType(rType);                   /* result operand */
12354     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType); /* 8B or 16B */
12355     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType);    /* vector operand 1 */
12356     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(rType);    /* vector operand 2 */
12357     vecSpec1->compositeOpnds = 1;                                            /* composite operand */
12358 
12359     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(MOP_vtbl1vvv, AArch64CG::kMd[MOP_vtbl1vvv]);
12360     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12361     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
12362     GetCurBB()->AppendInsn(vInsn);
12363     return res;
12364 }
12365 
SelectVectorMadd(Operand * o1,PrimType oTyp1,Operand * o2,PrimType oTyp2,Operand * o3,PrimType oTyp3)12366 RegOperand *AArch64CGFunc::SelectVectorMadd(Operand *o1, PrimType oTyp1, Operand *o2, PrimType oTyp2, Operand *o3,
12367                                             PrimType oTyp3)
12368 {
12369     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oTyp1); /* operand 1 and result */
12370     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(oTyp2); /* vector operand 2 */
12371     VectorRegSpec *vecSpec3 = GetMemoryPool()->New<VectorRegSpec>(oTyp3); /* vector operand 2 */
12372 
12373     MOperator mop = IsPrimitiveUnSignedVector(oTyp1) ? MOP_vumaddvvv : MOP_vsmaddvvv;
12374     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
12375     vInsn.AddOpndChain(*o1).AddOpndChain(*o2).AddOpndChain(*o3);
12376     vInsn.PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2).PushRegSpecEntry(vecSpec3);
12377     GetCurBB()->AppendInsn(vInsn);
12378     return static_cast<RegOperand *>(o1);
12379 }
12380 
SelectVectorMull(PrimType rType,Operand * o1,PrimType oTyp1,Operand * o2,PrimType oTyp2,bool isLow)12381 RegOperand *AArch64CGFunc::SelectVectorMull(PrimType rType, Operand *o1, PrimType oTyp1, Operand *o2, PrimType oTyp2,
12382                                             bool isLow)
12383 {
12384     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12385     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12386     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oTyp1); /* vector operand 1 */
12387     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(oTyp2); /* vector operand 1 */
12388 
12389     MOperator mop;
12390     if (isLow) {
12391         mop = IsPrimitiveUnSignedVector(rType) ? MOP_vumullvvv : MOP_vsmullvvv;
12392     } else {
12393         mop = IsPrimitiveUnSignedVector(rType) ? MOP_vumull2vvv : MOP_vsmull2vvv;
12394     }
12395     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
12396     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12397     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
12398     GetCurBB()->AppendInsn(vInsn);
12399     return res;
12400 }
12401 
SelectVectorBinOp(PrimType rType,Operand * o1,PrimType oty1,Operand * o2,PrimType oty2,Opcode opc)12402 RegOperand *AArch64CGFunc::SelectVectorBinOp(PrimType rType, Operand *o1, PrimType oty1, Operand *o2, PrimType oty2,
12403                                              Opcode opc)
12404 {
12405     PrepareVectorOperands(&o1, oty1, &o2, oty2);
12406     DEBUG_ASSERT(oty1 == oty2, "vector operand type mismatch");
12407 
12408     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12409     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12410     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oty1); /* source operand 1 */
12411     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(oty2); /* source operand 2 */
12412 
12413     MOperator mOp;
12414     if (opc == OP_add) {
12415         mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vaddvvv : MOP_vadduuu;
12416     } else if (opc == OP_sub) {
12417         mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vsubvvv : MOP_vsubuuu;
12418     } else if (opc == OP_mul) {
12419         mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vmulvvv : MOP_vmuluuu;
12420     } else {
12421         CHECK_FATAL(0, "Invalid opcode for SelectVectorBinOp");
12422     }
12423     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12424     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12425     /* dest pushed first, popped first */
12426     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
12427     GetCurBB()->AppendInsn(vInsn);
12428     return res;
12429 }
12430 
SelectVectorBitwiseOp(PrimType rType,Operand * o1,PrimType oty1,Operand * o2,PrimType oty2,Opcode opc)12431 RegOperand *AArch64CGFunc::SelectVectorBitwiseOp(PrimType rType, Operand *o1, PrimType oty1, Operand *o2, PrimType oty2,
12432                                                  Opcode opc)
12433 {
12434     PrepareVectorOperands(&o1, oty1, &o2, oty2);
12435     DEBUG_ASSERT(oty1 == oty2, "vector operand type mismatch");
12436 
12437     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12438     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12439     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12440     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12441 
12442     MOperator mOp;
12443     if (opc == OP_band) {
12444         mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vandvvv : MOP_vanduuu;
12445     } else if (opc == OP_bior) {
12446         mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vorvvv : MOP_voruuu;
12447     } else if (opc == OP_bxor) {
12448         mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vxorvvv : MOP_vxoruuu;
12449     } else {
12450         CHECK_FATAL(0, "Invalid opcode for SelectVectorBitwiseOp");
12451     }
12452     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12453     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12454     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
12455     GetCurBB()->AppendInsn(vInsn);
12456     return res;
12457 }
12458 
SelectVectorNarrow(PrimType rType,Operand * o1,PrimType otyp)12459 RegOperand *AArch64CGFunc::SelectVectorNarrow(PrimType rType, Operand *o1, PrimType otyp)
12460 {
12461     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12462     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12463     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(otyp); /* vector operand */
12464 
12465     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(MOP_vxtnuv, AArch64CG::kMd[MOP_vxtnuv]);
12466     vInsn.AddOpndChain(*res).AddOpndChain(*o1);
12467     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1);
12468     GetCurBB()->AppendInsn(vInsn);
12469     return res;
12470 }
12471 
SelectVectorNarrow2(PrimType rType,Operand * o1,PrimType oty1,Operand * o2,PrimType oty2)12472 RegOperand *AArch64CGFunc::SelectVectorNarrow2(PrimType rType, Operand *o1, PrimType oty1, Operand *o2, PrimType oty2)
12473 {
12474     (void)oty1;                                      /* 1st opnd was loaded already, type no longer needed */
12475     RegOperand *res = static_cast<RegOperand *>(o1); /* o1 is also the result */
12476     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12477     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(oty2); /* vector opnd2 */
12478 
12479     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(MOP_vxtn2uv, AArch64CG::kMd[MOP_vxtn2uv]);
12480     vInsn.AddOpndChain(*res).AddOpndChain(*o2);
12481     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec2);
12482     GetCurBB()->AppendInsn(vInsn);
12483     return res;
12484 }
12485 
SelectVectorNot(PrimType rType,Operand * o1)12486 RegOperand *AArch64CGFunc::SelectVectorNot(PrimType rType, Operand *o1)
12487 {
12488     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12489     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12490     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12491 
12492     MOperator mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vnotvv : MOP_vnotuu;
12493     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12494     vInsn.AddOpndChain(*res).AddOpndChain(*o1);
12495     vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1);
12496     GetCurBB()->AppendInsn(vInsn);
12497     return res;
12498 }
12499 
SelectVectorNeg(PrimType rType,Operand * o1)12500 RegOperand *AArch64CGFunc::SelectVectorNeg(PrimType rType, Operand *o1)
12501 {
12502     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12503     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12504     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12505 
12506     MOperator mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vnegvv : MOP_vneguu;
12507     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12508     vInsn.AddOpndChain(*res).AddOpndChain(*o1);
12509     vInsn.PushRegSpecEntry(vecSpecDest);
12510     vInsn.PushRegSpecEntry(vecSpec1);
12511     GetCurBB()->AppendInsn(vInsn);
12512     return res;
12513 }
12514 
12515 /*
12516  * Called internally for auto-vec, no intrinsics for now
12517  */
SelectVectorSelect(Operand & cond,PrimType rType,Operand & o0,Operand & o1)12518 RegOperand *AArch64CGFunc::SelectVectorSelect(Operand &cond, PrimType rType, Operand &o0, Operand &o1)
12519 {
12520     rType = GetPrimTypeSize(rType) > k8ByteSize ? PTY_v16u8 : PTY_v8u8;
12521     RegOperand *res = &CreateRegisterOperandOfType(rType);
12522     SelectCopy(*res, rType, cond, rType);
12523     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12524     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType);
12525     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(rType);
12526 
12527     uint32 mOp = GetPrimTypeBitSize(rType) > k64BitSize ? MOP_vbslvvv : MOP_vbsluuu;
12528     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12529     vInsn.AddOpndChain(*res).AddOpndChain(o0).AddOpndChain(o1);
12530     vInsn.PushRegSpecEntry(vecSpecDest);
12531     vInsn.PushRegSpecEntry(vecSpec1);
12532     vInsn.PushRegSpecEntry(vecSpec2);
12533     GetCurBB()->AppendInsn(vInsn);
12534     return res;
12535 }
12536 
SelectVectorShiftRNarrow(PrimType rType,Operand * o1,PrimType oType,Operand * o2,bool isLow)12537 RegOperand *AArch64CGFunc::SelectVectorShiftRNarrow(PrimType rType, Operand *o1, PrimType oType, Operand *o2,
12538                                                     bool isLow)
12539 {
12540     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12541     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12542     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oType); /* vector operand 1 */
12543 
12544     ImmOperand *imm = static_cast<ImmOperand *>(o2);
12545     MOperator mOp;
12546     if (isLow) {
12547         mOp = MOP_vshrnuvi;
12548     } else {
12549         CHECK_FATAL(0, "NYI: vshrn_high_");
12550     }
12551     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12552     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*imm);
12553     vInsn.PushRegSpecEntry(vecSpecDest);
12554     vInsn.PushRegSpecEntry(vecSpec1);
12555     GetCurBB()->AppendInsn(vInsn);
12556     return res;
12557 }
12558 
SelectVectorSubWiden(PrimType resType,Operand * o1,PrimType otyp1,Operand * o2,PrimType otyp2,bool isLow,bool isWide)12559 RegOperand *AArch64CGFunc::SelectVectorSubWiden(PrimType resType, Operand *o1, PrimType otyp1, Operand *o2,
12560                                                 PrimType otyp2, bool isLow, bool isWide)
12561 {
12562     RegOperand *res = &CreateRegisterOperandOfType(resType); /* result reg */
12563     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(resType);
12564     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(otyp1); /* vector operand 1 */
12565     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(otyp2); /* vector operand 2 */
12566 
12567     MOperator mOp;
12568     if (!isWide) {
12569         if (isLow) {
12570             mOp = IsUnsignedInteger(otyp1) ? MOP_vusublvuu : MOP_vssublvuu;
12571         } else {
12572             mOp = IsUnsignedInteger(otyp1) ? MOP_vusubl2vvv : MOP_vssubl2vvv;
12573         }
12574     } else {
12575         if (isLow) {
12576             mOp = IsUnsignedInteger(otyp1) ? MOP_vusubwvvu : MOP_vssubwvvu;
12577         } else {
12578             mOp = IsUnsignedInteger(otyp1) ? MOP_vusubw2vvv : MOP_vssubw2vvv;
12579         }
12580     }
12581     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12582     vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12583     vInsn.PushRegSpecEntry(vecSpecDest);
12584     vInsn.PushRegSpecEntry(vecSpec1);
12585     vInsn.PushRegSpecEntry(vecSpec2);
12586     GetCurBB()->AppendInsn(vInsn);
12587     return res;
12588 }
12589 
SelectVectorZip(PrimType rType,Operand * o1,Operand * o2)12590 void AArch64CGFunc::SelectVectorZip(PrimType rType, Operand *o1, Operand *o2)
12591 {
12592     RegOperand *res1 = &CreateRegisterOperandOfType(rType); /* result operand 1 */
12593     RegOperand *res2 = &CreateRegisterOperandOfType(rType); /* result operand 2 */
12594     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12595     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12596     VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 2 */
12597 
12598     VectorInsn &vInsn1 = GetInsnBuilder()->BuildVectorInsn(MOP_vzip1vvv, AArch64CG::kMd[MOP_vzip1vvv]);
12599     vInsn1.AddOpndChain(*res1).AddOpndChain(*o1).AddOpndChain(*o2);
12600     vInsn1.PushRegSpecEntry(vecSpecDest);
12601     vInsn1.PushRegSpecEntry(vecSpec1);
12602     vInsn1.PushRegSpecEntry(vecSpec2);
12603     GetCurBB()->AppendInsn(vInsn1);
12604 
12605     VectorInsn &vInsn2 = GetInsnBuilder()->BuildVectorInsn(MOP_vzip2vvv, AArch64CG::kMd[MOP_vzip2vvv]);
12606     vInsn2.AddOpndChain(*res2).AddOpndChain(*o1).AddOpndChain(*o2);
12607     vInsn2.PushRegSpecEntry(vecSpecDest);
12608     vInsn2.PushRegSpecEntry(vecSpec1);
12609     vInsn2.PushRegSpecEntry(vecSpec2);
12610     GetCurBB()->AppendInsn(vInsn2);
12611 
12612     if (GetPrimTypeSize(rType) <= k16ByteSize) {
12613         Operand *preg1 = &GetOrCreatePhysicalRegisterOperand(V0, k64BitSize, kRegTyFloat);
12614         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xvmovd, *preg1, *res1));
12615         Operand *preg2 = &GetOrCreatePhysicalRegisterOperand(V1, k64BitSize, kRegTyFloat);
12616         GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xvmovd, *preg2, *res2));
12617     }
12618 }
12619 
SelectVectorWiden(PrimType rType,Operand * o1,PrimType otyp,bool isLow)12620 RegOperand *AArch64CGFunc::SelectVectorWiden(PrimType rType, Operand *o1, PrimType otyp, bool isLow)
12621 {
12622     RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12623     VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12624     VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(otyp); /* vector operand */
12625 
12626     MOperator mOp;
12627     if (isLow) {
12628         mOp = IsPrimitiveUnSignedVector(rType) ? MOP_vuxtlvu : MOP_vsxtlvu;
12629     } else {
12630         mOp = IsPrimitiveUnSignedVector(rType) ? MOP_vuxtl2vv : MOP_vsxtl2vv;
12631     }
12632     VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12633     vInsn.AddOpndChain(*res).AddOpndChain(*o1);
12634     vInsn.PushRegSpecEntry(vecSpecDest);
12635     vInsn.PushRegSpecEntry(vecSpec1);
12636     GetCurBB()->AppendInsn(vInsn);
12637     return res;
12638 }
12639 
12640 /*
12641  * Check the distance between the first insn of BB with the lable(targ_labidx)
12642  * and the insn with targ_id. If the distance greater than kShortBRDistance
12643  * return false.
12644  */
DistanceCheck(const BB & bb,LabelIdx targLabIdx,uint32 targId) const12645 bool AArch64CGFunc::DistanceCheck(const BB &bb, LabelIdx targLabIdx, uint32 targId) const
12646 {
12647     for (auto *tBB : bb.GetSuccs()) {
12648         if (tBB->GetLabIdx() != targLabIdx) {
12649             continue;
12650         }
12651         Insn *tInsn = tBB->GetFirstInsn();
12652         while (tInsn == nullptr || !tInsn->IsMachineInstruction()) {
12653             if (tInsn == nullptr) {
12654                 tBB = tBB->GetNext();
12655                 if (tBB == nullptr) { /* tailcallopt may make the target block empty */
12656                     return true;
12657                 }
12658                 tInsn = tBB->GetFirstInsn();
12659             } else {
12660                 tInsn = tInsn->GetNext();
12661             }
12662         }
12663         uint32 tmp = (tInsn->GetId() > targId) ? (tInsn->GetId() - targId) : (targId - tInsn->GetId());
12664         return (tmp < kShortBRDistance);
12665     }
12666     CHECK_FATAL(false, "CFG error");
12667 }
12668 } /* namespace maplebe */
12669