1 /*
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "aarch64_cg.h"
17 #include "aarch64_cgfunc.h"
18 #include <vector>
19 #include <cstdint>
20 #include <sys/stat.h>
21 #include <atomic>
22 #include "cfi.h"
23 #include "mpl_logging.h"
24 #include "rt.h"
25 #include "opcode_info.h"
26 #include "mir_builder.h"
27 #include "mir_symbol_builder.h"
28 #include "mpl_atomic.h"
29 #include "metadata_layout.h"
30 #include "emit.h"
31 #include "simplify.h"
32 #include <algorithm>
33
34 namespace maplebe {
35 using namespace maple;
36 CondOperand AArch64CGFunc::ccOperands[kCcLast] = {
37 CondOperand(CC_EQ), CondOperand(CC_NE), CondOperand(CC_CS), CondOperand(CC_HS), CondOperand(CC_CC),
38 CondOperand(CC_LO), CondOperand(CC_MI), CondOperand(CC_PL), CondOperand(CC_VS), CondOperand(CC_VC),
39 CondOperand(CC_HI), CondOperand(CC_LS), CondOperand(CC_GE), CondOperand(CC_LT), CondOperand(CC_GT),
40 CondOperand(CC_LE), CondOperand(CC_AL),
41 };
42
43 namespace {
44 constexpr int32 kSignedDimension = 2; /* signed and unsigned */
45 constexpr int32 kIntByteSizeDimension = 4; /* 1 byte, 2 byte, 4 bytes, 8 bytes */
46 constexpr int32 kFloatByteSizeDimension = 3; /* 4 bytes, 8 bytes, 16 bytes(vector) */
47 constexpr int32 kShiftAmount12 = 12; /* for instruction that can use shift, shift amount must be 0 or 12 */
48
49 MOperator ldIs[kSignedDimension][kIntByteSizeDimension] = {
50 /* unsigned == 0 */
51 {MOP_wldrb, MOP_wldrh, MOP_wldr, MOP_xldr},
52 /* signed == 1 */
53 {MOP_wldrsb, MOP_wldrsh, MOP_wldr, MOP_xldr}};
54
55 MOperator stIs[kSignedDimension][kIntByteSizeDimension] = {
56 /* unsigned == 0 */
57 {MOP_wstrb, MOP_wstrh, MOP_wstr, MOP_xstr},
58 /* signed == 1 */
59 {MOP_wstrb, MOP_wstrh, MOP_wstr, MOP_xstr}};
60
61 MOperator ldIsAcq[kSignedDimension][kIntByteSizeDimension] = {
62 /* unsigned == 0 */
63 {MOP_wldarb, MOP_wldarh, MOP_wldar, MOP_xldar},
64 /* signed == 1 */
65 {MOP_undef, MOP_undef, MOP_wldar, MOP_xldar}};
66
67 MOperator stIsRel[kSignedDimension][kIntByteSizeDimension] = {
68 /* unsigned == 0 */
69 {MOP_wstlrb, MOP_wstlrh, MOP_wstlr, MOP_xstlr},
70 /* signed == 1 */
71 {MOP_wstlrb, MOP_wstlrh, MOP_wstlr, MOP_xstlr}};
72
73 MOperator ldFs[kFloatByteSizeDimension] = {MOP_sldr, MOP_dldr, MOP_qldr};
74 MOperator stFs[kFloatByteSizeDimension] = {MOP_sstr, MOP_dstr, MOP_qstr};
75
76 MOperator ldFsAcq[kFloatByteSizeDimension] = {MOP_undef, MOP_undef, MOP_undef};
77 MOperator stFsRel[kFloatByteSizeDimension] = {MOP_undef, MOP_undef, MOP_undef};
78
79 /* extended to unsigned ints */
80 MOperator uextIs[kIntByteSizeDimension][kIntByteSizeDimension] = {
81 /* u8 u16 u32 u64 */
82 {MOP_undef, MOP_xuxtb32, MOP_xuxtb32, MOP_xuxtb32}, /* u8/i8 */
83 {MOP_undef, MOP_undef, MOP_xuxth32, MOP_xuxth32}, /* u16/i16 */
84 {MOP_undef, MOP_undef, MOP_xuxtw64, MOP_xuxtw64}, /* u32/i32 */
85 {MOP_undef, MOP_undef, MOP_undef, MOP_undef} /* u64/u64 */
86 };
87
88 /* extended to signed ints */
89 MOperator extIs[kIntByteSizeDimension][kIntByteSizeDimension] = {
90 /* i8 i16 i32 i64 */
91 {MOP_undef, MOP_xsxtb32, MOP_xsxtb32, MOP_xsxtb64}, /* u8/i8 */
92 {MOP_undef, MOP_undef, MOP_xsxth32, MOP_xsxth64}, /* u16/i16 */
93 {MOP_undef, MOP_undef, MOP_undef, MOP_xsxtw64}, /* u32/i32 */
94 {MOP_undef, MOP_undef, MOP_undef, MOP_undef} /* u64/u64 */
95 };
96
PickLdStInsn(bool isLoad,uint32 bitSize,PrimType primType,AArch64isa::MemoryOrdering memOrd)97 MOperator PickLdStInsn(bool isLoad, uint32 bitSize, PrimType primType, AArch64isa::MemoryOrdering memOrd)
98 {
99 DEBUG_ASSERT(__builtin_popcount(static_cast<uint32>(memOrd)) <= 1, "must be kMoNone or kMoAcquire");
100 DEBUG_ASSERT(bitSize >= k8BitSize, "PTY_u1 should have been lowered?");
101 DEBUG_ASSERT(__builtin_popcount(bitSize) == 1, "PTY_u1 should have been lowered?");
102 if (isLoad) {
103 DEBUG_ASSERT((memOrd == AArch64isa::kMoNone) || (memOrd == AArch64isa::kMoAcquire) ||
104 (memOrd == AArch64isa::kMoAcquireRcpc) || (memOrd == AArch64isa::kMoLoacquire),
105 "unknown Memory Order");
106 } else {
107 DEBUG_ASSERT((memOrd == AArch64isa::kMoNone) || (memOrd == AArch64isa::kMoRelease) ||
108 (memOrd == AArch64isa::kMoLorelease),
109 "unknown Memory Order");
110 }
111
112 /* __builtin_ffs(x) returns: 0 -> 0, 1 -> 1, 2 -> 2, 4 -> 3, 8 -> 4 */
113 if ((IsPrimitiveInteger(primType) || primType == PTY_agg) && !IsPrimitiveVector(primType)) {
114 MOperator(*table)[kIntByteSizeDimension];
115 if (isLoad) {
116 table = (memOrd == AArch64isa::kMoAcquire) ? ldIsAcq : ldIs;
117 } else {
118 table = (memOrd == AArch64isa::kMoRelease) ? stIsRel : stIs;
119 }
120
121 int32 signedUnsigned = IsUnsignedInteger(primType) ? 0 : 1;
122 if (primType == PTY_agg) {
123 CHECK_FATAL(bitSize >= k8BitSize, " unexpect agg size");
124 bitSize = static_cast<uint32>(RoundUp(bitSize, k8BitSize));
125 DEBUG_ASSERT((bitSize & (bitSize - 1)) == 0, "bitlen error");
126 }
127
128 /* __builtin_ffs(x) returns: 8 -> 4, 16 -> 5, 32 -> 6, 64 -> 7 */
129 if (primType == PTY_i128 || primType == PTY_u128) {
130 bitSize = k64BitSize;
131 }
132 uint32 size = static_cast<uint32>(__builtin_ffs(static_cast<int32>(bitSize))) - k4BitSize;
133 DEBUG_ASSERT(size <= 3, "wrong bitSize"); // size must <= 3
134 return table[signedUnsigned][size];
135 } else {
136 MOperator *table = nullptr;
137 if (isLoad) {
138 table = (memOrd == AArch64isa::kMoAcquire) ? ldFsAcq : ldFs;
139 } else {
140 table = (memOrd == AArch64isa::kMoRelease) ? stFsRel : stFs;
141 }
142
143 /* __builtin_ffs(x) returns: 32 -> 6, 64 -> 7, 128 -> 8 */
144 uint32 size = static_cast<uint32>(__builtin_ffs(static_cast<int32>(bitSize))) - k6BitSize;
145 DEBUG_ASSERT(size <= k2BitSize, "size must be 0 to 2");
146 return table[size];
147 }
148 }
149 } // namespace
150
IsBlkassignForPush(const BlkassignoffNode & bNode)151 bool IsBlkassignForPush(const BlkassignoffNode &bNode)
152 {
153 BaseNode *dest = bNode.Opnd(0);
154 bool spBased = false;
155 if (dest->GetOpCode() == OP_regread) {
156 RegreadNode &node = static_cast<RegreadNode &>(*dest);
157 if (-node.GetRegIdx() == kSregSp) {
158 spBased = true;
159 }
160 }
161 return spBased;
162 }
163
GetOrCreateResOperand(const BaseNode & parent,PrimType primType)164 RegOperand &AArch64CGFunc::GetOrCreateResOperand(const BaseNode &parent, PrimType primType)
165 {
166 RegOperand *resOpnd = nullptr;
167 if (parent.GetOpCode() == OP_regassign) {
168 auto ®AssignNode = static_cast<const RegassignNode &>(parent);
169 PregIdx pregIdx = regAssignNode.GetRegIdx();
170 if (IsSpecialPseudoRegister(pregIdx)) {
171 /* if it is one of special registers */
172 resOpnd = &GetOrCreateSpecialRegisterOperand(-pregIdx, primType);
173 } else {
174 resOpnd = &GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx));
175 }
176 } else {
177 resOpnd = &CreateRegisterOperandOfType(primType);
178 }
179 return *resOpnd;
180 }
181
PickLdInsn(uint32 bitSize,PrimType primType,AArch64isa::MemoryOrdering memOrd) const182 MOperator AArch64CGFunc::PickLdInsn(uint32 bitSize, PrimType primType, AArch64isa::MemoryOrdering memOrd) const
183 {
184 return PickLdStInsn(true, bitSize, primType, memOrd);
185 }
186
PickStInsn(uint32 bitSize,PrimType primType,AArch64isa::MemoryOrdering memOrd) const187 MOperator AArch64CGFunc::PickStInsn(uint32 bitSize, PrimType primType, AArch64isa::MemoryOrdering memOrd) const
188 {
189 return PickLdStInsn(false, bitSize, primType, memOrd);
190 }
191
PickExtInsn(PrimType dtype,PrimType stype) const192 MOperator AArch64CGFunc::PickExtInsn(PrimType dtype, PrimType stype) const
193 {
194 int32 sBitSize = static_cast<int32>(GetPrimTypeBitSize(stype));
195 int32 dBitSize = static_cast<int32>(GetPrimTypeBitSize(dtype));
196 /* __builtin_ffs(x) returns: 0 -> 0, 1 -> 1, 2 -> 2, 4 -> 3, 8 -> 4 */
197 if (IsPrimitiveInteger(stype) && IsPrimitiveInteger(dtype)) {
198 MOperator(*table)[kIntByteSizeDimension];
199 table = IsUnsignedInteger(stype) ? uextIs : extIs;
200 if (stype == PTY_i128 || stype == PTY_u128) {
201 sBitSize = static_cast<int32>(k64BitSize);
202 }
203 /* __builtin_ffs(x) returns: 8 -> 4, 16 -> 5, 32 -> 6, 64 -> 7 */
204 uint32 row = static_cast<uint32>(__builtin_ffs(sBitSize)) - k4BitSize;
205 DEBUG_ASSERT(row <= k3BitSize, "wrong bitSize");
206 if (dtype == PTY_i128 || dtype == PTY_u128) {
207 dBitSize = static_cast<int32>(k64BitSize);
208 }
209 uint32 col = static_cast<uint32>(__builtin_ffs(dBitSize)) - k4BitSize;
210 DEBUG_ASSERT(col <= k3BitSize, "wrong bitSize");
211 return table[row][col];
212 }
213 CHECK_FATAL(0, "extend not primitive integer");
214 return MOP_undef;
215 }
216
PickMovBetweenRegs(PrimType destType,PrimType srcType) const217 MOperator AArch64CGFunc::PickMovBetweenRegs(PrimType destType, PrimType srcType) const
218 {
219 if (IsPrimitiveVector(destType) && IsPrimitiveVector(srcType)) {
220 return GetPrimTypeSize(srcType) == k8ByteSize ? MOP_vmovuu : MOP_vmovvv;
221 }
222 if (IsPrimitiveInteger(destType) && IsPrimitiveInteger(srcType)) {
223 return GetPrimTypeSize(srcType) <= k4ByteSize ? MOP_wmovrr : MOP_xmovrr;
224 }
225 if (IsPrimitiveFloat(destType) && IsPrimitiveFloat(srcType)) {
226 return GetPrimTypeSize(srcType) <= k4ByteSize ? MOP_xvmovs : MOP_xvmovd;
227 }
228 if (IsPrimitiveInteger(destType) && IsPrimitiveFloat(srcType)) {
229 return GetPrimTypeSize(srcType) <= k4ByteSize ? MOP_xvmovrs : MOP_xvmovrd;
230 }
231 if (IsPrimitiveFloat(destType) && IsPrimitiveInteger(srcType)) {
232 return GetPrimTypeSize(srcType) <= k4ByteSize ? MOP_xvmovsr : MOP_xvmovdr;
233 }
234 if (IsPrimitiveInteger(destType) && IsPrimitiveVector(srcType)) {
235 return GetPrimTypeSize(srcType) == k8ByteSize
236 ? MOP_vwmovru
237 : GetPrimTypeSize(destType) <= k4ByteSize ? MOP_vwmovrv : MOP_vxmovrv;
238 }
239 CHECK_FATAL(false, "unexpected operand primtype for mov");
240 return MOP_undef;
241 }
242
PickMovInsn(const RegOperand & lhs,const RegOperand & rhs) const243 MOperator AArch64CGFunc::PickMovInsn(const RegOperand &lhs, const RegOperand &rhs) const
244 {
245 CHECK_FATAL(lhs.GetRegisterType() == rhs.GetRegisterType(), "PickMovInsn: unequal kind NYI");
246 CHECK_FATAL(lhs.GetSize() == rhs.GetSize(), "PickMovInsn: unequal size NYI");
247 DEBUG_ASSERT(((lhs.GetSize() < k64BitSize) || (lhs.GetRegisterType() == kRegTyFloat)),
248 "should split the 64 bits or more mov");
249 if (lhs.GetRegisterType() == kRegTyInt) {
250 return MOP_wmovrr;
251 }
252 if (lhs.GetRegisterType() == kRegTyFloat) {
253 return (lhs.GetSize() <= k32BitSize) ? MOP_xvmovs : MOP_xvmovd;
254 }
255 DEBUG_ASSERT(false, "PickMovInsn: kind NYI");
256 return MOP_undef;
257 }
258
SelectLoadAcquire(Operand & dest,PrimType dtype,Operand & src,PrimType stype,AArch64isa::MemoryOrdering memOrd,bool isDirect)259 void AArch64CGFunc::SelectLoadAcquire(Operand &dest, PrimType dtype, Operand &src, PrimType stype,
260 AArch64isa::MemoryOrdering memOrd, bool isDirect)
261 {
262 DEBUG_ASSERT(src.GetKind() == Operand::kOpdMem, "Just checking");
263 DEBUG_ASSERT(memOrd != AArch64isa::kMoNone, "Just checking");
264
265 uint32 ssize = isDirect ? src.GetSize() : GetPrimTypeBitSize(dtype);
266 uint32 dsize = GetPrimTypeBitSize(dtype);
267 MOperator mOp = PickLdInsn(ssize, stype, memOrd);
268
269 Operand *newSrc = &src;
270 auto &memOpnd = static_cast<MemOperand &>(src);
271 OfstOperand *immOpnd = memOpnd.GetOffsetImmediate();
272 int32 offset = static_cast<int32>(immOpnd->GetOffsetValue());
273 RegOperand *origBaseReg = memOpnd.GetBaseRegister();
274 if (offset != 0) {
275 RegOperand &resOpnd = CreateRegisterOperandOfType(PTY_i64);
276 DEBUG_ASSERT(origBaseReg != nullptr, "nullptr check");
277 SelectAdd(resOpnd, *origBaseReg, *immOpnd, PTY_i64);
278 newSrc = &CreateReplacementMemOperand(ssize, resOpnd, 0);
279 }
280
281 std::string key;
282 if (isDirect && GetCG()->GenerateVerboseCG()) {
283 key = GenerateMemOpndVerbose(src);
284 }
285
286 /* Check if the right load-acquire instruction is available. */
287 if (mOp != MOP_undef) {
288 Insn &insn = GetInsnBuilder()->BuildInsn(mOp, dest, *newSrc);
289 if (isDirect && GetCG()->GenerateVerboseCG()) {
290 insn.SetComment(key);
291 }
292 GetCurBB()->AppendInsn(insn);
293 } else {
294 if (IsPrimitiveFloat(stype)) {
295 /* Uses signed integer version ldar followed by a floating-point move(fmov). */
296 DEBUG_ASSERT(stype == dtype, "Just checking");
297 PrimType itype = (stype == PTY_f32) ? PTY_i32 : PTY_i64;
298 RegOperand ®Opnd = CreateRegisterOperandOfType(itype);
299 Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(ssize, itype, memOrd), regOpnd, *newSrc);
300 if (isDirect && GetCG()->GenerateVerboseCG()) {
301 insn.SetComment(key);
302 }
303 GetCurBB()->AppendInsn(insn);
304 mOp = (stype == PTY_f32) ? MOP_xvmovsr : MOP_xvmovdr;
305 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, dest, regOpnd));
306 } else {
307 /* Use unsigned version ldarb/ldarh followed by a sign-extension instruction(sxtb/sxth). */
308 DEBUG_ASSERT((ssize == k8BitSize) || (ssize == k16BitSize), "Just checking");
309 PrimType utype = (ssize == k8BitSize) ? PTY_u8 : PTY_u16;
310 Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(ssize, utype, memOrd), dest, *newSrc);
311 if (isDirect && GetCG()->GenerateVerboseCG()) {
312 insn.SetComment(key);
313 }
314 GetCurBB()->AppendInsn(insn);
315 mOp = ((dsize == k32BitSize) ? ((ssize == k8BitSize) ? MOP_xsxtb32 : MOP_xsxth32)
316 : ((ssize == k8BitSize) ? MOP_xsxtb64 : MOP_xsxth64));
317 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, dest, dest));
318 }
319 }
320 }
321
SelectStoreRelease(Operand & dest,PrimType dtype,Operand & src,PrimType stype,AArch64isa::MemoryOrdering memOrd,bool isDirect)322 void AArch64CGFunc::SelectStoreRelease(Operand &dest, PrimType dtype, Operand &src, PrimType stype,
323 AArch64isa::MemoryOrdering memOrd, bool isDirect)
324 {
325 DEBUG_ASSERT(dest.GetKind() == Operand::kOpdMem, "Just checking");
326
327 uint32 dsize = isDirect ? dest.GetSize() : GetPrimTypeBitSize(stype);
328 MOperator mOp = PickStInsn(dsize, stype, memOrd);
329
330 Operand *newDest = &dest;
331 MemOperand *memOpnd = static_cast<MemOperand *>(&dest);
332 OfstOperand *immOpnd = memOpnd->GetOffsetImmediate();
333 int32 offset = static_cast<int32>(immOpnd->GetOffsetValue());
334 RegOperand *origBaseReg = memOpnd->GetBaseRegister();
335 if (offset != 0) {
336 RegOperand &resOpnd = CreateRegisterOperandOfType(PTY_i64);
337 DEBUG_ASSERT(origBaseReg != nullptr, "nullptr check");
338 SelectAdd(resOpnd, *origBaseReg, *immOpnd, PTY_i64);
339 newDest = &CreateReplacementMemOperand(dsize, resOpnd, 0);
340 }
341
342 std::string key;
343 if (isDirect && GetCG()->GenerateVerboseCG()) {
344 key = GenerateMemOpndVerbose(dest);
345 }
346
347 /* Check if the right store-release instruction is available. */
348 if (mOp != MOP_undef) {
349 Insn &insn = GetInsnBuilder()->BuildInsn(mOp, src, *newDest);
350 if (isDirect && GetCG()->GenerateVerboseCG()) {
351 insn.SetComment(key);
352 }
353 GetCurBB()->AppendInsn(insn);
354 } else {
355 /* Use a floating-point move(fmov) followed by a stlr. */
356 DEBUG_ASSERT(IsPrimitiveFloat(stype), "must be float type");
357 CHECK_FATAL(stype == dtype, "Just checking");
358 PrimType itype = (stype == PTY_f32) ? PTY_i32 : PTY_i64;
359 RegOperand ®Opnd = CreateRegisterOperandOfType(itype);
360 mOp = (stype == PTY_f32) ? MOP_xvmovrs : MOP_xvmovrd;
361 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, regOpnd, src));
362 Insn &insn = GetInsnBuilder()->BuildInsn(PickStInsn(dsize, itype, memOrd), regOpnd, *newDest);
363 if (isDirect && GetCG()->GenerateVerboseCG()) {
364 insn.SetComment(key);
365 }
366 GetCurBB()->AppendInsn(insn);
367 }
368 }
369
SelectCopyImm(Operand & dest,PrimType dType,ImmOperand & src,PrimType sType)370 void AArch64CGFunc::SelectCopyImm(Operand &dest, PrimType dType, ImmOperand &src, PrimType sType)
371 {
372 if (IsPrimitiveInteger(dType) != IsPrimitiveInteger(sType)) {
373 RegOperand &tempReg = CreateRegisterOperandOfType(sType);
374 SelectCopyImm(tempReg, src, sType);
375 SelectCopy(dest, dType, tempReg, sType);
376 } else {
377 SelectCopyImm(dest, src, sType);
378 }
379 }
380
SelectCopyImm(Operand & dest,ImmOperand & src,PrimType dtype)381 void AArch64CGFunc::SelectCopyImm(Operand &dest, ImmOperand &src, PrimType dtype)
382 {
383 uint32 dsize = GetPrimTypeBitSize(dtype);
384 DEBUG_ASSERT(IsPrimitiveInteger(dtype), "The type of destination operand must be Integer");
385 DEBUG_ASSERT(((dsize == k8BitSize) || (dsize == k16BitSize) || (dsize == k32BitSize) || (dsize == k64BitSize)),
386 "The destination operand must be >= 8-bit");
387 if (src.IsSingleInstructionMovable()) {
388 MOperator mOp = (dsize == k32BitSize) ? MOP_wmovri32 : MOP_xmovri64;
389 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, dest, src));
390 return;
391 }
392 uint64 srcVal = static_cast<uint64>(src.GetValue());
393 /* using mov/movk to load the immediate value */
394 if (dsize == k8BitSize) {
395 /* compute lower 8 bits value */
396 if (dtype == PTY_u8) {
397 /* zero extend */
398 srcVal = (srcVal << k56BitSize) >> k56BitSize;
399 dtype = PTY_u16;
400 } else {
401 /* sign extend */
402 srcVal = (static_cast<int64>(srcVal) << k56BitSize) >> k56BitSize;
403 dtype = PTY_i16;
404 }
405 dsize = k16BitSize;
406 }
407 if (dsize == k16BitSize) {
408 if (dtype == PTY_u16) {
409 /* check lower 16 bits and higher 16 bits respectively */
410 DEBUG_ASSERT((srcVal & 0x0000FFFFULL) != 0, "unexpected value");
411 DEBUG_ASSERT(((srcVal >> k16BitSize) & 0x0000FFFFULL) == 0, "unexpected value");
412 DEBUG_ASSERT((srcVal & 0x0000FFFFULL) != 0xFFFFULL, "unexpected value");
413 /* create an imm opereand which represents lower 16 bits of the immediate */
414 ImmOperand &srcLower = CreateImmOperand(static_cast<int64>(srcVal & 0x0000FFFFULL), k16BitSize, false);
415 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wmovri32, dest, srcLower));
416 return;
417 } else {
418 /* sign extend and let `dsize == 32` case take care of it */
419 srcVal = (static_cast<int64>(srcVal) << k48BitSize) >> k48BitSize;
420 dsize = k32BitSize;
421 }
422 }
423 if (dsize == k32BitSize) {
424 /* check lower 16 bits and higher 16 bits respectively */
425 DEBUG_ASSERT((srcVal & 0x0000FFFFULL) != 0, "unexpected val");
426 DEBUG_ASSERT(((srcVal >> k16BitSize) & 0x0000FFFFULL) != 0, "unexpected val");
427 DEBUG_ASSERT((srcVal & 0x0000FFFFULL) != 0xFFFFULL, "unexpected val");
428 DEBUG_ASSERT(((srcVal >> k16BitSize) & 0x0000FFFFULL) != 0xFFFFULL, "unexpected val");
429 /* create an imm opereand which represents lower 16 bits of the immediate */
430 ImmOperand &srcLower = CreateImmOperand(static_cast<int64>(srcVal & 0x0000FFFFULL), k16BitSize, false);
431 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wmovri32, dest, srcLower));
432 /* create an imm opereand which represents upper 16 bits of the immediate */
433 ImmOperand &srcUpper =
434 CreateImmOperand(static_cast<int64>((srcVal >> k16BitSize) & 0x0000FFFFULL), k16BitSize, false);
435 BitShiftOperand *lslOpnd = GetLogicalShiftLeftOperand(k16BitSize, false);
436 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wmovkri16, dest, srcUpper, *lslOpnd));
437 } else {
438 /*
439 * partition it into 4 16-bit chunks
440 * if more 0's than 0xFFFF's, use movz as the initial instruction.
441 * otherwise, movn.
442 */
443 bool useMovz = BetterUseMOVZ(srcVal);
444 bool useMovk = false;
445 /* get lower 32 bits of the immediate */
446 uint64 chunkLval = srcVal & 0xFFFFFFFFULL;
447 /* get upper 32 bits of the immediate */
448 uint64 chunkHval = (srcVal >> k32BitSize) & 0xFFFFFFFFULL;
449 int32 maxLoopTime = 4;
450
451 if (chunkLval == chunkHval) {
452 /* compute lower 32 bits, and then copy to higher 32 bits, so only 2 chunks need be processed */
453 maxLoopTime = 2;
454 }
455
456 uint64 sa = 0;
457
458 for (int64 i = 0; i < maxLoopTime; ++i, sa += k16BitSize) {
459 /* create an imm opereand which represents the i-th 16-bit chunk of the immediate */
460 uint64 chunkVal = (srcVal >> (static_cast<uint64>(sa))) & 0x0000FFFFULL;
461 if (useMovz ? (chunkVal == 0) : (chunkVal == 0x0000FFFFULL)) {
462 continue;
463 }
464 ImmOperand &src16 = CreateImmOperand(static_cast<int64>(chunkVal), k16BitSize, false);
465 BitShiftOperand *lslOpnd = GetLogicalShiftLeftOperand(sa, true);
466 if (!useMovk) {
467 /* use movz or movn */
468 if (!useMovz) {
469 src16.BitwiseNegate();
470 }
471 GetCurBB()->AppendInsn(
472 GetInsnBuilder()->BuildInsn(useMovz ? MOP_xmovzri16 : MOP_xmovnri16, dest, src16, *lslOpnd));
473 useMovk = true;
474 } else {
475 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xmovkri16, dest, src16, *lslOpnd));
476 }
477 }
478
479 if (maxLoopTime == 2) { /* as described above, only 2 chunks need be processed */
480 /* copy lower 32 bits to higher 32 bits */
481 ImmOperand &immOpnd = CreateImmOperand(k32BitSize, k8BitSize, false);
482 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xbfirri6i6, dest, dest, immOpnd, immOpnd));
483 }
484 }
485 }
486
GenerateMemOpndVerbose(const Operand & src) const487 std::string AArch64CGFunc::GenerateMemOpndVerbose(const Operand &src) const
488 {
489 DEBUG_ASSERT(src.GetKind() == Operand::kOpdMem, "Just checking");
490 const MIRSymbol *symSecond = static_cast<const MemOperand *>(&src)->GetSymbol();
491 if (symSecond != nullptr) {
492 std::string key;
493 MIRStorageClass sc = symSecond->GetStorageClass();
494 if (sc == kScFormal) {
495 key = "param: ";
496 } else if (sc == kScAuto) {
497 key = "local var: ";
498 } else {
499 key = "global: ";
500 }
501 key += symSecond->GetName();
502 return key;
503 }
504 return "";
505 }
506
SelectCopyMemOpnd(Operand & dest,PrimType dtype,uint32 dsize,Operand & src,PrimType stype)507 void AArch64CGFunc::SelectCopyMemOpnd(Operand &dest, PrimType dtype, uint32 dsize, Operand &src, PrimType stype)
508 {
509 AArch64isa::MemoryOrdering memOrd = AArch64isa::kMoNone;
510 const MIRSymbol *sym = static_cast<MemOperand *>(&src)->GetSymbol();
511 if ((sym != nullptr) && (sym->GetStorageClass() == kScGlobal) && sym->GetAttr(ATTR_memory_order_acquire)) {
512 memOrd = AArch64isa::kMoAcquire;
513 }
514
515 if (memOrd != AArch64isa::kMoNone) {
516 AArch64CGFunc::SelectLoadAcquire(dest, dtype, src, stype, memOrd, true);
517 return;
518 }
519 Insn *insn = nullptr;
520 uint32 ssize = src.GetSize();
521 PrimType regTy = PTY_void;
522 RegOperand *loadReg = nullptr;
523 MOperator mop = MOP_undef;
524 if (IsPrimitiveFloat(stype) || IsPrimitiveVector(stype)) {
525 CHECK_FATAL(dsize == ssize, "dsize %u expect equals ssize %u", dtype, ssize);
526 insn = &GetInsnBuilder()->BuildInsn(PickLdInsn(ssize, stype), dest, src);
527 } else {
528 if (stype == PTY_agg && dtype == PTY_agg) {
529 mop = MOP_undef;
530 } else {
531 mop = PickExtInsn(dtype, stype);
532 }
533 if (ssize == (GetPrimTypeSize(dtype) * kBitsPerByte) || mop == MOP_undef) {
534 insn = &GetInsnBuilder()->BuildInsn(PickLdInsn(ssize, stype), dest, src);
535 } else {
536 regTy = dsize == k64BitSize ? dtype : PTY_i32;
537 loadReg = &CreateRegisterOperandOfType(regTy);
538 insn = &GetInsnBuilder()->BuildInsn(PickLdInsn(ssize, stype), *loadReg, src);
539 }
540 }
541
542 if (GetCG()->GenerateVerboseCG()) {
543 insn->SetComment(GenerateMemOpndVerbose(src));
544 }
545
546 GetCurBB()->AppendInsn(*insn);
547 if (regTy != PTY_void && mop != MOP_undef) {
548 DEBUG_ASSERT(loadReg != nullptr, "loadReg should not be nullptr");
549 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mop, dest, *loadReg));
550 }
551 }
552
IsImmediateValueInRange(MOperator mOp,int64 immVal,bool is64Bits,bool isIntactIndexed,bool isPostIndexed,bool isPreIndexed) const553 bool AArch64CGFunc::IsImmediateValueInRange(MOperator mOp, int64 immVal, bool is64Bits, bool isIntactIndexed,
554 bool isPostIndexed, bool isPreIndexed) const
555 {
556 bool isInRange = false;
557 switch (mOp) {
558 case MOP_xstr:
559 case MOP_wstr:
560 isInRange =
561 (isIntactIndexed &&
562 ((!is64Bits && (immVal >= kStrAllLdrAllImmLowerBound) && (immVal <= kStrLdrImm32UpperBound)) ||
563 (is64Bits && (immVal >= kStrAllLdrAllImmLowerBound) && (immVal <= kStrLdrImm64UpperBound)))) ||
564 ((isPostIndexed || isPreIndexed) && (immVal >= kStrLdrPerPostLowerBound) &&
565 (immVal <= kStrLdrPerPostUpperBound));
566 break;
567 case MOP_wstrb:
568 isInRange =
569 (isIntactIndexed && (immVal >= kStrAllLdrAllImmLowerBound) && (immVal <= kStrbLdrbImmUpperBound)) ||
570 ((isPostIndexed || isPreIndexed) && (immVal >= kStrLdrPerPostLowerBound) &&
571 (immVal <= kStrLdrPerPostUpperBound));
572 break;
573 case MOP_wstrh:
574 isInRange =
575 (isIntactIndexed && (immVal >= kStrAllLdrAllImmLowerBound) && (immVal <= kStrhLdrhImmUpperBound)) ||
576 ((isPostIndexed || isPreIndexed) && (immVal >= kStrLdrPerPostLowerBound) &&
577 (immVal <= kStrLdrPerPostUpperBound));
578 break;
579 default:
580 break;
581 }
582 return isInRange;
583 }
584
IsStoreMop(MOperator mOp) const585 bool AArch64CGFunc::IsStoreMop(MOperator mOp) const
586 {
587 switch (mOp) {
588 case MOP_sstr:
589 case MOP_dstr:
590 case MOP_qstr:
591 case MOP_xstr:
592 case MOP_wstr:
593 case MOP_wstrb:
594 case MOP_wstrh:
595 return true;
596 default:
597 return false;
598 }
599 }
600
SplitMovImmOpndInstruction(int64 immVal,RegOperand & destReg,Insn * curInsn)601 void AArch64CGFunc::SplitMovImmOpndInstruction(int64 immVal, RegOperand &destReg, Insn *curInsn)
602 {
603 bool useMovz = BetterUseMOVZ(immVal);
604 bool useMovk = false;
605 /* get lower 32 bits of the immediate */
606 uint64 chunkLval = static_cast<uint64>(immVal) & 0xFFFFFFFFULL;
607 /* get upper 32 bits of the immediate */
608 uint64 chunkHval = (static_cast<uint64>(immVal) >> k32BitSize) & 0xFFFFFFFFULL;
609 int32 maxLoopTime = 4;
610
611 if (chunkLval == chunkHval) {
612 /* compute lower 32 bits, and then copy to higher 32 bits, so only 2 chunks need be processed */
613 maxLoopTime = 2;
614 }
615
616 uint64 sa = 0;
617 auto *bb = (curInsn != nullptr) ? curInsn->GetBB() : GetCurBB();
618 for (int64 i = 0; i < maxLoopTime; ++i, sa += k16BitSize) {
619 /* create an imm opereand which represents the i-th 16-bit chunk of the immediate */
620 uint64 chunkVal = (static_cast<uint64>(immVal) >> sa) & 0x0000FFFFULL;
621 if (useMovz ? (chunkVal == 0) : (chunkVal == 0x0000FFFFULL)) {
622 continue;
623 }
624 ImmOperand &src16 = CreateImmOperand(static_cast<int64>(chunkVal), k16BitSize, false);
625 BitShiftOperand *lslOpnd = GetLogicalShiftLeftOperand(sa, true);
626 Insn *newInsn = nullptr;
627 if (!useMovk) {
628 /* use movz or movn */
629 if (!useMovz) {
630 src16.BitwiseNegate();
631 }
632 MOperator mOpCode = useMovz ? MOP_xmovzri16 : MOP_xmovnri16;
633 newInsn = &GetInsnBuilder()->BuildInsn(mOpCode, destReg, src16, *lslOpnd);
634 useMovk = true;
635 } else {
636 newInsn = &GetInsnBuilder()->BuildInsn(MOP_xmovkri16, destReg, src16, *lslOpnd);
637 }
638 if (curInsn != nullptr) {
639 bb->InsertInsnBefore(*curInsn, *newInsn);
640 } else {
641 bb->AppendInsn(*newInsn);
642 }
643 }
644
645 if (maxLoopTime == 2) { // compute lower 32 bits, and copy to higher 32 bits, so only 2 chunks need be processed
646 /* copy lower 32 bits to higher 32 bits */
647 ImmOperand &immOpnd = CreateImmOperand(k32BitSize, k8BitSize, false);
648 Insn &insn = GetInsnBuilder()->BuildInsn(MOP_xbfirri6i6, destReg, destReg, immOpnd, immOpnd);
649 if (curInsn != nullptr) {
650 bb->InsertInsnBefore(*curInsn, insn);
651 } else {
652 bb->AppendInsn(insn);
653 }
654 }
655 }
656
SelectCopyRegOpnd(Operand & dest,PrimType dtype,Operand::OperandType opndType,uint32 dsize,Operand & src,PrimType stype)657 void AArch64CGFunc::SelectCopyRegOpnd(Operand &dest, PrimType dtype, Operand::OperandType opndType, uint32 dsize,
658 Operand &src, PrimType stype)
659 {
660 if (opndType != Operand::kOpdMem) {
661 if (!CGOptions::IsArm64ilp32()) {
662 DEBUG_ASSERT(stype != PTY_a32, "");
663 }
664 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickMovBetweenRegs(dtype, stype), dest, src));
665 return;
666 }
667 AArch64isa::MemoryOrdering memOrd = AArch64isa::kMoNone;
668 const MIRSymbol *sym = static_cast<MemOperand *>(&dest)->GetSymbol();
669 if ((sym != nullptr) && (sym->GetStorageClass() == kScGlobal) && sym->GetAttr(ATTR_memory_order_release)) {
670 memOrd = AArch64isa::kMoRelease;
671 }
672
673 if (memOrd != AArch64isa::kMoNone) {
674 AArch64CGFunc::SelectStoreRelease(dest, dtype, src, stype, memOrd, true);
675 return;
676 }
677
678 bool is64Bits = (dest.GetSize() == k64BitSize) ? true : false;
679 MOperator strMop = PickStInsn(dsize, stype);
680 if (!dest.IsMemoryAccessOperand()) {
681 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(strMop, src, dest));
682 return;
683 }
684
685 MemOperand *memOpnd = static_cast<MemOperand *>(&dest);
686 DEBUG_ASSERT(memOpnd != nullptr, "memOpnd should not be nullptr");
687 if (memOpnd->GetAddrMode() == MemOperand::kAddrModeLo12Li) {
688 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(strMop, src, dest));
689 return;
690 }
691 if (memOpnd->GetOffsetOperand() == nullptr) {
692 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(strMop, src, dest));
693 return;
694 }
695 ImmOperand *immOpnd = static_cast<ImmOperand *>(memOpnd->GetOffsetOperand());
696 DEBUG_ASSERT(immOpnd != nullptr, "immOpnd should not be nullptr");
697 int64 immVal = immOpnd->GetValue();
698 bool isIntactIndexed = memOpnd->IsIntactIndexed();
699 bool isPostIndexed = memOpnd->IsPostIndexed();
700 bool isPreIndexed = memOpnd->IsPreIndexed();
701 DEBUG_ASSERT(!isPostIndexed, "memOpnd should not be post-index type");
702 DEBUG_ASSERT(!isPreIndexed, "memOpnd should not be pre-index type");
703 bool isInRange = false;
704 if (!GetMirModule().IsCModule()) {
705 isInRange = IsImmediateValueInRange(strMop, immVal, is64Bits, isIntactIndexed, isPostIndexed, isPreIndexed);
706 } else {
707 isInRange = IsOperandImmValid(strMop, memOpnd, kInsnSecondOpnd);
708 }
709 bool isMopStr = IsStoreMop(strMop);
710 if (isInRange || !isMopStr) {
711 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(strMop, src, dest));
712 return;
713 }
714 DEBUG_ASSERT(memOpnd->GetBaseRegister() != nullptr, "nullptr check");
715 if (isIntactIndexed) {
716 memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, dsize);
717 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(strMop, src, *memOpnd));
718 } else if (isPostIndexed || isPreIndexed) {
719 RegOperand ® = CreateRegisterOperandOfType(PTY_i64);
720 MOperator mopMov = MOP_xmovri64;
721 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopMov, reg, *immOpnd));
722 MOperator mopAdd = MOP_xaddrrr;
723 MemOperand &newDest =
724 GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, GetPrimTypeBitSize(dtype), memOpnd->GetBaseRegister(), nullptr,
725 &GetOrCreateOfstOpnd(0, k32BitSize), nullptr);
726 Insn &insn1 = GetInsnBuilder()->BuildInsn(strMop, src, newDest);
727 Insn &insn2 = GetInsnBuilder()->BuildInsn(mopAdd, *newDest.GetBaseRegister(), *newDest.GetBaseRegister(), reg);
728 if (isPostIndexed) {
729 GetCurBB()->AppendInsn(insn1);
730 GetCurBB()->AppendInsn(insn2);
731 } else {
732 /* isPreIndexed */
733 GetCurBB()->AppendInsn(insn2);
734 GetCurBB()->AppendInsn(insn1);
735 }
736 }
737 }
738
SelectCopy(Operand & dest,PrimType dtype,Operand & src,PrimType stype)739 void AArch64CGFunc::SelectCopy(Operand &dest, PrimType dtype, Operand &src, PrimType stype)
740 {
741 DEBUG_ASSERT(dest.IsRegister() || dest.IsMemoryAccessOperand(), "");
742 uint32 dsize = GetPrimTypeBitSize(dtype);
743 if (dest.IsRegister()) {
744 dsize = dest.GetSize();
745 }
746 Operand::OperandType opnd0Type = dest.GetKind();
747 Operand::OperandType opnd1Type = src.GetKind();
748 DEBUG_ASSERT(((dsize >= src.GetSize()) || (opnd0Type == Operand::kOpdRegister) || (opnd0Type == Operand::kOpdMem)),
749 "NYI");
750 DEBUG_ASSERT(((opnd0Type == Operand::kOpdRegister) || (src.GetKind() == Operand::kOpdRegister)),
751 "either src or dest should be register");
752
753 switch (opnd1Type) {
754 case Operand::kOpdMem:
755 SelectCopyMemOpnd(dest, dtype, dsize, src, stype);
756 break;
757 case Operand::kOpdOffset:
758 case Operand::kOpdImmediate:
759 SelectCopyImm(dest, dtype, static_cast<ImmOperand &>(src), stype);
760 break;
761 case Operand::kOpdFPImmediate:
762 CHECK_FATAL(static_cast<ImmOperand &>(src).GetValue() == 0, "NIY");
763 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn((dsize == k32BitSize) ? MOP_xvmovsr : MOP_xvmovdr, dest,
764 GetZeroOpnd(dsize)));
765 break;
766 case Operand::kOpdRegister: {
767 if (opnd0Type == Operand::kOpdRegister && IsPrimitiveVector(stype)) {
768 /* check vector reg to vector reg move */
769 CHECK_FATAL(IsPrimitiveVector(dtype), "invalid vectreg to vectreg move");
770 MOperator mop = (dsize <= k64BitSize) ? MOP_vmovuu : MOP_vmovvv;
771 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
772 vInsn.AddOpndChain(dest).AddOpndChain(src);
773 auto *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(dsize >> k3ByteSize, k8BitSize);
774 auto *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(dsize >> k3ByteSize, k8BitSize);
775 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpecSrc);
776 GetCurBB()->AppendInsn(vInsn);
777 break;
778 }
779 if (dest.IsRegister()) {
780 RegOperand &desReg = static_cast<RegOperand &>(dest);
781 RegOperand &srcReg = static_cast<RegOperand &>(src);
782 if (desReg.GetRegisterNumber() == srcReg.GetRegisterNumber()) {
783 break;
784 }
785 }
786 SelectCopyRegOpnd(dest, dtype, opnd0Type, dsize, src, stype);
787 break;
788 }
789 default:
790 CHECK_FATAL(false, "NYI");
791 }
792 }
793
794 /* This function copies src to a register, the src can be an imm, mem or a label */
SelectCopy(Operand & src,PrimType stype,PrimType dtype)795 RegOperand &AArch64CGFunc::SelectCopy(Operand &src, PrimType stype, PrimType dtype)
796 {
797 RegOperand &dest = CreateRegisterOperandOfType(dtype);
798 SelectCopy(dest, dtype, src, stype);
799 return dest;
800 }
801
802 /*
803 * We need to adjust the offset of a stack allocated local variable
804 * if we store FP/SP before any other local variables to save an instruction.
805 * See AArch64CGFunc::OffsetAdjustmentForFPLR() in aarch64_cgfunc.cpp
806 *
807 * That is when we !UsedStpSubPairForCallFrameAllocation().
808 *
809 * Because we need to use the STP/SUB instruction pair to store FP/SP 'after'
810 * local variables when the call frame size is greater that the max offset
811 * value allowed for the STP instruction (we cannot use STP w/ prefix, LDP w/
812 * postfix), if UsedStpSubPairForCallFrameAllocation(), we don't need to
813 * adjust the offsets.
814 */
IsImmediateOffsetOutOfRange(const MemOperand & memOpnd,uint32 bitLen)815 bool AArch64CGFunc::IsImmediateOffsetOutOfRange(const MemOperand &memOpnd, uint32 bitLen)
816 {
817 DEBUG_ASSERT(bitLen >= k8BitSize, "bitlen error");
818 DEBUG_ASSERT(bitLen <= k128BitSize, "bitlen error");
819
820 if (bitLen >= k8BitSize) {
821 bitLen = static_cast<uint32>(RoundUp(bitLen, k8BitSize));
822 }
823 DEBUG_ASSERT((bitLen & (bitLen - 1)) == 0, "bitlen error");
824
825 MemOperand::AArch64AddressingMode mode = memOpnd.GetAddrMode();
826 if ((mode == MemOperand::kAddrModeBOi) && memOpnd.IsIntactIndexed()) {
827 int32 offsetValue = static_cast<int32>(memOpnd.GetOffsetImmediate()->GetOffsetValue());
828 if (memOpnd.GetOffsetImmediate()->GetVary() == kUnAdjustVary) {
829 offsetValue +=
830 static_cast<int32>(static_cast<AArch64MemLayout *>(GetMemlayout())->RealStackFrameSize() + 0xff);
831 }
832 offsetValue += kIntregBytelen << 1; /* Refer to the above comment */
833 return MemOperand::IsPIMMOffsetOutOfRange(offsetValue, bitLen);
834 } else {
835 return false;
836 }
837 }
838
IsOperandImmValid(MOperator mOp,Operand * o,uint32 opndIdx)839 bool AArch64CGFunc::IsOperandImmValid(MOperator mOp, Operand *o, uint32 opndIdx)
840 {
841 const InsnDesc *md = &AArch64CG::kMd[mOp];
842 auto *opndProp = md->opndMD[opndIdx];
843
844 Operand::OperandType opndTy = opndProp->GetOperandType();
845 if (opndTy == Operand::kOpdMem) {
846 auto *memOpnd = static_cast<MemOperand *>(o);
847 if (memOpnd->GetAddrMode() == MemOperand::kAddrModeBOrX) {
848 return true;
849 }
850 if (md->IsLoadStorePair() ||
851 (memOpnd->GetAddrMode() == MemOperand::kAddrModeBOi && memOpnd->IsIntactIndexed())) {
852 int64 offsetValue = memOpnd->GetOffsetImmediate()->GetOffsetValue();
853 if (memOpnd->GetOffsetImmediate()->GetVary() == kUnAdjustVary) {
854 offsetValue += static_cast<AArch64MemLayout *>(GetMemlayout())->RealStackFrameSize() + 0xffL;
855 }
856 return md->IsValidImmOpnd(offsetValue);
857 } else if (memOpnd->GetAddrMode() == MemOperand::kAddrModeLo12Li) {
858 int32 offsetValue = static_cast<int32>(memOpnd->GetOffsetImmediate()->GetOffsetValue());
859 return offsetValue == 0;
860 } else {
861 CHECK_FATAL(!memOpnd->IsIntactIndexed(), "CHECK WHAT?");
862 int32 offsetValue = static_cast<int32>(memOpnd->GetOffsetImmediate()->GetOffsetValue());
863 return (offsetValue <= static_cast<int32>(k256BitSize) && offsetValue >= kNegative256BitSize);
864 }
865 } else if (opndTy == Operand::kOpdImmediate) {
866 return md->IsValidImmOpnd(static_cast<ImmOperand *>(o)->GetValue());
867 }
868 return true;
869 }
870
CreateReplacementMemOperand(uint32 bitLen,RegOperand & baseReg,int64 offset)871 MemOperand &AArch64CGFunc::CreateReplacementMemOperand(uint32 bitLen, RegOperand &baseReg, int64 offset)
872 {
873 return CreateMemOpnd(baseReg, offset, bitLen);
874 }
875
CheckIfSplitOffsetWithAdd(const MemOperand & memOpnd,uint32 bitLen) const876 bool AArch64CGFunc::CheckIfSplitOffsetWithAdd(const MemOperand &memOpnd, uint32 bitLen) const
877 {
878 if (memOpnd.GetAddrMode() != MemOperand::kAddrModeBOi || !memOpnd.IsIntactIndexed()) {
879 return false;
880 }
881 OfstOperand *ofstOpnd = memOpnd.GetOffsetImmediate();
882 int32 opndVal = static_cast<int32>(ofstOpnd->GetOffsetValue());
883 int32 maxPimm = memOpnd.GetMaxPIMM(bitLen);
884 int32 q0 = opndVal / maxPimm;
885 int32 addend = q0 * maxPimm;
886 int32 r0 = opndVal - addend;
887 int32 alignment = memOpnd.GetImmediateOffsetAlignment(bitLen);
888 int32 r1 = static_cast<uint32>(r0) & ((1u << static_cast<uint32>(alignment)) - 1);
889 addend = addend + r1;
890 return (addend > 0);
891 }
892
GetBaseRegForSplit(uint32 baseRegNum)893 RegOperand *AArch64CGFunc::GetBaseRegForSplit(uint32 baseRegNum)
894 {
895 RegOperand *resOpnd = nullptr;
896 if (baseRegNum == AArch64reg::kRinvalid) {
897 resOpnd = &CreateRegisterOperandOfType(PTY_i64);
898 } else if (AArch64isa::IsPhysicalRegister(baseRegNum)) {
899 resOpnd = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(baseRegNum),
900 GetPointerSize() * kBitsPerByte, kRegTyInt);
901 } else {
902 resOpnd = &GetOrCreateVirtualRegisterOperand(baseRegNum);
903 }
904 return resOpnd;
905 }
906
907 /*
908 * When immediate of str/ldr is over 256bits, it should be aligned according to the reg byte size.
909 * Here we split the offset into (512 * n) and +/-(new Offset) when misaligned, to make sure that
910 * the new offet is always under 256 bits.
911 */
ConstraintOffsetToSafeRegion(uint32 bitLen,const MemOperand & memOpnd)912 MemOperand &AArch64CGFunc::ConstraintOffsetToSafeRegion(uint32 bitLen, const MemOperand &memOpnd)
913 {
914 auto it = hashMemOpndTable.find(memOpnd);
915 if (it != hashMemOpndTable.end()) {
916 hashMemOpndTable.erase(memOpnd);
917 }
918 int32 offsetValue = static_cast<int32>(memOpnd.GetOffsetImmediate()->GetOffsetValue());
919 int32 multiplier = (offsetValue / k512BitSize) + static_cast<int32>(offsetValue % k512BitSize > k256BitSize);
920 int32 addMount = multiplier * k512BitSizeInt;
921 int32 newOffset = offsetValue - addMount;
922 RegOperand *baseReg = memOpnd.GetBaseRegister();
923 ImmOperand &immAddMount = CreateImmOperand(addMount, k64BitSize, true);
924 if (memOpnd.GetOffsetImmediate()->GetVary() == kUnAdjustVary) {
925 immAddMount.SetVary(kUnAdjustVary);
926 }
927
928 RegOperand *resOpnd = GetBaseRegForSplit(kRinvalid);
929 SelectAdd(*resOpnd, *baseReg, immAddMount, PTY_i64);
930 MemOperand &newMemOpnd = CreateReplacementMemOperand(bitLen, *resOpnd, newOffset);
931 newMemOpnd.SetStackMem(memOpnd.IsStackMem());
932 return newMemOpnd;
933 }
934
SplitAndGetRemained(const MemOperand & memOpnd,uint32 bitLen,RegOperand * resOpnd,int64 ofstVal,bool isDest,Insn * insn,bool forPair)935 ImmOperand &AArch64CGFunc::SplitAndGetRemained(const MemOperand &memOpnd, uint32 bitLen, RegOperand *resOpnd,
936 int64 ofstVal, bool isDest, Insn *insn, bool forPair)
937 {
938 auto it = hashMemOpndTable.find(memOpnd);
939 if (it != hashMemOpndTable.end()) {
940 hashMemOpndTable.erase(memOpnd);
941 }
942 /*
943 * opndVal == Q0 * 32760(16380) + R0
944 * R0 == Q1 * 8(4) + R1
945 * ADDEND == Q0 * 32760(16380) + R1
946 * NEW_OFFSET = Q1 * 8(4)
947 * we want to generate two instructions:
948 * ADD TEMP_REG, X29, ADDEND
949 * LDR/STR TEMP_REG, [ TEMP_REG, #NEW_OFFSET ]
950 */
951 int32 maxPimm = 0;
952 if (!forPair) {
953 maxPimm = MemOperand::GetMaxPIMM(bitLen);
954 } else {
955 maxPimm = MemOperand::GetMaxPairPIMM(bitLen);
956 }
957 DEBUG_ASSERT(maxPimm != 0, "get max pimm failed");
958
959 int64 q0 = ofstVal / maxPimm + (ofstVal < 0 ? -1 : 0);
960 int64 addend = q0 * maxPimm;
961 int64 r0 = ofstVal - addend;
962 int64 alignment = MemOperand::GetImmediateOffsetAlignment(bitLen);
963 auto q1 = static_cast<int64>(static_cast<uint64>(r0) >> static_cast<uint64>(alignment));
964 auto r1 = static_cast<int64>(static_cast<uint64>(r0) & ((1u << static_cast<uint64>(alignment)) - 1));
965 auto remained = static_cast<int64>(static_cast<uint64>(q1) << static_cast<uint64>(alignment));
966 addend = addend + r1;
967 if (addend > 0) {
968 int64 suffixClear = 0xfff;
969 if (forPair) {
970 suffixClear = 0xff;
971 }
972 int64 remainedTmp = remained + (addend & suffixClear);
973 if (!MemOperand::IsPIMMOffsetOutOfRange(static_cast<int32>(remainedTmp), bitLen) &&
974 ((static_cast<uint64>(remainedTmp) & ((1u << static_cast<uint64>(alignment)) - 1)) == 0)) {
975 remained = remainedTmp;
976 addend = (addend & ~suffixClear);
977 }
978 }
979 ImmOperand &immAddend = CreateImmOperand(addend, k64BitSize, true);
980 if (memOpnd.GetOffsetImmediate()->GetVary() == kUnAdjustVary) {
981 immAddend.SetVary(kUnAdjustVary);
982 }
983 return immAddend;
984 }
985
SplitOffsetWithAddInstruction(const MemOperand & memOpnd,uint32 bitLen,uint32 baseRegNum,bool isDest,Insn * insn,bool forPair)986 MemOperand &AArch64CGFunc::SplitOffsetWithAddInstruction(const MemOperand &memOpnd, uint32 bitLen, uint32 baseRegNum,
987 bool isDest, Insn *insn, bool forPair)
988 {
989 DEBUG_ASSERT((memOpnd.GetAddrMode() == MemOperand::kAddrModeBOi), "expect kAddrModeBOi memOpnd");
990 DEBUG_ASSERT(memOpnd.IsIntactIndexed(), "expect intactIndexed memOpnd");
991 OfstOperand *ofstOpnd = memOpnd.GetOffsetImmediate();
992 int64 ofstVal = ofstOpnd->GetOffsetValue();
993 RegOperand *resOpnd = GetBaseRegForSplit(baseRegNum);
994 ImmOperand &immAddend = SplitAndGetRemained(memOpnd, bitLen, resOpnd, ofstVal, isDest, insn, forPair);
995 int64 remained = (ofstVal - immAddend.GetValue());
996 RegOperand *origBaseReg = memOpnd.GetBaseRegister();
997 DEBUG_ASSERT(origBaseReg != nullptr, "nullptr check");
998 if (insn == nullptr) {
999 SelectAdd(*resOpnd, *origBaseReg, immAddend, PTY_i64);
1000 } else {
1001 SelectAddAfterInsn(*resOpnd, *origBaseReg, immAddend, PTY_i64, isDest, *insn);
1002 }
1003 MemOperand &newMemOpnd = CreateReplacementMemOperand(bitLen, *resOpnd, remained);
1004 newMemOpnd.SetStackMem(memOpnd.IsStackMem());
1005 return newMemOpnd;
1006 }
1007
SelectDassign(DassignNode & stmt,Operand & opnd0)1008 void AArch64CGFunc::SelectDassign(DassignNode &stmt, Operand &opnd0)
1009 {
1010 SelectDassign(stmt.GetStIdx(), stmt.GetFieldID(), stmt.GetRHS()->GetPrimType(), opnd0);
1011 }
1012
1013 /*
1014 * Used for SelectDassign when do optimization for volatile store, because the stlr instruction only allow
1015 * store to the memory addrress with the register base offset 0.
1016 * STLR <Wt>, [<Xn|SP>{,#0}], 32-bit variant (size = 10)
1017 * STLR <Xt>, [<Xn|SP>{,#0}], 64-bit variant (size = 11)
1018 * So the function do the prehandle of the memory operand to satisify the Store-Release..
1019 */
ExtractNewMemBase(const MemOperand & memOpnd)1020 RegOperand *AArch64CGFunc::ExtractNewMemBase(const MemOperand &memOpnd)
1021 {
1022 const MIRSymbol *sym = memOpnd.GetSymbol();
1023 MemOperand::AArch64AddressingMode mode = memOpnd.GetAddrMode();
1024 if (mode == MemOperand::kAddrModeLiteral) {
1025 return nullptr;
1026 }
1027 RegOperand *baseOpnd = memOpnd.GetBaseRegister();
1028 DEBUG_ASSERT(baseOpnd != nullptr, "nullptr check");
1029 RegOperand &resultOpnd =
1030 CreateRegisterOperandOfType(baseOpnd->GetRegisterType(), baseOpnd->GetSize() / kBitsPerByte);
1031 bool is64Bits = (baseOpnd->GetSize() == k64BitSize);
1032 if (mode == MemOperand::kAddrModeLo12Li) {
1033 StImmOperand &stImm = CreateStImmOperand(*sym, 0, 0);
1034 Insn &addInsn = GetInsnBuilder()->BuildInsn(MOP_xadrpl12, resultOpnd, *baseOpnd, stImm);
1035 addInsn.SetComment("new add insn");
1036 GetCurBB()->AppendInsn(addInsn);
1037 } else if (mode == MemOperand::kAddrModeBOi) {
1038 OfstOperand *offsetOpnd = memOpnd.GetOffsetImmediate();
1039 if (offsetOpnd->GetOffsetValue() != 0) {
1040 MOperator mOp = is64Bits ? MOP_xaddrri12 : MOP_waddrri12;
1041 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resultOpnd, *baseOpnd, *offsetOpnd));
1042 } else {
1043 return baseOpnd;
1044 }
1045 } else {
1046 CHECK_FATAL(mode == MemOperand::kAddrModeBOrX, "unexpect addressing mode.");
1047 RegOperand *regOpnd = static_cast<const MemOperand *>(&memOpnd)->GetIndexRegister();
1048 MOperator mOp = is64Bits ? MOP_xaddrrr : MOP_waddrrr;
1049 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resultOpnd, *baseOpnd, *regOpnd));
1050 }
1051 return &resultOpnd;
1052 }
1053
1054 /*
1055 * NOTE: I divided SelectDassign so that we can create "virtual" assignments
1056 * when selecting other complex Maple IR instructions. For example, the atomic
1057 * exchange and other intrinsics will need to assign its results to local
1058 * variables. Such Maple IR instructions are pltform-specific (e.g.
1059 * atomic_exchange can be implemented as one single machine intruction on x86_64
1060 * and ARMv8.1, but ARMv8.0 needs an LL/SC loop), therefore they cannot (in
1061 * principle) be lowered at BELowerer or CGLowerer.
1062 */
SelectDassign(StIdx stIdx,FieldID fieldId,PrimType rhsPType,Operand & opnd0)1063 void AArch64CGFunc::SelectDassign(StIdx stIdx, FieldID fieldId, PrimType rhsPType, Operand &opnd0)
1064 {
1065 MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(stIdx);
1066 int32 offset = 0;
1067 bool parmCopy = false;
1068 if (fieldId != 0) {
1069 MIRStructType *structType = static_cast<MIRStructType *>(symbol->GetType());
1070 DEBUG_ASSERT(structType != nullptr, "SelectDassign: non-zero fieldID for non-structure");
1071 offset = GetBecommon().GetFieldOffset(*structType, fieldId).first;
1072 parmCopy = IsParamStructCopy(*symbol);
1073 }
1074 uint32 regSize = GetPrimTypeBitSize(rhsPType);
1075 MIRType *type = symbol->GetType();
1076 Operand &stOpnd = LoadIntoRegister(opnd0, IsPrimitiveInteger(rhsPType) || IsPrimitiveVectorInteger(rhsPType),
1077 regSize, IsSignedInteger(type->GetPrimType()));
1078 MOperator mOp = MOP_undef;
1079 if ((type->GetKind() == kTypeStruct) || (type->GetKind() == kTypeUnion)) {
1080 MIRStructType *structType = static_cast<MIRStructType *>(type);
1081 type = structType->GetFieldType(fieldId);
1082 } else if (type->GetKind() == kTypeClass) {
1083 MIRClassType *classType = static_cast<MIRClassType *>(type);
1084 type = classType->GetFieldType(fieldId);
1085 }
1086
1087 uint32 dataSize = GetPrimTypeBitSize(type->GetPrimType());
1088 if (type->GetPrimType() == PTY_agg) {
1089 dataSize = GetPrimTypeBitSize(PTY_a64);
1090 }
1091 MemOperand *memOpnd = nullptr;
1092 if (parmCopy) {
1093 memOpnd = &LoadStructCopyBase(*symbol, offset, static_cast<int>(dataSize));
1094 } else {
1095 memOpnd = &GetOrCreateMemOpnd(*symbol, offset, dataSize);
1096 }
1097 if ((memOpnd->GetMemVaryType() == kNotVary) && IsImmediateOffsetOutOfRange(*memOpnd, dataSize)) {
1098 memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, dataSize);
1099 }
1100
1101 /* In bpl mode, a func symbol's type is represented as a MIRFuncType instead of a MIRPtrType (pointing to
1102 * MIRFuncType), so we allow `kTypeFunction` to appear here */
1103 DEBUG_ASSERT(((type->GetKind() == kTypeScalar) || (type->GetKind() == kTypePointer) ||
1104 (type->GetKind() == kTypeFunction) || (type->GetKind() == kTypeStruct) ||
1105 (type->GetKind() == kTypeUnion) || (type->GetKind() == kTypeArray)),
1106 "NYI dassign type");
1107 PrimType ptyp = type->GetPrimType();
1108 if (ptyp == PTY_agg) {
1109 ptyp = PTY_a64;
1110 }
1111
1112 AArch64isa::MemoryOrdering memOrd = AArch64isa::kMoNone;
1113 if (isVolStore) {
1114 RegOperand *baseOpnd = ExtractNewMemBase(*memOpnd);
1115 if (baseOpnd != nullptr) {
1116 memOpnd = &CreateMemOpnd(*baseOpnd, 0, dataSize);
1117 memOrd = AArch64isa::kMoRelease;
1118 isVolStore = false;
1119 }
1120 }
1121
1122 memOpnd = memOpnd->IsOffsetMisaligned(dataSize) ? &ConstraintOffsetToSafeRegion(dataSize, *memOpnd) : memOpnd;
1123 if (symbol->GetAsmAttr() != UStrIdx(0) && symbol->GetStorageClass() != kScPstatic &&
1124 symbol->GetStorageClass() != kScFstatic) {
1125 std::string regDesp = GlobalTables::GetUStrTable().GetStringFromStrIdx(symbol->GetAsmAttr());
1126 RegOperand &specifiedOpnd = GetOrCreatePhysicalRegisterOperand(regDesp);
1127 SelectCopy(specifiedOpnd, type->GetPrimType(), opnd0, rhsPType);
1128 } else if (memOrd == AArch64isa::kMoNone) {
1129 mOp = PickStInsn(GetPrimTypeBitSize(ptyp), ptyp);
1130 Insn &insn = GetInsnBuilder()->BuildInsn(mOp, stOpnd, *memOpnd);
1131 if (GetCG()->GenerateVerboseCG()) {
1132 insn.SetComment(GenerateMemOpndVerbose(*memOpnd));
1133 }
1134 GetCurBB()->AppendInsn(insn);
1135 } else {
1136 AArch64CGFunc::SelectStoreRelease(*memOpnd, ptyp, stOpnd, ptyp, memOrd, true);
1137 }
1138 }
1139
SelectDassignoff(DassignoffNode & stmt,Operand & opnd0)1140 void AArch64CGFunc::SelectDassignoff(DassignoffNode &stmt, Operand &opnd0)
1141 {
1142 MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(stmt.stIdx);
1143 int64 offset = stmt.offset;
1144 uint32 size = GetPrimTypeSize(stmt.GetPrimType()) * k8ByteSize;
1145 MOperator mOp = (size == k16BitSize)
1146 ? MOP_wstrh
1147 : ((size == k32BitSize) ? MOP_wstr : ((size == k64BitSize) ? MOP_xstr : MOP_undef));
1148 CHECK_FATAL(mOp != MOP_undef, "illegal size for dassignoff");
1149 MemOperand *memOpnd = &GetOrCreateMemOpnd(*symbol, offset, size);
1150 if ((memOpnd->GetMemVaryType() == kNotVary) &&
1151 (IsImmediateOffsetOutOfRange(*memOpnd, size) || (offset % k8BitSize != 0))) {
1152 memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, size);
1153 }
1154 Operand &stOpnd = LoadIntoRegister(opnd0, true, size, false);
1155 memOpnd = memOpnd->IsOffsetMisaligned(size) ? &ConstraintOffsetToSafeRegion(size, *memOpnd) : memOpnd;
1156 Insn &insn = GetInsnBuilder()->BuildInsn(mOp, stOpnd, *memOpnd);
1157 GetCurBB()->AppendInsn(insn);
1158 }
1159
SelectAssertNull(UnaryStmtNode & stmt)1160 void AArch64CGFunc::SelectAssertNull(UnaryStmtNode &stmt)
1161 {
1162 Operand *opnd0 = HandleExpr(stmt, *stmt.Opnd(0));
1163 RegOperand &baseReg = LoadIntoRegister(*opnd0, PTY_a64);
1164 auto &zwr = GetZeroOpnd(k32BitSize);
1165 auto &mem = CreateMemOpnd(baseReg, 0, k32BitSize);
1166 Insn &loadRef = GetInsnBuilder()->BuildInsn(MOP_wldr, zwr, mem);
1167 loadRef.SetDoNotRemove(true);
1168 if (GetCG()->GenerateVerboseCG()) {
1169 loadRef.SetComment("null pointer check");
1170 }
1171 GetCurBB()->AppendInsn(loadRef);
1172 }
1173
SelectAbort()1174 void AArch64CGFunc::SelectAbort()
1175 {
1176 RegOperand &inOpnd = GetOrCreatePhysicalRegisterOperand(R16, k64BitSize, kRegTyInt);
1177 auto &mem = CreateMemOpnd(inOpnd, 0, k64BitSize);
1178 Insn &movXzr = GetInsnBuilder()->BuildInsn(MOP_xmovri64, inOpnd, CreateImmOperand(0, k64BitSize, false));
1179 Insn &loadRef = GetInsnBuilder()->BuildInsn(MOP_wldr, GetZeroOpnd(k64BitSize), mem);
1180 loadRef.SetDoNotRemove(true);
1181 movXzr.SetDoNotRemove(true);
1182 GetCurBB()->AppendInsn(movXzr);
1183 GetCurBB()->AppendInsn(loadRef);
1184 }
1185
GetRegPrefixFromPrimType(PrimType pType,uint32 size,const std::string & constraint)1186 static std::string GetRegPrefixFromPrimType(PrimType pType, uint32 size, const std::string &constraint)
1187 {
1188 std::string regPrefix = "";
1189 /* memory access check */
1190 if (constraint.find("m") != std::string::npos || constraint.find("Q") != std::string::npos) {
1191 regPrefix += "[";
1192 }
1193 if (IsPrimitiveVector(pType)) {
1194 regPrefix += "v";
1195 } else if (IsPrimitiveInteger(pType)) {
1196 if (size == k32BitSize) {
1197 regPrefix += "w";
1198 } else {
1199 regPrefix += "x";
1200 }
1201 } else {
1202 if (size == k32BitSize) {
1203 regPrefix += "s";
1204 } else {
1205 regPrefix += "d";
1206 }
1207 }
1208 return regPrefix;
1209 }
1210
SelectAsm(AsmNode & node)1211 void AArch64CGFunc::SelectAsm(AsmNode &node)
1212 {
1213 SetHasAsm();
1214 if (Globals::GetInstance()->GetOptimLevel() > CGOptions::kLevel0) {
1215 if (GetCG()->GetCGOptions().DoLinearScanRegisterAllocation()) {
1216 LogInfo::MapleLogger() << "Using coloring RA\n";
1217 const_cast<CGOptions &>(GetCG()->GetCGOptions()).SetOption(CGOptions::kDoColorRegAlloc);
1218 const_cast<CGOptions &>(GetCG()->GetCGOptions()).ClearOption(CGOptions::kDoLinearScanRegAlloc);
1219 }
1220 }
1221 Operand *asmString = &CreateStringOperand(node.asmString);
1222 ListOperand *listInputOpnd = CreateListOpnd(*GetFuncScopeAllocator());
1223 ListOperand *listOutputOpnd = CreateListOpnd(*GetFuncScopeAllocator());
1224 ListOperand *listClobber = CreateListOpnd(*GetFuncScopeAllocator());
1225 ListConstraintOperand *listInConstraint = memPool->New<ListConstraintOperand>(*GetFuncScopeAllocator());
1226 ListConstraintOperand *listOutConstraint = memPool->New<ListConstraintOperand>(*GetFuncScopeAllocator());
1227 ListConstraintOperand *listInRegPrefix = memPool->New<ListConstraintOperand>(*GetFuncScopeAllocator());
1228 ListConstraintOperand *listOutRegPrefix = memPool->New<ListConstraintOperand>(*GetFuncScopeAllocator());
1229 std::list<std::pair<Operand *, PrimType>> rPlusOpnd;
1230 bool noReplacement = false;
1231 if (node.asmString.find('$') == std::string::npos) {
1232 /* no replacements */
1233 noReplacement = true;
1234 }
1235 /* input constraints should be processed before OP_asm instruction */
1236 for (size_t i = 0; i < node.numOpnds; ++i) {
1237 /* process input constraint */
1238 std::string str = GlobalTables::GetUStrTable().GetStringFromStrIdx(node.inputConstraints[i]);
1239 bool isOutputTempNode = false;
1240 if (str[0] == '+') {
1241 isOutputTempNode = true;
1242 }
1243 listInConstraint->stringList.push_back(static_cast<StringOperand *>(&CreateStringOperand(str)));
1244 /* process input operands */
1245 switch (node.Opnd(i)->op) {
1246 case OP_dread: {
1247 DreadNode &dread = static_cast<DreadNode &>(*node.Opnd(i));
1248 Operand *inOpnd = SelectDread(node, dread);
1249 PrimType pType = dread.GetPrimType();
1250 listInputOpnd->PushOpnd(static_cast<RegOperand &>(*inOpnd));
1251 listInRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1252 &CreateStringOperand(GetRegPrefixFromPrimType(pType, inOpnd->GetSize(), str))));
1253 if (isOutputTempNode) {
1254 rPlusOpnd.emplace_back(std::make_pair(inOpnd, pType));
1255 }
1256 break;
1257 }
1258 case OP_addrof: {
1259 auto &addrofNode = static_cast<AddrofNode &>(*node.Opnd(i));
1260 Operand *inOpnd = SelectAddrof(addrofNode, node);
1261 listInputOpnd->PushOpnd(static_cast<RegOperand &>(*inOpnd));
1262 PrimType pType = addrofNode.GetPrimType();
1263 listInRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1264 &CreateStringOperand(GetRegPrefixFromPrimType(pType, inOpnd->GetSize(), str))));
1265 if (isOutputTempNode) {
1266 rPlusOpnd.emplace_back(std::make_pair(inOpnd, pType));
1267 }
1268 break;
1269 }
1270 case OP_constval: {
1271 CHECK_FATAL(!isOutputTempNode, "Unexpect");
1272 auto &constNode = static_cast<ConstvalNode &>(*node.Opnd(i));
1273 CHECK_FATAL(constNode.GetConstVal()->GetKind() == kConstInt,
1274 "expect MIRIntConst does not support float yet");
1275 MIRIntConst *mirIntConst = safe_cast<MIRIntConst>(constNode.GetConstVal());
1276 CHECK_FATAL(mirIntConst != nullptr, "just checking");
1277 int64 scale = mirIntConst->GetExtValue();
1278 if (str.find("r") != std::string::npos) {
1279 bool isSigned = scale < 0;
1280 ImmOperand &immOpnd = CreateImmOperand(scale, k64BitSize, isSigned);
1281 /* set default type as a 64 bit reg */
1282 PrimType pty = isSigned ? PTY_i64 : PTY_u64;
1283 auto &tempReg = static_cast<Operand &>(CreateRegisterOperandOfType(pty));
1284 SelectCopy(tempReg, pty, immOpnd, isSigned ? PTY_i64 : PTY_u64);
1285 listInputOpnd->PushOpnd(static_cast<RegOperand &>(tempReg));
1286 listInRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1287 &CreateStringOperand(GetRegPrefixFromPrimType(pty, tempReg.GetSize(), str))));
1288 } else {
1289 RegOperand &inOpnd = GetOrCreatePhysicalRegisterOperand(RZR, k64BitSize, kRegTyInt);
1290 listInputOpnd->PushOpnd(static_cast<RegOperand &>(inOpnd));
1291
1292 listInRegPrefix->stringList.push_back(
1293 static_cast<StringOperand *>(&CreateStringOperand("i" + std::to_string(scale))));
1294 }
1295 break;
1296 }
1297 case OP_regread: {
1298 auto ®readNode = static_cast<RegreadNode &>(*node.Opnd(i));
1299 PregIdx pregIdx = regreadNode.GetRegIdx();
1300 RegOperand &inOpnd = GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx));
1301 listInputOpnd->PushOpnd(static_cast<RegOperand &>(inOpnd));
1302 MIRPreg *preg = GetFunction().GetPregTab()->PregFromPregIdx(pregIdx);
1303 PrimType pType = preg->GetPrimType();
1304 listInRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1305 &CreateStringOperand(GetRegPrefixFromPrimType(pType, inOpnd.GetSize(), str))));
1306 if (isOutputTempNode) {
1307 rPlusOpnd.emplace_back(std::make_pair(&static_cast<Operand &>(inOpnd), pType));
1308 }
1309 break;
1310 }
1311 default:
1312 CHECK_FATAL(0, "Inline asm input expression not handled");
1313 }
1314 }
1315 std::vector<Operand *> intrnOpnds;
1316 intrnOpnds.emplace_back(asmString);
1317 intrnOpnds.emplace_back(listOutputOpnd);
1318 intrnOpnds.emplace_back(listClobber);
1319 intrnOpnds.emplace_back(listInputOpnd);
1320 intrnOpnds.emplace_back(listOutConstraint);
1321 intrnOpnds.emplace_back(listInConstraint);
1322 intrnOpnds.emplace_back(listOutRegPrefix);
1323 intrnOpnds.emplace_back(listInRegPrefix);
1324 Insn *asmInsn = &GetInsnBuilder()->BuildInsn(MOP_asm, intrnOpnds);
1325 GetCurBB()->AppendInsn(*asmInsn);
1326
1327 /* process listOutputOpnd */
1328 for (size_t i = 0; i < node.asmOutputs.size(); ++i) {
1329 bool isOutputTempNode = false;
1330 RegOperand *rPOpnd = nullptr;
1331 /* process output constraint */
1332 std::string str = GlobalTables::GetUStrTable().GetStringFromStrIdx(node.outputConstraints[i]);
1333
1334 listOutConstraint->stringList.push_back(static_cast<StringOperand *>(&CreateStringOperand(str)));
1335 if (str[0] == '+') {
1336 CHECK_FATAL(!rPlusOpnd.empty(), "Need r+ operand");
1337 rPOpnd = static_cast<RegOperand *>((rPlusOpnd.begin()->first));
1338 listOutputOpnd->PushOpnd(*rPOpnd);
1339 listOutRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1340 &CreateStringOperand(GetRegPrefixFromPrimType(rPlusOpnd.begin()->second, rPOpnd->GetSize(), str))));
1341 if (!rPlusOpnd.empty()) {
1342 rPlusOpnd.pop_front();
1343 }
1344 isOutputTempNode = true;
1345 }
1346 if (str.find("Q") != std::string::npos || str.find("m") != std::string::npos) {
1347 continue;
1348 }
1349 /* process output operands */
1350 StIdx stIdx = node.asmOutputs[i].first;
1351 RegFieldPair regFieldPair = node.asmOutputs[i].second;
1352 if (regFieldPair.IsReg()) {
1353 PregIdx pregIdx = static_cast<PregIdx>(regFieldPair.GetPregIdx());
1354 MIRPreg *mirPreg = mirModule.CurFunction()->GetPregTab()->PregFromPregIdx(pregIdx);
1355 RegOperand *outOpnd = isOutputTempNode
1356 ? rPOpnd
1357 : &GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx));
1358 PrimType srcType = mirPreg->GetPrimType();
1359 PrimType destType = srcType;
1360 if (GetPrimTypeBitSize(destType) < k32BitSize) {
1361 destType = IsSignedInteger(destType) ? PTY_i32 : PTY_u32;
1362 }
1363 RegType rtype = GetRegTyFromPrimTy(srcType);
1364 RegOperand &opnd0 = isOutputTempNode
1365 ? GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx))
1366 : CreateVirtualRegisterOperand(NewVReg(rtype, GetPrimTypeSize(srcType)));
1367 SelectCopy(opnd0, destType, *outOpnd, srcType);
1368 if (!isOutputTempNode) {
1369 listOutputOpnd->PushOpnd(static_cast<RegOperand &>(*outOpnd));
1370 listOutRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1371 &CreateStringOperand(GetRegPrefixFromPrimType(srcType, outOpnd->GetSize(), str))));
1372 }
1373 } else {
1374 MIRSymbol *var;
1375 if (stIdx.IsGlobal()) {
1376 var = GlobalTables::GetGsymTable().GetSymbolFromStidx(stIdx.Idx());
1377 } else {
1378 var = mirModule.CurFunction()->GetSymbolTabItem(stIdx.Idx());
1379 }
1380 CHECK_FATAL(var != nullptr, "var should not be nullptr");
1381 if (!noReplacement || var->GetAsmAttr() != UStrIdx(0)) {
1382 RegOperand *outOpnd = nullptr;
1383 PrimType pty = GlobalTables::GetTypeTable().GetTypeTable().at(var->GetTyIdx())->GetPrimType();
1384 if (var->GetAsmAttr() != UStrIdx(0)) {
1385 std::string regDesp = GlobalTables::GetUStrTable().GetStringFromStrIdx(var->GetAsmAttr());
1386 outOpnd = &GetOrCreatePhysicalRegisterOperand(regDesp);
1387 } else {
1388 RegType rtype = GetRegTyFromPrimTy(pty);
1389 outOpnd =
1390 isOutputTempNode ? rPOpnd : &CreateVirtualRegisterOperand(NewVReg(rtype, GetPrimTypeSize(pty)));
1391 }
1392 SaveReturnValueInLocal(node.asmOutputs, i, PTY_a64, *outOpnd, node);
1393 if (!isOutputTempNode) {
1394 listOutputOpnd->PushOpnd(static_cast<RegOperand &>(*outOpnd));
1395 listOutRegPrefix->stringList.push_back(static_cast<StringOperand *>(
1396 &CreateStringOperand(GetRegPrefixFromPrimType(pty, outOpnd->GetSize(), str))));
1397 }
1398 }
1399 }
1400 }
1401 if (noReplacement) {
1402 return;
1403 }
1404
1405 /* process listClobber */
1406 for (size_t i = 0; i < node.clobberList.size(); ++i) {
1407 std::string str = GlobalTables::GetUStrTable().GetStringFromStrIdx(node.clobberList[i]);
1408 auto regno = static_cast<regno_t>(str[1] - '0');
1409 if (str[2] >= '0' && str[2] <= '9') { // if third char (index 2) is num, add to regno
1410 regno = regno * kDecimalMax + static_cast<uint32>((str[2] - '0'));
1411 }
1412 RegOperand *reg;
1413 switch (str[0]) {
1414 case 'w': {
1415 reg = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(regno + R0), k32BitSize, kRegTyInt);
1416 listClobber->PushOpnd(*reg);
1417 break;
1418 }
1419 case 'x': {
1420 reg = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(regno + R0), k64BitSize, kRegTyInt);
1421 listClobber->PushOpnd(*reg);
1422 break;
1423 }
1424 case 's': {
1425 reg = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(regno + V0), k32BitSize, kRegTyFloat);
1426 listClobber->PushOpnd(*reg);
1427 break;
1428 }
1429 case 'd': {
1430 reg = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(regno + V0), k64BitSize, kRegTyFloat);
1431 listClobber->PushOpnd(*reg);
1432 break;
1433 }
1434 case 'v': {
1435 reg = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(regno + V0), k64BitSize, kRegTyFloat);
1436 listClobber->PushOpnd(*reg);
1437 break;
1438 }
1439 case 'c': {
1440 asmInsn->SetAsmDefCondCode();
1441 break;
1442 }
1443 case 'm': {
1444 asmInsn->SetAsmModMem();
1445 break;
1446 }
1447 default:
1448 CHECK_FATAL(0, "Inline asm clobber list not handled");
1449 }
1450 }
1451 }
1452
SelectRegassign(RegassignNode & stmt,Operand & opnd0)1453 void AArch64CGFunc::SelectRegassign(RegassignNode &stmt, Operand &opnd0)
1454 {
1455 if (GetCG()->IsLmbc()) {
1456 PrimType lhsSize = stmt.GetPrimType();
1457 PrimType rhsSize = stmt.Opnd(0)->GetPrimType();
1458 if (lhsSize != rhsSize && stmt.Opnd(0)->GetOpCode() == OP_ireadoff) {
1459 Insn *prev = GetCurBB()->GetLastInsn();
1460 if (prev->GetMachineOpcode() == MOP_wldrsb || prev->GetMachineOpcode() == MOP_wldrsh) {
1461 opnd0.SetSize(GetPrimTypeBitSize(stmt.GetPrimType()));
1462 prev->SetMOP(AArch64CG::kMd[prev->GetMachineOpcode() == MOP_wldrsb ? MOP_xldrsb : MOP_xldrsh]);
1463 } else if (prev->GetMachineOpcode() == MOP_wldr && stmt.GetPrimType() == PTY_i64) {
1464 opnd0.SetSize(GetPrimTypeBitSize(stmt.GetPrimType()));
1465 prev->SetMOP(AArch64CG::kMd[MOP_xldrsw]);
1466 }
1467 }
1468 }
1469 RegOperand *regOpnd = nullptr;
1470 PregIdx pregIdx = stmt.GetRegIdx();
1471 if (IsSpecialPseudoRegister(pregIdx)) {
1472 if (GetCG()->IsLmbc() && stmt.GetPrimType() == PTY_agg) {
1473 if (static_cast<RegOperand &>(opnd0).IsOfIntClass()) {
1474 regOpnd = &GetOrCreateSpecialRegisterOperand(-pregIdx, PTY_i64);
1475 } else if (opnd0.GetSize() <= k4ByteSize) {
1476 regOpnd = &GetOrCreateSpecialRegisterOperand(-pregIdx, PTY_f32);
1477 } else {
1478 regOpnd = &GetOrCreateSpecialRegisterOperand(-pregIdx, PTY_f64);
1479 }
1480 } else {
1481 regOpnd = &GetOrCreateSpecialRegisterOperand(-pregIdx, stmt.GetPrimType());
1482 }
1483 } else {
1484 regOpnd = &GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx));
1485 }
1486 /* look at rhs */
1487 PrimType rhsType = stmt.Opnd(0)->GetPrimType();
1488 if (GetCG()->IsLmbc() && rhsType == PTY_agg) {
1489 /* This occurs when a call returns a small struct */
1490 /* The subtree should already taken care of the agg type that is in excess of 8 bytes */
1491 rhsType = PTY_i64;
1492 }
1493 PrimType dtype = rhsType;
1494 if (GetPrimTypeBitSize(dtype) < k32BitSize) {
1495 DEBUG_ASSERT(IsPrimitiveInteger(dtype), "");
1496 dtype = IsSignedInteger(dtype) ? PTY_i32 : PTY_u32;
1497 }
1498 DEBUG_ASSERT(regOpnd != nullptr, "null ptr check!");
1499 SelectCopy(*regOpnd, dtype, opnd0, rhsType);
1500 if (GetCG()->GenerateVerboseCG()) {
1501 if (GetCurBB()->GetLastInsn()) {
1502 GetCurBB()->GetLastInsn()->AppendComment(" regassign %" + std::to_string(pregIdx) + "; ");
1503 } else if (GetCurBB()->GetPrev()->GetLastInsn()) {
1504 GetCurBB()->GetPrev()->GetLastInsn()->AppendComment(" regassign %" + std::to_string(pregIdx) + "; ");
1505 }
1506 }
1507
1508 if ((Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) && (pregIdx >= 0)) {
1509 MemOperand *dest = GetPseudoRegisterSpillMemoryOperand(pregIdx);
1510 PrimType stype = GetTypeFromPseudoRegIdx(pregIdx);
1511 MIRPreg *preg = GetFunction().GetPregTab()->PregFromPregIdx(pregIdx);
1512 uint32 srcBitLength = GetPrimTypeSize(preg->GetPrimType()) * kBitsPerByte;
1513 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(srcBitLength, stype), *regOpnd, *dest));
1514 } else if (regOpnd->GetRegisterNumber() == R0 || regOpnd->GetRegisterNumber() == R1) {
1515 Insn &pseudo = GetInsnBuilder()->BuildInsn(MOP_pseudo_ret_int, *regOpnd);
1516 GetCurBB()->AppendInsn(pseudo);
1517 } else if (regOpnd->GetRegisterNumber() >= V0 && regOpnd->GetRegisterNumber() <= V3) {
1518 Insn &pseudo = GetInsnBuilder()->BuildInsn(MOP_pseudo_ret_float, *regOpnd);
1519 GetCurBB()->AppendInsn(pseudo);
1520 }
1521 if (stmt.GetPrimType() == PTY_ref) {
1522 regOpnd->SetIsReference(true);
1523 AddReferenceReg(regOpnd->GetRegisterNumber());
1524 }
1525 if (pregIdx > 0) {
1526 // special MIRPreg is not supported
1527 SetPregIdx2Opnd(pregIdx, *regOpnd);
1528 }
1529 const auto &derived2BaseRef = GetFunction().GetDerived2BaseRef();
1530 auto itr = derived2BaseRef.find(pregIdx);
1531 if (itr != derived2BaseRef.end()) {
1532 auto *opnd = GetOpndFromPregIdx(itr->first);
1533 CHECK_FATAL(opnd != nullptr, "pregIdx has not been assigned Operand");
1534 auto &derivedRegOpnd = static_cast<RegOperand &>(*opnd);
1535 opnd = GetOpndFromPregIdx(itr->second);
1536 CHECK_FATAL(opnd != nullptr, "pregIdx has not been assigned Operand");
1537 auto &baseRegOpnd = static_cast<RegOperand &>(*opnd);
1538 derivedRegOpnd.SetBaseRefOpnd(baseRegOpnd);
1539 }
1540 }
1541
FixLargeMemOpnd(MemOperand & memOpnd,uint32 align)1542 MemOperand *AArch64CGFunc::FixLargeMemOpnd(MemOperand &memOpnd, uint32 align)
1543 {
1544 MemOperand *lhsMemOpnd = &memOpnd;
1545 if ((lhsMemOpnd->GetMemVaryType() == kNotVary) && IsImmediateOffsetOutOfRange(*lhsMemOpnd, align * kBitsPerByte)) {
1546 RegOperand *addReg = &CreateRegisterOperandOfType(PTY_i64);
1547 lhsMemOpnd = &SplitOffsetWithAddInstruction(*lhsMemOpnd, align * k8BitSize, addReg->GetRegisterNumber());
1548 }
1549 return lhsMemOpnd;
1550 }
1551
FixLargeMemOpnd(MOperator mOp,MemOperand & memOpnd,uint32 dSize,uint32 opndIdx)1552 MemOperand *AArch64CGFunc::FixLargeMemOpnd(MOperator mOp, MemOperand &memOpnd, uint32 dSize, uint32 opndIdx)
1553 {
1554 auto *a64MemOpnd = &memOpnd;
1555 if ((a64MemOpnd->GetMemVaryType() == kNotVary) && !IsOperandImmValid(mOp, &memOpnd, opndIdx)) {
1556 if (opndIdx == kInsnSecondOpnd) {
1557 a64MemOpnd = &SplitOffsetWithAddInstruction(*a64MemOpnd, dSize);
1558 } else if (opndIdx == kInsnThirdOpnd) {
1559 a64MemOpnd =
1560 &SplitOffsetWithAddInstruction(*a64MemOpnd, dSize, AArch64reg::kRinvalid, false, nullptr, true);
1561 } else {
1562 CHECK_FATAL(false, "NYI");
1563 }
1564 }
1565 return a64MemOpnd;
1566 }
1567
GenLargeAggFormalMemOpnd(const MIRSymbol & sym,uint32 align,int64 offset,bool needLow12)1568 MemOperand *AArch64CGFunc::GenLargeAggFormalMemOpnd(const MIRSymbol &sym, uint32 align, int64 offset, bool needLow12)
1569 {
1570 MemOperand *memOpnd;
1571 if (sym.GetStorageClass() == kScFormal && GetBecommon().GetTypeSize(sym.GetTyIdx()) > k16ByteSize) {
1572 /* formal of size of greater than 16 is copied by the caller and the pointer to it is passed. */
1573 /* otherwise it is passed in register and is accessed directly. */
1574 memOpnd = &GetOrCreateMemOpnd(sym, 0, align * kBitsPerByte);
1575 RegOperand *vreg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
1576 Insn &ldInsn = GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_i64), *vreg, *memOpnd);
1577 GetCurBB()->AppendInsn(ldInsn);
1578 memOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, vreg, nullptr,
1579 &GetOrCreateOfstOpnd(static_cast<uint64>(offset), k32BitSize), nullptr);
1580 } else {
1581 memOpnd = &GetOrCreateMemOpnd(sym, offset, align * kBitsPerByte, false, needLow12);
1582 }
1583 return FixLargeMemOpnd(*memOpnd, align);
1584 }
1585
PrepareMemcpyParamOpnd(bool isLo12,const MIRSymbol & symbol,int64 offsetVal,RegOperand & BaseReg)1586 RegOperand *AArch64CGFunc::PrepareMemcpyParamOpnd(bool isLo12, const MIRSymbol &symbol, int64 offsetVal,
1587 RegOperand &BaseReg)
1588 {
1589 RegOperand *tgtAddr = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
1590 if (isLo12) {
1591 StImmOperand &stImm = CreateStImmOperand(symbol, 0, 0);
1592 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrpl12, *tgtAddr, BaseReg, stImm));
1593 } else {
1594 ImmOperand &imm = CreateImmOperand(offsetVal, k64BitSize, false);
1595 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *tgtAddr, BaseReg, imm));
1596 }
1597 return tgtAddr;
1598 }
1599
PrepareMemcpyParamOpnd(int64 offset,Operand & exprOpnd)1600 RegOperand *AArch64CGFunc::PrepareMemcpyParamOpnd(int64 offset, Operand &exprOpnd)
1601 {
1602 RegOperand *tgtAddr = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
1603 OfstOperand *ofstOpnd = &GetOrCreateOfstOpnd(static_cast<uint64>(offset), k32BitSize);
1604 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *tgtAddr, exprOpnd, *ofstOpnd));
1605 return tgtAddr;
1606 }
1607
PrepareMemcpyParamOpnd(uint64 copySize)1608 RegOperand *AArch64CGFunc::PrepareMemcpyParamOpnd(uint64 copySize)
1609 {
1610 RegOperand *vregMemcpySize = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
1611 ImmOperand *sizeOpnd = &CreateImmOperand(static_cast<int64>(copySize), k64BitSize, false);
1612 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wmovri32, *vregMemcpySize, *sizeOpnd));
1613 return vregMemcpySize;
1614 }
1615
AggtStrLdrInsert(bool bothUnion,Insn * lastStrLdr,Insn & newStrLdr)1616 Insn *AArch64CGFunc::AggtStrLdrInsert(bool bothUnion, Insn *lastStrLdr, Insn &newStrLdr)
1617 {
1618 if (bothUnion) {
1619 if (lastStrLdr == nullptr) {
1620 GetCurBB()->AppendInsn(newStrLdr);
1621 } else {
1622 GetCurBB()->InsertInsnAfter(*lastStrLdr, newStrLdr);
1623 }
1624 } else {
1625 GetCurBB()->AppendInsn(newStrLdr);
1626 }
1627 return &newStrLdr;
1628 }
1629
GetOrCreateLocator(CallConvKind cc)1630 CCImpl *AArch64CGFunc::GetOrCreateLocator(CallConvKind cc)
1631 {
1632 auto it = hashCCTable.find(cc);
1633 if (it != hashCCTable.end()) {
1634 it->second->Init();
1635 return it->second;
1636 }
1637 CCImpl *res = nullptr;
1638 if (cc == kCCall) {
1639 res = memPool->New<AArch64CallConvImpl>(GetBecommon());
1640 } else if (cc == kWebKitJS) {
1641 res = memPool->New<AArch64WebKitJSCC>(GetBecommon());
1642 } else if (cc == kGHC) {
1643 res = memPool->New<GHCCC>(GetBecommon());
1644 } else {
1645 CHECK_FATAL(false, "unsupported yet");
1646 }
1647 hashCCTable[cc] = res;
1648 return res;
1649 }
SelectAggDassign(DassignNode & stmt)1650 void AArch64CGFunc::SelectAggDassign(DassignNode &stmt)
1651 {
1652 MIRSymbol *lhsSymbol = GetFunction().GetLocalOrGlobalSymbol(stmt.GetStIdx());
1653 uint32 lhsOffset = 0;
1654 MIRType *lhsType = lhsSymbol->GetType();
1655 bool bothUnion = false;
1656 if (stmt.GetFieldID() != 0) {
1657 MIRStructType *structType = static_cast<MIRStructType *>(lhsSymbol->GetType());
1658 DEBUG_ASSERT(structType != nullptr, "SelectAggDassign: non-zero fieldID for non-structure");
1659 lhsType = structType->GetFieldType(stmt.GetFieldID());
1660 lhsOffset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, stmt.GetFieldID()).first);
1661 bothUnion |= (structType->GetKind() == kTypeUnion);
1662 }
1663 uint32 lhsAlign = GetBecommon().GetTypeAlign(lhsType->GetTypeIndex());
1664 uint64 lhsSize = GetBecommon().GetTypeSize(lhsType->GetTypeIndex());
1665
1666 uint32 rhsAlign;
1667 uint32 alignUsed;
1668 uint32 rhsOffset = 0;
1669 if (stmt.GetRHS()->GetOpCode() == OP_dread) {
1670 AddrofNode *rhsDread = static_cast<AddrofNode *>(stmt.GetRHS());
1671 MIRSymbol *rhsSymbol = GetFunction().GetLocalOrGlobalSymbol(rhsDread->GetStIdx());
1672 MIRType *rhsType = rhsSymbol->GetType();
1673 if (rhsDread->GetFieldID() != 0) {
1674 MIRStructType *structType = static_cast<MIRStructType *>(rhsSymbol->GetType());
1675 DEBUG_ASSERT(structType != nullptr, "SelectAggDassign: non-zero fieldID for non-structure");
1676 rhsType = structType->GetFieldType(rhsDread->GetFieldID());
1677 rhsOffset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, rhsDread->GetFieldID()).first);
1678 bothUnion &= (structType->GetKind() == kTypeUnion);
1679 }
1680 bothUnion &= (rhsSymbol == lhsSymbol);
1681 rhsAlign = GetBecommon().GetTypeAlign(rhsType->GetTypeIndex());
1682 alignUsed = std::min(lhsAlign, rhsAlign);
1683 DEBUG_ASSERT(alignUsed != 0, "expect non-zero");
1684 uint32 copySize = GetAggCopySize(lhsOffset, rhsOffset, alignUsed);
1685 MemOperand *rhsBaseMemOpnd;
1686 if (IsParamStructCopy(*rhsSymbol)) {
1687 rhsBaseMemOpnd = &LoadStructCopyBase(*rhsSymbol, rhsOffset, static_cast<int>(copySize * k8BitSize));
1688 } else {
1689 rhsBaseMemOpnd = &GetOrCreateMemOpnd(*rhsSymbol, rhsOffset, copySize * k8BitSize, false, true);
1690 rhsBaseMemOpnd = FixLargeMemOpnd(*rhsBaseMemOpnd, copySize);
1691 }
1692 RegOperand *rhsBaseReg = rhsBaseMemOpnd->GetBaseRegister();
1693 int64 rhsOffsetVal = rhsBaseMemOpnd->GetOffsetOperand()->GetValue();
1694 MemOperand *lhsBaseMemOpnd = GenLargeAggFormalMemOpnd(*lhsSymbol, copySize, lhsOffset, true);
1695 RegOperand *lhsBaseReg = lhsBaseMemOpnd->GetBaseRegister();
1696 int64 lhsOffsetVal = lhsBaseMemOpnd->GetOffsetOperand()->GetValue();
1697 bool rhsIsLo12 = (rhsBaseMemOpnd->GetAddrMode() == MemOperand::kAddrModeLo12Li);
1698 bool lhsIsLo12 = (lhsBaseMemOpnd->GetAddrMode() == MemOperand::kAddrModeLo12Li);
1699 if (lhsSize > kParmMemcpySize) {
1700 std::vector<Operand *> opndVec;
1701 RegOperand *regResult = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
1702 opndVec.push_back(regResult); /* result */
1703
1704 opndVec.push_back(PrepareMemcpyParamOpnd(lhsIsLo12, *lhsSymbol, lhsOffsetVal, *lhsBaseReg)); /* param 0 */
1705
1706 opndVec.push_back(PrepareMemcpyParamOpnd(rhsIsLo12, *rhsSymbol, rhsOffsetVal, *rhsBaseReg)); /* param 1 */
1707
1708 opndVec.push_back(PrepareMemcpyParamOpnd(lhsSize)); /* param 2 */
1709
1710 SelectLibCall("memcpy", opndVec, PTY_a64, PTY_a64);
1711
1712 return;
1713 }
1714 Insn *lastLdr = nullptr;
1715 Insn *lastStr = nullptr;
1716 for (uint32 i = 0; i < (lhsSize / copySize); i++) {
1717 uint64 rhsBaseOffset = i * copySize + static_cast<uint64>(rhsOffsetVal);
1718 uint64 lhsBaseOffset = i * copySize + static_cast<uint64>(lhsOffsetVal);
1719 MemOperand::AArch64AddressingMode addrMode =
1720 rhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
1721 MIRSymbol *sym = rhsIsLo12 ? rhsSymbol : nullptr;
1722 OfstOperand &rhsOfstOpnd = GetOrCreateOfstOpnd(rhsBaseOffset, k32BitSize);
1723 /* generate the load */
1724 MemOperand *rhsMemOpnd =
1725 &GetOrCreateMemOpnd(addrMode, copySize * k8BitSize, rhsBaseReg, nullptr, &rhsOfstOpnd, sym);
1726 /* generate the load */
1727 RegOperand &result = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, copySize)));
1728 bool doPair = (!rhsIsLo12 && !lhsIsLo12 && (copySize >= k4BitSize) && ((i + 1) < (lhsSize / copySize)));
1729 RegOperand *result1 = nullptr;
1730 Insn *newLoadInsn = nullptr;
1731 if (doPair) {
1732 MOperator mOpLDP = (copySize == k4BitSize) ? MOP_wldp : MOP_xldp;
1733 result1 = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, copySize)));
1734 rhsMemOpnd = FixLargeMemOpnd(mOpLDP, *rhsMemOpnd, copySize * k8BitSize, kInsnThirdOpnd);
1735 newLoadInsn = &GetInsnBuilder()->BuildInsn(mOpLDP, result, *result1, *rhsMemOpnd);
1736 } else {
1737 MOperator mOp = PickLdInsn(copySize * k8BitSize, PTY_u32);
1738 rhsMemOpnd = FixLargeMemOpnd(mOp, *rhsMemOpnd, copySize * k8BitSize, kInsnSecondOpnd);
1739 newLoadInsn = &GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd);
1740 }
1741 DEBUG_ASSERT(newLoadInsn != nullptr, "build load instruction failed in SelectAggDassign");
1742 lastLdr = AggtStrLdrInsert(bothUnion, lastLdr, *newLoadInsn);
1743 /* generate the store */
1744 OfstOperand &lhsOfstOpnd = GetOrCreateOfstOpnd(lhsBaseOffset, k32BitSize);
1745 addrMode = lhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
1746 sym = lhsIsLo12 ? lhsSymbol : nullptr;
1747 Insn *newStoreInsn = nullptr;
1748 MemOperand *lhsMemOpnd =
1749 &GetOrCreateMemOpnd(addrMode, copySize * k8BitSize, lhsBaseReg, nullptr, &lhsOfstOpnd, sym);
1750 if (doPair) {
1751 MOperator mOpSTP = (copySize == k4BitSize) ? MOP_wstp : MOP_xstp;
1752 lhsMemOpnd = FixLargeMemOpnd(mOpSTP, *lhsMemOpnd, copySize * k8BitSize, kInsnThirdOpnd);
1753 DEBUG_ASSERT(result1 != nullptr, "result1 should not be nullptr");
1754 newStoreInsn = &GetInsnBuilder()->BuildInsn(mOpSTP, result, *result1, *lhsMemOpnd);
1755 i++;
1756 } else {
1757 MOperator mOp = PickStInsn(copySize * k8BitSize, PTY_u32);
1758 lhsMemOpnd = FixLargeMemOpnd(mOp, *lhsMemOpnd, copySize * k8BitSize, kInsnSecondOpnd);
1759 newStoreInsn = &GetInsnBuilder()->BuildInsn(mOp, result, *lhsMemOpnd);
1760 }
1761 DEBUG_ASSERT(newStoreInsn != nullptr, "build store instruction failed in SelectAggDassign");
1762 lastStr = AggtStrLdrInsert(bothUnion, lastStr, *newStoreInsn);
1763 }
1764 /* take care of extra content at the end less than the unit */
1765 uint64 lhsSizeCovered = (lhsSize / copySize) * copySize;
1766 uint32 newAlignUsed = copySize;
1767 while (lhsSizeCovered < lhsSize) {
1768 newAlignUsed = newAlignUsed >> 1;
1769 CHECK_FATAL(newAlignUsed != 0, "expect non-zero");
1770 if ((lhsSizeCovered + newAlignUsed) > lhsSize) {
1771 continue;
1772 }
1773 /* generate the load */
1774 MemOperand *rhsMemOpnd;
1775 MemOperand::AArch64AddressingMode addrMode =
1776 rhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
1777 MIRSymbol *sym = rhsIsLo12 ? rhsSymbol : nullptr;
1778 OfstOperand &rhsOfstOpnd =
1779 GetOrCreateOfstOpnd(lhsSizeCovered + static_cast<uint64>(rhsOffsetVal), k32BitSize);
1780 rhsMemOpnd =
1781 &GetOrCreateMemOpnd(addrMode, newAlignUsed * k8BitSize, rhsBaseReg, nullptr, &rhsOfstOpnd, sym);
1782 rhsMemOpnd = FixLargeMemOpnd(*rhsMemOpnd, newAlignUsed);
1783 regno_t vRegNO = NewVReg(kRegTyInt, std::max(4u, newAlignUsed));
1784 RegOperand &result = CreateVirtualRegisterOperand(vRegNO);
1785 MOperator mOp = PickLdInsn(newAlignUsed * k8BitSize, PTY_u32);
1786 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd));
1787 /* generate the store */
1788 addrMode = lhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
1789 sym = lhsIsLo12 ? lhsSymbol : nullptr;
1790 OfstOperand &lhsOfstOpnd =
1791 GetOrCreateOfstOpnd(lhsSizeCovered + static_cast<uint64>(lhsOffsetVal), k32BitSize);
1792 MemOperand *lhsMemOpnd;
1793 lhsMemOpnd =
1794 &GetOrCreateMemOpnd(addrMode, newAlignUsed * k8BitSize, lhsBaseReg, nullptr, &lhsOfstOpnd, sym);
1795 lhsMemOpnd = FixLargeMemOpnd(*lhsMemOpnd, newAlignUsed);
1796 mOp = PickStInsn(newAlignUsed * k8BitSize, PTY_u32);
1797 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *lhsMemOpnd));
1798 lhsSizeCovered += newAlignUsed;
1799 }
1800 } else if (stmt.GetRHS()->GetOpCode() == OP_iread) {
1801 IreadNode *rhsIread = static_cast<IreadNode *>(stmt.GetRHS());
1802 RegOperand *addrOpnd = static_cast<RegOperand *>(HandleExpr(*rhsIread, *rhsIread->Opnd(0)));
1803 addrOpnd = &LoadIntoRegister(*addrOpnd, rhsIread->Opnd(0)->GetPrimType());
1804 MIRPtrType *rhsPointerType =
1805 static_cast<MIRPtrType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(rhsIread->GetTyIdx()));
1806 MIRType *rhsType = static_cast<MIRStructType *>(
1807 GlobalTables::GetTypeTable().GetTypeFromTyIdx(rhsPointerType->GetPointedTyIdx()));
1808 bool isRefField = false;
1809 if (rhsIread->GetFieldID() != 0) {
1810 MIRStructType *rhsStructType = static_cast<MIRStructType *>(rhsType);
1811 DEBUG_ASSERT(rhsStructType != nullptr, "SelectAggDassign: non-zero fieldID for non-structure");
1812 rhsType = rhsStructType->GetFieldType(rhsIread->GetFieldID());
1813 rhsOffset = static_cast<uint32>(GetBecommon().GetFieldOffset(*rhsStructType, rhsIread->GetFieldID()).first);
1814 isRefField = GetBecommon().IsRefField(*rhsStructType, rhsIread->GetFieldID());
1815 }
1816 rhsAlign = GetBecommon().GetTypeAlign(rhsType->GetTypeIndex());
1817 alignUsed = std::min(lhsAlign, rhsAlign);
1818 DEBUG_ASSERT(alignUsed != 0, "expect non-zero");
1819 uint32 copySize = GetAggCopySize(rhsOffset, lhsOffset, alignUsed);
1820 MemOperand *lhsBaseMemOpnd = GenLargeAggFormalMemOpnd(*lhsSymbol, copySize, lhsOffset, true);
1821 RegOperand *lhsBaseReg = lhsBaseMemOpnd->GetBaseRegister();
1822 int64 lhsOffsetVal = lhsBaseMemOpnd->GetOffsetOperand()->GetValue();
1823 bool lhsIsLo12 = (lhsBaseMemOpnd->GetAddrMode() == MemOperand::kAddrModeLo12Li);
1824 if (lhsSize > kParmMemcpySize) {
1825 std::vector<Operand *> opndVec;
1826 RegOperand *regResult = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
1827 opndVec.push_back(regResult); /* result */
1828
1829 opndVec.push_back(PrepareMemcpyParamOpnd(lhsIsLo12, *lhsSymbol, lhsOffsetVal, *lhsBaseReg)); /* param 0 */
1830
1831 opndVec.push_back(PrepareMemcpyParamOpnd(rhsOffset, *addrOpnd)); /* param 1 */
1832
1833 opndVec.push_back(PrepareMemcpyParamOpnd(lhsSize)); /* param 2 */
1834
1835 SelectLibCall("memcpy", opndVec, PTY_a64, PTY_a64);
1836
1837 return;
1838 }
1839 for (uint32 i = 0; i < (lhsSize / copySize); i++) {
1840 uint64 rhsBaseOffset = rhsOffset + i * copySize;
1841 uint64 lhsBaseOffset = static_cast<uint64>(lhsOffsetVal) + i * copySize;
1842 /* generate the load */
1843 OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(rhsBaseOffset, k32BitSize);
1844 MemOperand *rhsMemOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, copySize * k8BitSize, addrOpnd,
1845 nullptr, &ofstOpnd, nullptr);
1846 regno_t vRegNO = NewVReg(kRegTyInt, std::max(4u, copySize));
1847 RegOperand &result = CreateVirtualRegisterOperand(vRegNO);
1848 bool doPair = (!lhsIsLo12 && copySize >= k4BitSize) && ((i + 1) < (lhsSize / copySize));
1849 Insn *insn = nullptr;
1850 RegOperand *result1 = nullptr;
1851 if (doPair) {
1852 MOperator mOpLDP = (copySize == k4BitSize) ? MOP_wldp : MOP_xldp;
1853 regno_t vRegNO1 = NewVReg(kRegTyInt, std::max(4u, copySize));
1854 result1 = &CreateVirtualRegisterOperand(vRegNO1);
1855 rhsMemOpnd = FixLargeMemOpnd(mOpLDP, *rhsMemOpnd, copySize * k8BitSize, kInsnThirdOpnd);
1856 insn = &GetInsnBuilder()->BuildInsn(mOpLDP, result, *result1, *rhsMemOpnd);
1857 } else {
1858 MOperator mOp = PickLdInsn(copySize * k8BitSize, PTY_u32);
1859 rhsMemOpnd = FixLargeMemOpnd(mOp, *rhsMemOpnd, copySize * k8BitSize, kInsnSecondOpnd);
1860 insn = &GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd);
1861 }
1862 insn->MarkAsAccessRefField(isRefField);
1863 GetCurBB()->AppendInsn(*insn);
1864 /* generate the store */
1865 MemOperand::AArch64AddressingMode addrMode =
1866 lhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
1867 MIRSymbol *sym = lhsIsLo12 ? lhsSymbol : nullptr;
1868 OfstOperand &lhsOfstOpnd = GetOrCreateOfstOpnd(lhsBaseOffset, k32BitSize);
1869 MemOperand *lhsMemOpnd =
1870 &GetOrCreateMemOpnd(addrMode, copySize * k8BitSize, lhsBaseReg, nullptr, &lhsOfstOpnd, sym);
1871 if (doPair) {
1872 MOperator mOpSTP = (copySize == k4BitSize) ? MOP_wstp : MOP_xstp;
1873 lhsMemOpnd = FixLargeMemOpnd(mOpSTP, *lhsMemOpnd, copySize * k8BitSize, kInsnThirdOpnd);
1874 DEBUG_ASSERT(result1 != nullptr, "result1 should not be nullptr");
1875 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpSTP, result, *result1, *lhsMemOpnd));
1876 i++;
1877 } else {
1878 MOperator mOp = PickStInsn(copySize * k8BitSize, PTY_u32);
1879 lhsMemOpnd = FixLargeMemOpnd(mOp, *lhsMemOpnd, copySize * k8BitSize, kInsnSecondOpnd);
1880 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *lhsMemOpnd));
1881 }
1882 }
1883 /* take care of extra content at the end less than the unit of alignUsed */
1884 uint64 lhsSizeCovered = (lhsSize / copySize) * copySize;
1885 uint32 newAlignUsed = copySize;
1886 while (lhsSizeCovered < lhsSize) {
1887 newAlignUsed = newAlignUsed >> 1;
1888 CHECK_FATAL(newAlignUsed != 0, "expect non-zero");
1889 if ((lhsSizeCovered + newAlignUsed) > lhsSize) {
1890 continue;
1891 }
1892 /* generate the load */
1893 OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(rhsOffset + lhsSizeCovered, k32BitSize);
1894 MemOperand *rhsMemOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, newAlignUsed * k8BitSize, addrOpnd,
1895 nullptr, &ofstOpnd, nullptr);
1896 rhsMemOpnd = FixLargeMemOpnd(*rhsMemOpnd, newAlignUsed);
1897 RegOperand &result = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, newAlignUsed)));
1898 MOperator mOp = PickLdInsn(newAlignUsed * k8BitSize, PTY_u32);
1899 Insn &insn = GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd);
1900 insn.MarkAsAccessRefField(isRefField);
1901 GetCurBB()->AppendInsn(insn);
1902 /* generate the store */
1903 MemOperand::AArch64AddressingMode addrMode =
1904 lhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
1905 MIRSymbol *sym = lhsIsLo12 ? lhsSymbol : nullptr;
1906 OfstOperand &lhsOfstOpnd =
1907 GetOrCreateOfstOpnd(lhsSizeCovered + static_cast<uint64>(lhsOffsetVal), k32BitSize);
1908 MemOperand *lhsMemOpnd;
1909 lhsMemOpnd =
1910 &GetOrCreateMemOpnd(addrMode, newAlignUsed * k8BitSize, lhsBaseReg, nullptr, &lhsOfstOpnd, sym);
1911 lhsMemOpnd = FixLargeMemOpnd(*lhsMemOpnd, newAlignUsed);
1912 mOp = PickStInsn(newAlignUsed * k8BitSize, PTY_u32);
1913 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *lhsMemOpnd));
1914 lhsSizeCovered += newAlignUsed;
1915 }
1916 } else {
1917 DEBUG_ASSERT(stmt.GetRHS()->op == OP_regread, "SelectAggDassign: NYI");
1918 bool isRet = false;
1919 if (lhsType->GetKind() == kTypeStruct || lhsType->GetKind() == kTypeUnion) {
1920 RegreadNode *rhsregread = static_cast<RegreadNode *>(stmt.GetRHS());
1921 PregIdx pregIdx = rhsregread->GetRegIdx();
1922 if (IsSpecialPseudoRegister(pregIdx)) {
1923 if ((-pregIdx) == kSregRetval0) {
1924 CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
1925 AArch64CallConvImpl parmlocator(GetBecommon());
1926 CCLocInfo pLoc;
1927 PrimType retPtype;
1928 RegType regType;
1929 uint32 memSize;
1930 uint32 regSize;
1931 parmlocator.LocateRetVal(*lhsType, pLoc);
1932 AArch64reg r[kFourRegister];
1933 r[kFirstReg] = static_cast<AArch64reg>(pLoc.reg0);
1934 r[kSecondReg] = static_cast<AArch64reg>(pLoc.reg1);
1935 r[kThirdReg] = static_cast<AArch64reg>(pLoc.reg2);
1936 r[kFourthReg] = static_cast<AArch64reg>(pLoc.reg3);
1937 if (pLoc.numFpPureRegs) {
1938 regSize = (pLoc.fpSize == k4ByteSize) ? k32BitSize : k64BitSize;
1939 memSize = pLoc.fpSize;
1940 retPtype = (pLoc.fpSize == k4ByteSize) ? PTY_f32 : PTY_f64;
1941 regType = kRegTyFloat;
1942 } else {
1943 regSize = k64BitSize;
1944 memSize = k8BitSize;
1945 retPtype = PTY_u64;
1946 regType = kRegTyInt;
1947 }
1948 for (uint32 i = 0; i < kFourRegister; ++i) {
1949 if (r[i] == kRinvalid) {
1950 break;
1951 }
1952 RegOperand &parm = GetOrCreatePhysicalRegisterOperand(r[i], regSize, regType);
1953 Operand &mOpnd = GetOrCreateMemOpnd(*lhsSymbol, memSize * i, regSize);
1954 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(regSize, retPtype), parm, mOpnd));
1955 }
1956 isRet = true;
1957 }
1958 }
1959 }
1960 CHECK_FATAL(isRet, "SelectAggDassign: NYI");
1961 }
1962 }
1963
GetPointedToType(const MIRPtrType & pointerType)1964 static MIRType *GetPointedToType(const MIRPtrType &pointerType)
1965 {
1966 MIRType *aType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(pointerType.GetPointedTyIdx());
1967 if (aType->GetKind() == kTypeArray) {
1968 MIRArrayType *arrayType = static_cast<MIRArrayType *>(aType);
1969 return GlobalTables::GetTypeTable().GetTypeFromTyIdx(arrayType->GetElemTyIdx());
1970 }
1971 if (aType->GetKind() == kTypeFArray || aType->GetKind() == kTypeJArray) {
1972 MIRFarrayType *farrayType = static_cast<MIRFarrayType *>(aType);
1973 return GlobalTables::GetTypeTable().GetTypeFromTyIdx(farrayType->GetElemTyIdx());
1974 }
1975 return aType;
1976 }
1977
SelectIassign(IassignNode & stmt)1978 void AArch64CGFunc::SelectIassign(IassignNode &stmt)
1979 {
1980 int32 offset = 0;
1981 MIRPtrType *pointerType = static_cast<MIRPtrType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(stmt.GetTyIdx()));
1982 DEBUG_ASSERT(pointerType != nullptr, "expect a pointer type at iassign node");
1983 MIRType *pointedType = nullptr;
1984 bool isRefField = false;
1985 AArch64isa::MemoryOrdering memOrd = AArch64isa::kMoNone;
1986
1987 if (stmt.GetFieldID() != 0) {
1988 MIRType *pointedTy = GlobalTables::GetTypeTable().GetTypeFromTyIdx(pointerType->GetPointedTyIdx());
1989 MIRStructType *structType = nullptr;
1990 if (pointedTy->GetKind() != kTypeJArray) {
1991 structType = static_cast<MIRStructType *>(pointedTy);
1992 } else {
1993 /* it's a Jarray type. using it's parent's field info: java.lang.Object */
1994 structType = static_cast<MIRJarrayType *>(pointedTy)->GetParentType();
1995 }
1996 DEBUG_ASSERT(structType != nullptr, "SelectIassign: non-zero fieldID for non-structure");
1997 pointedType = structType->GetFieldType(stmt.GetFieldID());
1998 offset = GetBecommon().GetFieldOffset(*structType, stmt.GetFieldID()).first;
1999 isRefField = GetBecommon().IsRefField(*structType, stmt.GetFieldID());
2000 } else {
2001 pointedType = GetPointedToType(*pointerType);
2002 if (GetFunction().IsJava() && (pointedType->GetKind() == kTypePointer)) {
2003 MIRType *nextPointedType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(
2004 static_cast<MIRPtrType *>(pointedType)->GetPointedTyIdx());
2005 if (nextPointedType->GetKind() != kTypeScalar) {
2006 isRefField = true; /* write into an object array or a high-dimensional array */
2007 }
2008 }
2009 }
2010
2011 PrimType styp = stmt.GetRHS()->GetPrimType();
2012 Operand *valOpnd = HandleExpr(stmt, *stmt.GetRHS());
2013 Operand &srcOpnd = LoadIntoRegister(*valOpnd, (IsPrimitiveInteger(styp) || IsPrimitiveVectorInteger(styp)),
2014 GetPrimTypeBitSize(styp));
2015
2016 PrimType destType = pointedType->GetPrimType();
2017 if (destType == PTY_agg) {
2018 destType = PTY_a64;
2019 }
2020 if (IsPrimitiveVector(styp)) { /* a vector type */
2021 destType = styp;
2022 }
2023 DEBUG_ASSERT(stmt.Opnd(0) != nullptr, "null ptr check");
2024 MemOperand &memOpnd = CreateMemOpnd(destType, stmt, *stmt.Opnd(0), offset);
2025 auto dataSize = GetPrimTypeBitSize(destType);
2026 memOpnd = memOpnd.IsOffsetMisaligned(dataSize) ? ConstraintOffsetToSafeRegion(dataSize, memOpnd) : memOpnd;
2027 if (isVolStore && memOpnd.GetAddrMode() == MemOperand::kAddrModeBOi) {
2028 memOrd = AArch64isa::kMoRelease;
2029 isVolStore = false;
2030 }
2031
2032 if (memOrd == AArch64isa::kMoNone) {
2033 SelectCopy(memOpnd, destType, srcOpnd, destType);
2034 } else {
2035 AArch64CGFunc::SelectStoreRelease(memOpnd, destType, srcOpnd, destType, memOrd, false);
2036 }
2037 GetCurBB()->GetLastInsn()->MarkAsAccessRefField(isRefField);
2038 }
2039
SelectIassignoff(IassignoffNode & stmt)2040 void AArch64CGFunc::SelectIassignoff(IassignoffNode &stmt)
2041 {
2042 int32 offset = stmt.GetOffset();
2043 PrimType destType = stmt.GetPrimType();
2044
2045 MemOperand &memOpnd = CreateMemOpnd(destType, stmt, *stmt.GetBOpnd(0), offset);
2046 auto dataSize = GetPrimTypeBitSize(destType);
2047 memOpnd = memOpnd.IsOffsetMisaligned(dataSize) ? ConstraintOffsetToSafeRegion(dataSize, memOpnd) : memOpnd;
2048 Operand *valOpnd = HandleExpr(stmt, *stmt.GetBOpnd(1));
2049 Operand &srcOpnd = LoadIntoRegister(*valOpnd, true, GetPrimTypeBitSize(destType));
2050 SelectCopy(memOpnd, destType, srcOpnd, destType);
2051 }
2052
GenLmbcFpMemOperand(int32 offset,uint32 byteSize,AArch64reg baseRegno)2053 MemOperand *AArch64CGFunc::GenLmbcFpMemOperand(int32 offset, uint32 byteSize, AArch64reg baseRegno)
2054 {
2055 MemOperand *memOpnd;
2056 RegOperand *rfp = &GetOrCreatePhysicalRegisterOperand(baseRegno, k64BitSize, kRegTyInt);
2057 uint32 bitlen = byteSize * kBitsPerByte;
2058 if (offset < 0 && offset < kNegative256BitSize) {
2059 RegOperand *baseOpnd = &CreateRegisterOperandOfType(PTY_a64);
2060 ImmOperand &immOpnd = CreateImmOperand(offset, k32BitSize, true);
2061 Insn &addInsn = GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *baseOpnd, *rfp, immOpnd);
2062 GetCurBB()->AppendInsn(addInsn);
2063 OfstOperand *offsetOpnd = &CreateOfstOpnd(0, k32BitSize);
2064 memOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, bitlen, baseOpnd, nullptr, offsetOpnd, nullptr);
2065 } else {
2066 OfstOperand *offsetOpnd = &CreateOfstOpnd(static_cast<uint64>(static_cast<int64>(offset)), k32BitSize);
2067 memOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, bitlen, rfp, nullptr, offsetOpnd, nullptr);
2068 }
2069 memOpnd->SetStackMem(true);
2070 return memOpnd;
2071 }
2072
SelectIassignfpoff(IassignFPoffNode & stmt,Operand & opnd)2073 void AArch64CGFunc::SelectIassignfpoff(IassignFPoffNode &stmt, Operand &opnd)
2074 {
2075 int32 offset = stmt.GetOffset();
2076 PrimType primType = stmt.GetPrimType();
2077 MIRType *rType = GetLmbcCallReturnType();
2078 bool isPureFpStruct = false;
2079 uint32 numRegs = 0;
2080 if (rType && rType->GetPrimType() == PTY_agg && opnd.IsRegister() &&
2081 static_cast<RegOperand &>(opnd).IsPhysicalRegister()) {
2082 CHECK_FATAL(rType->GetSize() <= k16BitSize, "SelectIassignfpoff invalid agg size");
2083 uint32 fpSize;
2084 numRegs = FloatParamRegRequired(static_cast<MIRStructType *>(rType), fpSize);
2085 if (numRegs) {
2086 primType = (fpSize == k4ByteSize) ? PTY_f32 : PTY_f64;
2087 isPureFpStruct = true;
2088 }
2089 }
2090 uint32 byteSize = GetPrimTypeSize(primType);
2091 uint32 bitlen = byteSize * kBitsPerByte;
2092 if (isPureFpStruct) {
2093 for (uint32 i = 0; i < numRegs; ++i) {
2094 MemOperand *memOpnd = GenLmbcFpMemOperand(offset + static_cast<int32>(i * byteSize), byteSize);
2095 RegOperand &srcOpnd = GetOrCreatePhysicalRegisterOperand(AArch64reg(V0 + i), bitlen, kRegTyFloat);
2096 MOperator mOp = PickStInsn(bitlen, primType);
2097 Insn &store = GetInsnBuilder()->BuildInsn(mOp, srcOpnd, *memOpnd);
2098 GetCurBB()->AppendInsn(store);
2099 }
2100 } else {
2101 Operand &srcOpnd = LoadIntoRegister(opnd, primType);
2102 MemOperand *memOpnd = GenLmbcFpMemOperand(offset, byteSize);
2103 MOperator mOp = PickStInsn(bitlen, primType);
2104 Insn &store = GetInsnBuilder()->BuildInsn(mOp, srcOpnd, *memOpnd);
2105 GetCurBB()->AppendInsn(store);
2106 }
2107 }
2108
2109 /* Load and assign to a new register. To be moved to the correct call register OR stack
2110 location in LmbcSelectParmList */
SelectIassignspoff(PrimType pTy,int32 offset,Operand & opnd)2111 void AArch64CGFunc::SelectIassignspoff(PrimType pTy, int32 offset, Operand &opnd)
2112 {
2113 if (GetLmbcArgInfo() == nullptr) {
2114 LmbcArgInfo *p = memPool->New<LmbcArgInfo>(*GetFuncScopeAllocator());
2115 SetLmbcArgInfo(p);
2116 }
2117 uint32 byteLen = GetPrimTypeSize(pTy);
2118 uint32 bitLen = byteLen * kBitsPerByte;
2119 RegType regTy = GetRegTyFromPrimTy(pTy);
2120 int32 curRegArgs = GetLmbcArgsInRegs(regTy);
2121 if (curRegArgs < static_cast<int32>(k8ByteSize)) {
2122 RegOperand *res = &CreateVirtualRegisterOperand(NewVReg(regTy, byteLen));
2123 SelectCopy(*res, pTy, opnd, pTy);
2124 SetLmbcArgInfo(res, pTy, offset, 1);
2125 } else {
2126 /* Move into allocated space */
2127 Operand &memOpd = CreateMemOpnd(RSP, offset, byteLen);
2128 Operand ® = LoadIntoRegister(opnd, pTy);
2129 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(bitLen, pTy), reg, memOpd));
2130 }
2131 IncLmbcArgsInRegs(regTy); /* num of args in registers */
2132 IncLmbcTotalArgs(); /* num of args */
2133 }
2134
2135 /* Search for CALL/ICALL/ICALLPROTO node, must be called from a blkassignoff node */
LmbcGetAggTyFromCallSite(StmtNode * stmt,std::vector<TyIdx> ** parmList) const2136 MIRType *AArch64CGFunc::LmbcGetAggTyFromCallSite(StmtNode *stmt, std::vector<TyIdx> **parmList) const
2137 {
2138 for (; stmt != nullptr; stmt = stmt->GetNext()) {
2139 if (stmt->GetOpCode() == OP_call || stmt->GetOpCode() == OP_icallproto) {
2140 break;
2141 }
2142 }
2143 CHECK_FATAL(stmt && (stmt->GetOpCode() == OP_call || stmt->GetOpCode() == OP_icallproto),
2144 "blkassign sp not followed by call");
2145 uint32 nargs = GetLmbcTotalArgs();
2146 MIRType *ty = nullptr;
2147 if (stmt->GetOpCode() == OP_call) {
2148 CallNode *callNode = static_cast<CallNode *>(stmt);
2149 MIRFunction *fn = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(callNode->GetPUIdx());
2150 if (fn->GetFormalCount() > 0) {
2151 ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(fn->GetFormalDefVec()[nargs].formalTyIdx);
2152 }
2153 *parmList = &fn->GetParamTypes();
2154 // would return null if the actual parameter is bogus
2155 } else if (stmt->GetOpCode() == OP_icallproto) {
2156 IcallNode *icallproto = static_cast<IcallNode *>(stmt);
2157 MIRType *type = GlobalTables::GetTypeTable().GetTypeFromTyIdx(icallproto->GetRetTyIdx());
2158 MIRFuncType *fType = static_cast<MIRFuncType *>(type);
2159 ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(fType->GetNthParamType(nargs));
2160 *parmList = &fType->GetParamTypeList();
2161 } else {
2162 CHECK_FATAL(stmt->GetOpCode() == OP_icallproto, "LmbcGetAggTyFromCallSite:: unexpected call operator");
2163 }
2164 return ty;
2165 }
2166
2167 /* return true if blkassignoff for return, false otherwise */
LmbcSmallAggForRet(const BlkassignoffNode & bNode,const Operand * src)2168 bool AArch64CGFunc::LmbcSmallAggForRet(const BlkassignoffNode &bNode, const Operand *src)
2169 {
2170 PrimType pTy;
2171 uint32 size = 0;
2172 AArch64reg regno = static_cast<AArch64reg>(static_cast<const RegOperand *>(src)->GetRegisterNumber());
2173 MIRFunction *func = &GetFunction();
2174
2175 if (func->IsReturnStruct()) {
2176 /* This blkassignoff is for struct return? */
2177 uint32 loadSize;
2178 uint32 numRegs = 0;
2179 if (bNode.GetNext()->GetOpCode() == OP_return) {
2180 MIRStructType *ty = static_cast<MIRStructType *>(
2181 GlobalTables::GetTypeTable().GetTypeFromTyIdx(func->GetFuncRetStructTyIdx()));
2182 uint32 fpregs = FloatParamRegRequired(ty, size);
2183 if (fpregs > 0) {
2184 /* pure floating point in agg */
2185 numRegs = fpregs;
2186 pTy = (size == k4ByteSize) ? PTY_f32 : PTY_f64;
2187 loadSize = GetPrimTypeSize(pTy) * kBitsPerByte;
2188 for (uint32 i = 0; i < fpregs; i++) {
2189 int32 s = (i == 0) ? 0 : static_cast<int32>(i * size);
2190 MemOperand &mem = CreateMemOpnd(regno, s, size * kBitsPerByte);
2191 AArch64reg reg = static_cast<AArch64reg>(V0 + i);
2192 RegOperand *res = &GetOrCreatePhysicalRegisterOperand(reg, loadSize, kRegTyFloat);
2193 SelectCopy(*res, pTy, mem, pTy);
2194 }
2195 } else {
2196 /* int/float mixed */
2197 numRegs = kRegNum2;
2198 pTy = PTY_i64;
2199 size = k4ByteSize;
2200 switch (bNode.blockSize) {
2201 case k1ByteSize:
2202 pTy = PTY_i8;
2203 break;
2204 case k2ByteSize:
2205 pTy = PTY_i16;
2206 break;
2207 case k4ByteSize:
2208 pTy = PTY_i32;
2209 break;
2210 default:
2211 size = k8ByteSize; /* pTy remains i64 */
2212 break;
2213 }
2214 loadSize = GetPrimTypeSize(pTy) * kBitsPerByte;
2215 MemOperand &mem = CreateMemOpnd(regno, 0, size * kBitsPerByte);
2216 RegOperand *res = &GetOrCreatePhysicalRegisterOperand(R0, loadSize, kRegTyInt);
2217 SelectCopy(*res, pTy, mem, pTy);
2218 if (bNode.blockSize > static_cast<int32>(k8ByteSize)) {
2219 MemOperand &newMem = CreateMemOpnd(regno, k8ByteSize, size * kBitsPerByte);
2220 res = &GetOrCreatePhysicalRegisterOperand(R1, loadSize, kRegTyInt);
2221 SelectCopy(*res, pTy, newMem, pTy);
2222 }
2223 }
2224 bool intReg = fpregs == 0;
2225 for (uint32 i = 0; i < numRegs; i++) {
2226 AArch64reg preg = static_cast<AArch64reg>((intReg ? R0 : V0) + i);
2227 MOperator mop = intReg ? MOP_pseudo_ret_int : MOP_pseudo_ret_float;
2228 RegOperand &dest = GetOrCreatePhysicalRegisterOperand(preg, loadSize, intReg ? kRegTyInt : kRegTyFloat);
2229 Insn &pseudo = GetInsnBuilder()->BuildInsn(mop, dest);
2230 GetCurBB()->AppendInsn(pseudo);
2231 }
2232 return true;
2233 }
2234 }
2235 return false;
2236 }
2237
2238 /* return true if blkassignoff for return, false otherwise */
LmbcSmallAggForCall(BlkassignoffNode & bNode,const Operand * src,std::vector<TyIdx> ** parmList)2239 bool AArch64CGFunc::LmbcSmallAggForCall(BlkassignoffNode &bNode, const Operand *src, std::vector<TyIdx> **parmList)
2240 {
2241 AArch64reg regno = static_cast<AArch64reg>(static_cast<const RegOperand *>(src)->GetRegisterNumber());
2242 if (IsBlkassignForPush(bNode)) {
2243 PrimType pTy = PTY_i64;
2244 MIRStructType *ty = static_cast<MIRStructType *>(LmbcGetAggTyFromCallSite(&bNode, parmList));
2245 uint32 size = 0;
2246 uint32 fpregs = ty ? FloatParamRegRequired(ty, size) : 0; /* fp size determined */
2247 if (fpregs > 0) {
2248 /* pure floating point in agg */
2249 pTy = (size == k4ByteSize) ? PTY_f32 : PTY_f64;
2250 for (uint32 i = 0; i < fpregs; i++) {
2251 int32 s = (i == 0) ? 0 : static_cast<int32>(i * size);
2252 MemOperand &mem = CreateMemOpnd(regno, s, size * kBitsPerByte);
2253 RegOperand *res = &CreateVirtualRegisterOperand(NewVReg(kRegTyFloat, size));
2254 SelectCopy(*res, pTy, mem, pTy);
2255 SetLmbcArgInfo(res, pTy, 0, static_cast<int32>(fpregs));
2256 IncLmbcArgsInRegs(kRegTyFloat);
2257 }
2258 IncLmbcTotalArgs();
2259 return true;
2260 } else if (bNode.blockSize <= static_cast<int32>(k16ByteSize)) {
2261 /* integer/mixed types in register/s */
2262 size = k4ByteSize;
2263 switch (bNode.blockSize) {
2264 case k1ByteSize:
2265 pTy = PTY_i8;
2266 break;
2267 case k2ByteSize:
2268 pTy = PTY_i16;
2269 break;
2270 case k4ByteSize:
2271 pTy = PTY_i32;
2272 break;
2273 default:
2274 size = k8ByteSize; /* pTy remains i64 */
2275 break;
2276 }
2277 MemOperand &mem = CreateMemOpnd(regno, 0, size * kBitsPerByte);
2278 RegOperand *res = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, size));
2279 SelectCopy(*res, pTy, mem, pTy);
2280 SetLmbcArgInfo(res, pTy, bNode.offset,
2281 bNode.blockSize > static_cast<int32>(k8ByteSize) ? kThirdReg : kSecondReg);
2282 IncLmbcArgsInRegs(kRegTyInt);
2283 if (bNode.blockSize > static_cast<int32>(k8ByteSize)) {
2284 MemOperand &newMem = CreateMemOpnd(regno, k8ByteSize, size * kBitsPerByte);
2285 RegOperand *newRes = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, size));
2286 SelectCopy(*newRes, pTy, newMem, pTy);
2287 SetLmbcArgInfo(newRes, pTy, bNode.offset + k8ByteSizeInt, kThirdReg);
2288 IncLmbcArgsInRegs(kRegTyInt);
2289 }
2290 IncLmbcTotalArgs();
2291 return true;
2292 }
2293 }
2294 return false;
2295 }
2296
2297 /* This function is incomplete and may be removed when Lmbc IR is changed
2298 to have the lowerer figures out the address of the large agg to reside */
LmbcFindTotalStkUsed(std::vector<TyIdx> * paramList)2299 uint32 AArch64CGFunc::LmbcFindTotalStkUsed(std::vector<TyIdx> *paramList)
2300 {
2301 CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
2302 AArch64CallConvImpl parmlocator(GetBecommon());
2303 CCLocInfo pLoc;
2304 for (TyIdx tyIdx : *paramList) {
2305 MIRType *ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(tyIdx);
2306 (void)parmlocator.LocateNextParm(*ty, pLoc);
2307 }
2308 return 0;
2309 }
2310
2311 /* All arguments passed as registers */
LmbcTotalRegsUsed()2312 uint32 AArch64CGFunc::LmbcTotalRegsUsed()
2313 {
2314 if (GetLmbcArgInfo() == nullptr) {
2315 return 0; /* no arg */
2316 }
2317 MapleVector<int32> ®s = GetLmbcCallArgNumOfRegs();
2318 MapleVector<PrimType> &types = GetLmbcCallArgTypes();
2319 uint32 iCnt = 0;
2320 uint32 fCnt = 0;
2321 for (uint32 i = 0; i < regs.size(); i++) {
2322 if (IsPrimitiveInteger(types[i])) {
2323 if ((iCnt + static_cast<uint32>(regs[i])) <= k8ByteSize) {
2324 iCnt += static_cast<uint32>(regs[i]);
2325 };
2326 } else {
2327 if ((fCnt + static_cast<uint32>(regs[i])) <= k8ByteSize) {
2328 fCnt += static_cast<uint32>(regs[i]);
2329 };
2330 }
2331 }
2332 return iCnt + fCnt;
2333 }
2334
2335 /* If blkassignoff for argument, this function loads the agg arguments into
2336 virtual registers, disregard if there is sufficient physicall call
2337 registers. Argument > 16-bytes are copied to preset space and ptr
2338 result is loaded into virtual register.
2339 If blassign is not for argument, this function simply memcpy */
SelectBlkassignoff(BlkassignoffNode & bNode,Operand * src)2340 void AArch64CGFunc::SelectBlkassignoff(BlkassignoffNode &bNode, Operand *src)
2341 {
2342 CHECK_FATAL(src->GetKind() == Operand::kOpdRegister, "blkassign src type not in register");
2343 std::vector<TyIdx> *parmList;
2344 if (GetLmbcArgInfo() == nullptr) {
2345 LmbcArgInfo *p = memPool->New<LmbcArgInfo>(*GetFuncScopeAllocator());
2346 SetLmbcArgInfo(p);
2347 }
2348 if (LmbcSmallAggForRet(bNode, src)) {
2349 return;
2350 } else if (LmbcSmallAggForCall(bNode, src, &parmList)) {
2351 return;
2352 }
2353 Operand *dest = HandleExpr(bNode, *bNode.Opnd(0));
2354 RegOperand *regResult = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
2355 /* memcpy for agg assign OR large agg for arg/ret */
2356 int32 offset = bNode.offset;
2357 if (IsBlkassignForPush(bNode)) {
2358 /* large agg for call, addr to be pushed in SelectCall */
2359 offset = GetLmbcTotalStkUsed();
2360 if (offset < 0) {
2361 /* length of ALL stack based args for this call, this location is where the
2362 next large agg resides, its addr will then be passed */
2363 offset = LmbcFindTotalStkUsed(parmList) + LmbcTotalRegsUsed();
2364 }
2365 SetLmbcTotalStkUsed(offset + bNode.blockSize); /* next use */
2366 SetLmbcArgInfo(regResult, PTY_i64, 0, 1); /* 1 reg for ptr */
2367 IncLmbcArgsInRegs(kRegTyInt);
2368 IncLmbcTotalArgs();
2369 /* copy large agg arg to offset below */
2370 }
2371 std::vector<Operand *> opndVec;
2372 opndVec.push_back(regResult); /* result */
2373 opndVec.push_back(PrepareMemcpyParamOpnd(offset, *dest)); /* param 0 */
2374 opndVec.push_back(src); /* param 1 */
2375 opndVec.push_back(PrepareMemcpyParamOpnd(static_cast<uint64>(static_cast<int64>(bNode.blockSize)))); /* param 2 */
2376 SelectLibCall("memcpy", opndVec, PTY_a64, PTY_a64);
2377 }
2378
SelectAggIassign(IassignNode & stmt,Operand & AddrOpnd)2379 void AArch64CGFunc::SelectAggIassign(IassignNode &stmt, Operand &AddrOpnd)
2380 {
2381 DEBUG_ASSERT(stmt.Opnd(0) != nullptr, "null ptr check");
2382 Operand &lhsAddrOpnd = LoadIntoRegister(AddrOpnd, stmt.Opnd(0)->GetPrimType());
2383 uint32 lhsOffset = 0;
2384 MIRType *stmtType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(stmt.GetTyIdx());
2385 MIRPtrType *lhsPointerType = static_cast<MIRPtrType *>(stmtType);
2386 bool loadToRegs4StructReturn = false;
2387 if (mirModule.CurFunction()->StructReturnedInRegs()) {
2388 MIRSymbol *retSt = mirModule.CurFunction()->GetFormal(0);
2389 if (stmt.Opnd(0)->GetOpCode() == OP_dread) {
2390 DreadNode *dread = static_cast<DreadNode *>(stmt.Opnd(0));
2391 MIRSymbol *addrSym = mirModule.CurFunction()->GetLocalOrGlobalSymbol(dread->GetStIdx());
2392 loadToRegs4StructReturn = (retSt == addrSym);
2393 }
2394 }
2395 MIRType *lhsType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(lhsPointerType->GetPointedTyIdx());
2396 if (stmt.GetFieldID() != 0) {
2397 MIRStructType *structType = static_cast<MIRStructType *>(lhsType);
2398 DEBUG_ASSERT(structType != nullptr, "SelectAggIassign: non-zero fieldID for non-structure");
2399 lhsType = structType->GetFieldType(stmt.GetFieldID());
2400 lhsOffset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, stmt.GetFieldID()).first);
2401 } else if (lhsType->GetKind() == kTypeArray) {
2402 #if DEBUG
2403 MIRArrayType *arrayLhsType = static_cast<MIRArrayType *>(lhsType);
2404 /* access an array element */
2405 MIRType *lhsType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(arrayLhsType->GetElemTyIdx());
2406 MIRTypeKind typeKind = lhsType->GetKind();
2407 DEBUG_ASSERT(((typeKind == kTypeScalar) || (typeKind == kTypeStruct) || (typeKind == kTypeClass) ||
2408 (typeKind == kTypePointer)),
2409 "unexpected array element type in iassign");
2410 #endif
2411 } else if (lhsType->GetKind() == kTypeFArray) {
2412 #if DEBUG
2413 MIRFarrayType *farrayLhsType = static_cast<MIRFarrayType *>(lhsType);
2414 /* access an array element */
2415 MIRType *lhsElemType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(farrayLhsType->GetElemTyIdx());
2416 MIRTypeKind typeKind = lhsElemType->GetKind();
2417 DEBUG_ASSERT(((typeKind == kTypeScalar) || (typeKind == kTypeStruct) || (typeKind == kTypeClass) ||
2418 (typeKind == kTypePointer)),
2419 "unexpected array element type in iassign");
2420 #endif
2421 }
2422 uint32 lhsAlign = GetBecommon().GetTypeAlign(lhsType->GetTypeIndex());
2423 uint64 lhsSize = GetBecommon().GetTypeSize(lhsType->GetTypeIndex());
2424
2425 uint32 rhsAlign;
2426 uint32 alignUsed;
2427 uint32 rhsOffset = 0;
2428 if (stmt.GetRHS()->GetOpCode() == OP_dread) {
2429 AddrofNode *rhsDread = static_cast<AddrofNode *>(stmt.GetRHS());
2430 MIRSymbol *rhsSymbol = GetFunction().GetLocalOrGlobalSymbol(rhsDread->GetStIdx());
2431 MIRType *rhsType = rhsSymbol->GetType();
2432 if (rhsDread->GetFieldID() != 0) {
2433 MIRStructType *structType = static_cast<MIRStructType *>(rhsSymbol->GetType());
2434 DEBUG_ASSERT(structType != nullptr, "SelectAggIassign: non-zero fieldID for non-structure");
2435 rhsType = structType->GetFieldType(rhsDread->GetFieldID());
2436 rhsOffset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, rhsDread->GetFieldID()).first);
2437 }
2438 if (loadToRegs4StructReturn) {
2439 /* generate move to regs for agg return */
2440 CHECK_FATAL(lhsSize <= k16ByteSize, "SelectAggIassign: illegal struct size");
2441 CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
2442 AArch64CallConvImpl parmlocator(GetBecommon());
2443 CCLocInfo pLoc;
2444 parmlocator.LocateNextParm(*lhsType, pLoc, true, GetBecommon().GetMIRModule().CurFunction());
2445 /* aggregates are 8 byte aligned. */
2446 Operand *rhsmemopnd = nullptr;
2447 RegOperand *result[kFourRegister]; /* up to 2 int or 4 fp */
2448 uint32 loadSize;
2449 uint32 numRegs;
2450 RegType regType;
2451 PrimType retPty;
2452 bool fpParm = false;
2453 if (pLoc.numFpPureRegs) {
2454 loadSize = pLoc.fpSize;
2455 numRegs = pLoc.numFpPureRegs;
2456 fpParm = true;
2457 regType = kRegTyFloat;
2458 retPty = (pLoc.fpSize == k4ByteSize) ? PTY_f32 : PTY_f64;
2459 } else {
2460 if (CGOptions::IsBigEndian()) {
2461 loadSize = k8ByteSize;
2462 numRegs = (lhsSize <= k8ByteSize) ? kOneRegister : kTwoRegister;
2463 regType = kRegTyInt;
2464 retPty = PTY_u64;
2465 } else {
2466 loadSize = (lhsSize <= k4ByteSize) ? k4ByteSize : k8ByteSize;
2467 numRegs = (lhsSize <= k8ByteSize) ? kOneRegister : kTwoRegister;
2468 regType = kRegTyInt;
2469 retPty = PTY_u32;
2470 }
2471 }
2472 bool parmCopy = IsParamStructCopy(*rhsSymbol);
2473 for (uint32 i = 0; i < numRegs; i++) {
2474 if (parmCopy) {
2475 rhsmemopnd = &LoadStructCopyBase(
2476 *rhsSymbol, (rhsOffset + static_cast<int64>(i * (fpParm ? loadSize : k8ByteSize))),
2477 static_cast<int>(loadSize * kBitsPerByte));
2478 } else {
2479 rhsmemopnd = &GetOrCreateMemOpnd(
2480 *rhsSymbol, (rhsOffset + static_cast<int64>(i * (fpParm ? loadSize : k8ByteSize))),
2481 (loadSize * kBitsPerByte));
2482 }
2483 result[i] = &CreateVirtualRegisterOperand(NewVReg(regType, loadSize));
2484 MOperator mop1 = PickLdInsn(loadSize * kBitsPerByte, retPty);
2485 Insn &ld = GetInsnBuilder()->BuildInsn(mop1, *(result[i]), *rhsmemopnd);
2486 GetCurBB()->AppendInsn(ld);
2487 }
2488 AArch64reg regs[kFourRegister];
2489 regs[kFirstReg] = static_cast<AArch64reg>(pLoc.reg0);
2490 regs[kSecondReg] = static_cast<AArch64reg>(pLoc.reg1);
2491 regs[kThirdReg] = static_cast<AArch64reg>(pLoc.reg2);
2492 regs[kFourthReg] = static_cast<AArch64reg>(pLoc.reg3);
2493 for (uint32 i = 0; i < numRegs; i++) {
2494 AArch64reg preg;
2495 MOperator mop2;
2496 if (fpParm) {
2497 preg = regs[i];
2498 mop2 = (loadSize == k4ByteSize) ? MOP_xvmovs : MOP_xvmovd;
2499 } else {
2500 preg = (i == 0 ? R0 : R1);
2501 mop2 = (loadSize == k4ByteSize) ? MOP_wmovrr : MOP_xmovrr;
2502 }
2503 RegOperand &dest = GetOrCreatePhysicalRegisterOperand(preg, (loadSize * kBitsPerByte), regType);
2504 Insn &mov = GetInsnBuilder()->BuildInsn(mop2, dest, *(result[i]));
2505 GetCurBB()->AppendInsn(mov);
2506 }
2507 /* Create artificial dependency to extend the live range */
2508 for (uint32 i = 0; i < numRegs; i++) {
2509 AArch64reg preg;
2510 MOperator mop3;
2511 if (fpParm) {
2512 preg = regs[i];
2513 mop3 = MOP_pseudo_ret_float;
2514 } else {
2515 preg = (i == 0 ? R0 : R1);
2516 mop3 = MOP_pseudo_ret_int;
2517 }
2518 RegOperand &dest = GetOrCreatePhysicalRegisterOperand(preg, loadSize * kBitsPerByte, regType);
2519 Insn &pseudo = GetInsnBuilder()->BuildInsn(mop3, dest);
2520 GetCurBB()->AppendInsn(pseudo);
2521 }
2522 return;
2523 }
2524 rhsAlign = GetBecommon().GetTypeAlign(rhsType->GetTypeIndex());
2525 alignUsed = std::min(lhsAlign, rhsAlign);
2526 DEBUG_ASSERT(alignUsed != 0, "expect non-zero");
2527 uint32 copySize = GetAggCopySize(rhsOffset, lhsOffset, alignUsed);
2528 MemOperand *rhsBaseMemOpnd;
2529 if (IsParamStructCopy(*rhsSymbol)) {
2530 rhsBaseMemOpnd = &LoadStructCopyBase(*rhsSymbol, rhsOffset, static_cast<int>(copySize * k8BitSize));
2531 } else {
2532 rhsBaseMemOpnd = GenLargeAggFormalMemOpnd(*rhsSymbol, copySize, rhsOffset, true);
2533 }
2534 RegOperand *rhsBaseReg = rhsBaseMemOpnd->GetBaseRegister();
2535 int64 rhsOffsetVal = rhsBaseMemOpnd->GetOffsetOperand()->GetValue();
2536 bool rhsIsLo12 = (rhsBaseMemOpnd->GetAddrMode() == MemOperand::kAddrModeLo12Li);
2537 if (lhsSize > kParmMemcpySize) {
2538 std::vector<Operand *> opndVec;
2539 RegOperand *regResult = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
2540 opndVec.push_back(regResult); /* result */
2541
2542 opndVec.push_back(PrepareMemcpyParamOpnd(static_cast<int64>(lhsOffset), lhsAddrOpnd)); /* param 0 */
2543
2544 opndVec.push_back(PrepareMemcpyParamOpnd(rhsOffsetVal, *rhsBaseReg)); /* param 1 */
2545
2546 opndVec.push_back(PrepareMemcpyParamOpnd(lhsSize)); /* param 2 */
2547
2548 SelectLibCall("memcpy", opndVec, PTY_a64, PTY_a64);
2549
2550 return;
2551 }
2552 for (uint32 i = 0; i < (lhsSize / copySize); ++i) {
2553 uint32 rhsBaseOffset = static_cast<uint32>(rhsOffsetVal + i * copySize);
2554 uint32 lhsBaseOffset = lhsOffset + i * copySize;
2555 MemOperand::AArch64AddressingMode addrMode =
2556 rhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
2557 MIRSymbol *sym = rhsIsLo12 ? rhsSymbol : nullptr;
2558 OfstOperand &rhsOfstOpnd = GetOrCreateOfstOpnd(rhsBaseOffset, k32BitSize);
2559 MemOperand *rhsMemOpnd =
2560 &GetOrCreateMemOpnd(addrMode, copySize * k8BitSize, rhsBaseReg, nullptr, &rhsOfstOpnd, sym);
2561 rhsMemOpnd = FixLargeMemOpnd(*rhsMemOpnd, copySize);
2562 /* generate the load */
2563 RegOperand &result = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, copySize)));
2564 MOperator mOpLDP = (copySize == k4BitSize) ? MOP_wldp : MOP_xldp;
2565 bool doPair = (!rhsIsLo12 && (copySize >= k4BitSize) && ((i + 1) < (lhsSize / copySize)));
2566 RegOperand *result1 = nullptr;
2567 if (doPair) {
2568 regno_t vRegNO1 = NewVReg(kRegTyInt, std::max(4u, copySize));
2569 result1 = &CreateVirtualRegisterOperand(vRegNO1);
2570 rhsMemOpnd =
2571 FixLargeMemOpnd(mOpLDP, *static_cast<MemOperand *>(rhsMemOpnd), result.GetSize(), kInsnThirdOpnd);
2572 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpLDP, result, *result1, *rhsMemOpnd));
2573 } else {
2574 MOperator mOp = PickLdInsn(copySize * k8BitSize, PTY_u32);
2575 rhsMemOpnd =
2576 FixLargeMemOpnd(mOp, *static_cast<MemOperand *>(rhsMemOpnd), result.GetSize(), kInsnSecondOpnd);
2577 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd));
2578 }
2579 /* generate the store */
2580 OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(lhsBaseOffset, k32BitSize);
2581 MemOperand *lhsMemOpnd =
2582 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, copySize * k8BitSize,
2583 static_cast<RegOperand *>(&lhsAddrOpnd), nullptr, &ofstOpnd, nullptr);
2584 if (doPair) {
2585 MOperator mOpSTP = (copySize == k4BitSize) ? MOP_wstp : MOP_xstp;
2586 lhsMemOpnd = FixLargeMemOpnd(mOpSTP, *lhsMemOpnd, result.GetSize(), kInsnThirdOpnd);
2587 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpSTP, result, *result1, *lhsMemOpnd));
2588 i++;
2589 } else {
2590 MOperator mOp = PickStInsn(copySize * k8BitSize, PTY_u32);
2591 lhsMemOpnd = FixLargeMemOpnd(mOp, *lhsMemOpnd, copySize * k8BitSize, kInsnSecondOpnd);
2592 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *lhsMemOpnd));
2593 }
2594 }
2595 /* take care of extra content at the end less than the unit of alignUsed */
2596 uint64 lhsSizeCovered = (lhsSize / copySize) * copySize;
2597 uint32 newAlignUsed = copySize;
2598 while (lhsSizeCovered < lhsSize) {
2599 newAlignUsed = newAlignUsed >> 1;
2600 CHECK_FATAL(newAlignUsed != 0, "expect non-zero");
2601 if ((lhsSizeCovered + newAlignUsed) > lhsSize) {
2602 continue;
2603 }
2604 MemOperand::AArch64AddressingMode addrMode =
2605 rhsIsLo12 ? MemOperand::kAddrModeLo12Li : MemOperand::kAddrModeBOi;
2606 MIRSymbol *sym = rhsIsLo12 ? rhsSymbol : nullptr;
2607 OfstOperand &rhsOfstOpnd =
2608 GetOrCreateOfstOpnd(lhsSizeCovered + static_cast<uint64>(rhsOffsetVal), k32BitSize);
2609 MemOperand *rhsMemOpnd =
2610 &GetOrCreateMemOpnd(addrMode, newAlignUsed * k8BitSize, rhsBaseReg, nullptr, &rhsOfstOpnd, sym);
2611 /* generate the load */
2612 Operand &result = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, newAlignUsed)));
2613 MOperator mOp = PickLdInsn(newAlignUsed * k8BitSize, PTY_u32);
2614 rhsMemOpnd = FixLargeMemOpnd(mOp, *rhsMemOpnd, newAlignUsed * k8BitSize, kInsnSecondOpnd);
2615 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd));
2616 /* generate the store */
2617 OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(lhsOffset + lhsSizeCovered, k32BitSize);
2618 MemOperand &lhsMemOpnd = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, newAlignUsed * k8BitSize,
2619 static_cast<RegOperand *>(&lhsAddrOpnd), nullptr, &ofstOpnd,
2620 static_cast<MIRSymbol *>(nullptr));
2621 mOp = PickStInsn(newAlignUsed * k8BitSize, PTY_u32);
2622 lhsMemOpnd = *FixLargeMemOpnd(mOp, lhsMemOpnd, newAlignUsed * k8BitSize, kInsnSecondOpnd);
2623 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, lhsMemOpnd));
2624 lhsSizeCovered += newAlignUsed;
2625 }
2626 } else { /* rhs is iread */
2627 DEBUG_ASSERT(stmt.GetRHS()->GetOpCode() == OP_iread, "SelectAggDassign: NYI");
2628 IreadNode *rhsIread = static_cast<IreadNode *>(stmt.GetRHS());
2629 RegOperand *rhsAddrOpnd = static_cast<RegOperand *>(HandleExpr(*rhsIread, *rhsIread->Opnd(0)));
2630 rhsAddrOpnd = &LoadIntoRegister(*rhsAddrOpnd, rhsIread->Opnd(0)->GetPrimType());
2631 MIRPtrType *rhsPointerType =
2632 static_cast<MIRPtrType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(rhsIread->GetTyIdx()));
2633 MIRType *rhsType = static_cast<MIRStructType *>(
2634 GlobalTables::GetTypeTable().GetTypeFromTyIdx(rhsPointerType->GetPointedTyIdx()));
2635 bool isRefField = false;
2636 if (rhsIread->GetFieldID() != 0) {
2637 MIRStructType *rhsStructType = static_cast<MIRStructType *>(rhsType);
2638 DEBUG_ASSERT(rhsStructType, "SelectAggDassign: non-zero fieldID for non-structure");
2639 rhsType = rhsStructType->GetFieldType(rhsIread->GetFieldID());
2640 rhsOffset = static_cast<uint32>(GetBecommon().GetFieldOffset(*rhsStructType, rhsIread->GetFieldID()).first);
2641 isRefField = GetBecommon().IsRefField(*rhsStructType, rhsIread->GetFieldID());
2642 }
2643 if (loadToRegs4StructReturn) {
2644 /* generate move to regs. */
2645 CHECK_FATAL(lhsSize <= k16ByteSize, "SelectAggIassign: illegal struct size");
2646 RegOperand *result[kTwoRegister]; /* maximum 16 bytes, 2 registers */
2647 uint32 loadSize;
2648 if (CGOptions::IsBigEndian()) {
2649 loadSize = k8ByteSize;
2650 } else {
2651 loadSize = (lhsSize <= k4ByteSize) ? k4ByteSize : k8ByteSize;
2652 }
2653 uint32 numRegs = (lhsSize <= k8ByteSize) ? kOneRegister : kTwoRegister;
2654 for (uint32 i = 0; i < numRegs; i++) {
2655 OfstOperand *rhsOffOpnd = &GetOrCreateOfstOpnd(rhsOffset + i * loadSize, loadSize * kBitsPerByte);
2656 Operand &rhsmemopnd = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, loadSize * kBitsPerByte, rhsAddrOpnd,
2657 nullptr, rhsOffOpnd, nullptr);
2658 result[i] = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, loadSize));
2659 MOperator mop1 = PickLdInsn(loadSize * kBitsPerByte, PTY_u32);
2660 Insn &ld = GetInsnBuilder()->BuildInsn(mop1, *(result[i]), rhsmemopnd);
2661 ld.MarkAsAccessRefField(isRefField);
2662 GetCurBB()->AppendInsn(ld);
2663 }
2664 for (uint32 i = 0; i < numRegs; i++) {
2665 AArch64reg preg = (i == 0 ? R0 : R1);
2666 RegOperand &dest = GetOrCreatePhysicalRegisterOperand(preg, loadSize * kBitsPerByte, kRegTyInt);
2667 Insn &mov = GetInsnBuilder()->BuildInsn(MOP_xmovrr, dest, *(result[i]));
2668 GetCurBB()->AppendInsn(mov);
2669 }
2670 /* Create artificial dependency to extend the live range */
2671 for (uint32 i = 0; i < numRegs; i++) {
2672 AArch64reg preg = (i == 0 ? R0 : R1);
2673 RegOperand &dest = GetOrCreatePhysicalRegisterOperand(preg, loadSize * kBitsPerByte, kRegTyInt);
2674 Insn &pseudo = GetInsnBuilder()->BuildInsn(MOP_pseudo_ret_int, dest);
2675 GetCurBB()->AppendInsn(pseudo);
2676 }
2677 return;
2678 }
2679 rhsAlign = GetBecommon().GetTypeAlign(rhsType->GetTypeIndex());
2680 alignUsed = std::min(lhsAlign, rhsAlign);
2681 DEBUG_ASSERT(alignUsed != 0, "expect non-zero");
2682 uint32 copySize = GetAggCopySize(rhsOffset, lhsOffset, alignUsed);
2683 if (lhsSize > kParmMemcpySize) {
2684 std::vector<Operand *> opndVec;
2685 RegOperand *regResult = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
2686 opndVec.push_back(regResult); /* result */
2687
2688 opndVec.push_back(PrepareMemcpyParamOpnd(static_cast<int64>(lhsOffset), lhsAddrOpnd)); /* param 0 */
2689
2690 opndVec.push_back(PrepareMemcpyParamOpnd(static_cast<int64>(rhsOffset), *rhsAddrOpnd)); /* param 1 */
2691
2692 opndVec.push_back(PrepareMemcpyParamOpnd(lhsSize)); /* param 2 */
2693
2694 SelectLibCall("memcpy", opndVec, PTY_a64, PTY_a64);
2695
2696 return;
2697 }
2698 DEBUG_ASSERT(copySize != 0, "expect non-zero");
2699 for (uint32 i = 0; i < (lhsSize / copySize); i++) {
2700 /* generate the load */
2701 uint32 operandSize = copySize * k8BitSize;
2702 OfstOperand &rhsOfstOpnd = GetOrCreateOfstOpnd(rhsOffset + i * copySize, k32BitSize);
2703 MemOperand *rhsMemOpnd =
2704 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, operandSize, static_cast<RegOperand *>(rhsAddrOpnd),
2705 nullptr, &rhsOfstOpnd, nullptr);
2706 RegOperand &result = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, copySize)));
2707 bool doPair = ((copySize >= k4BitSize) && ((i + 1) < (lhsSize / copySize)));
2708 Insn *insn = nullptr;
2709 RegOperand *result1 = nullptr;
2710 if (doPair) {
2711 MOperator mOpLDP = (copySize == k4BitSize) ? MOP_wldp : MOP_xldp;
2712 rhsMemOpnd =
2713 FixLargeMemOpnd(mOpLDP, *static_cast<MemOperand *>(rhsMemOpnd), operandSize, kInsnThirdOpnd);
2714 result1 = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, std::max(4u, copySize)));
2715 insn = &GetInsnBuilder()->BuildInsn(mOpLDP, result, *result1, *rhsMemOpnd);
2716 } else {
2717 MOperator mOp = PickLdInsn(operandSize, PTY_u32);
2718 rhsMemOpnd = FixLargeMemOpnd(mOp, *static_cast<MemOperand *>(rhsMemOpnd), operandSize, kInsnSecondOpnd);
2719 insn = &GetInsnBuilder()->BuildInsn(mOp, result, *rhsMemOpnd);
2720 }
2721 insn->MarkAsAccessRefField(isRefField);
2722 GetCurBB()->AppendInsn(*insn);
2723 /* generate the store */
2724 OfstOperand &lhsOfstOpnd = GetOrCreateOfstOpnd(lhsOffset + i * copySize, k32BitSize);
2725 MemOperand *lhsMemOpnd =
2726 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, operandSize, static_cast<RegOperand *>(&lhsAddrOpnd),
2727 nullptr, &lhsOfstOpnd, nullptr);
2728 if (doPair) {
2729 MOperator mOpSTP = (copySize == k4BitSize) ? MOP_wstp : MOP_xstp;
2730 lhsMemOpnd =
2731 FixLargeMemOpnd(mOpSTP, *static_cast<MemOperand *>(lhsMemOpnd), operandSize, kInsnThirdOpnd);
2732 DEBUG_ASSERT(result1 != nullptr, "result1 should not be nullptr");
2733 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpSTP, result, *result1, *lhsMemOpnd));
2734 i++;
2735 } else {
2736 MOperator mOp = PickStInsn(operandSize, PTY_u32);
2737 lhsMemOpnd = FixLargeMemOpnd(mOp, *static_cast<MemOperand *>(lhsMemOpnd), operandSize, kInsnSecondOpnd);
2738 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, *lhsMemOpnd));
2739 }
2740 }
2741 /* take care of extra content at the end less than the unit */
2742 uint64 lhsSizeCovered = (lhsSize / copySize) * copySize;
2743 uint32 newAlignUsed = copySize;
2744 while (lhsSizeCovered < lhsSize) {
2745 newAlignUsed = newAlignUsed >> 1;
2746 CHECK_FATAL(newAlignUsed != 0, "expect non-zero");
2747 if ((lhsSizeCovered + newAlignUsed) > lhsSize) {
2748 continue;
2749 }
2750 /* generate the load */
2751 OfstOperand &rhsOfstOpnd = GetOrCreateOfstOpnd(rhsOffset + lhsSizeCovered, k32BitSize);
2752 uint32 memOpndSize = newAlignUsed * k8BitSize;
2753 MemOperand *rhsMemOpnd =
2754 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, memOpndSize, static_cast<RegOperand *>(rhsAddrOpnd),
2755 nullptr, &rhsOfstOpnd, nullptr);
2756 regno_t vRegNO = NewVReg(kRegTyInt, std::max(4u, newAlignUsed));
2757 RegOperand &result = CreateVirtualRegisterOperand(vRegNO);
2758 MOperator mOpLD = PickLdInsn(memOpndSize, PTY_u32);
2759 rhsMemOpnd = FixLargeMemOpnd(mOpLD, *rhsMemOpnd, memOpndSize, static_cast<uint32>(kInsnSecondOpnd));
2760 Insn &insn = GetInsnBuilder()->BuildInsn(mOpLD, result, *rhsMemOpnd);
2761 insn.MarkAsAccessRefField(isRefField);
2762 GetCurBB()->AppendInsn(insn);
2763 /* generate the store */
2764 OfstOperand &lhsOfstOpnd = GetOrCreateOfstOpnd(lhsOffset + lhsSizeCovered, k32BitSize);
2765 MemOperand *lhsMemOpnd =
2766 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, memOpndSize, static_cast<RegOperand *>(&lhsAddrOpnd),
2767 nullptr, &lhsOfstOpnd, nullptr);
2768 MOperator mOpST = PickStInsn(memOpndSize, PTY_u32);
2769 lhsMemOpnd = FixLargeMemOpnd(mOpST, *lhsMemOpnd, memOpndSize, static_cast<uint32>(kInsnSecondOpnd));
2770 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpST, result, *lhsMemOpnd));
2771 lhsSizeCovered += newAlignUsed;
2772 }
2773 }
2774 }
2775
SelectReturnSendOfStructInRegs(BaseNode * x)2776 void AArch64CGFunc::SelectReturnSendOfStructInRegs(BaseNode *x)
2777 {
2778 uint32 offset = 0;
2779 if (x->GetOpCode() == OP_dread) {
2780 DreadNode *dread = static_cast<DreadNode *>(x);
2781 MIRSymbol *sym = GetFunction().GetLocalOrGlobalSymbol(dread->GetStIdx());
2782 MIRType *mirType = sym->GetType();
2783 if (dread->GetFieldID() != 0) {
2784 MIRStructType *structType = static_cast<MIRStructType *>(mirType);
2785 mirType = structType->GetFieldType(dread->GetFieldID());
2786 offset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, dread->GetFieldID()).first);
2787 }
2788 uint32 typeSize = GetBecommon().GetTypeSize(mirType->GetTypeIndex());
2789 /* generate move to regs for agg return */
2790 CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
2791 AArch64CallConvImpl parmlocator(GetBecommon());
2792 CCLocInfo pLoc;
2793 (void)parmlocator.LocateNextParm(*mirType, pLoc, true, GetBecommon().GetMIRModule().CurFunction());
2794 /* aggregates are 8 byte aligned. */
2795 Operand *rhsmemopnd = nullptr;
2796 RegOperand *result[kFourRegister]; /* up to 2 int or 4 fp */
2797 uint32 loadSize;
2798 uint32 numRegs;
2799 RegType regType;
2800 PrimType retPty;
2801 bool fpParm = false;
2802 if (pLoc.numFpPureRegs) {
2803 loadSize = pLoc.fpSize;
2804 numRegs = pLoc.numFpPureRegs;
2805 fpParm = true;
2806 regType = kRegTyFloat;
2807 retPty = (pLoc.fpSize == k4ByteSize) ? PTY_f32 : PTY_f64;
2808 } else {
2809 if (CGOptions::IsBigEndian()) {
2810 loadSize = k8ByteSize;
2811 numRegs = (typeSize <= k8ByteSize) ? kOneRegister : kTwoRegister;
2812 regType = kRegTyInt;
2813 retPty = PTY_u64;
2814 } else {
2815 loadSize = (typeSize <= k4ByteSize) ? k4ByteSize : k8ByteSize;
2816 numRegs = (typeSize <= k8ByteSize) ? kOneRegister : kTwoRegister;
2817 regType = kRegTyInt;
2818 retPty = PTY_u32;
2819 }
2820 }
2821 bool parmCopy = IsParamStructCopy(*sym);
2822 for (uint32 i = 0; i < numRegs; i++) {
2823 if (parmCopy) {
2824 rhsmemopnd =
2825 &LoadStructCopyBase(*sym, (offset + static_cast<int64>(i * (fpParm ? loadSize : k8ByteSize))),
2826 static_cast<int>(loadSize * kBitsPerByte));
2827 } else {
2828 rhsmemopnd =
2829 &GetOrCreateMemOpnd(*sym, (offset + static_cast<int64>(i * (fpParm ? loadSize : k8ByteSize))),
2830 (loadSize * kBitsPerByte));
2831 }
2832 result[i] = &CreateVirtualRegisterOperand(NewVReg(regType, loadSize));
2833 MOperator mop1 = PickLdInsn(loadSize * kBitsPerByte, retPty);
2834 Insn &ld = GetInsnBuilder()->BuildInsn(mop1, *(result[i]), *rhsmemopnd);
2835 GetCurBB()->AppendInsn(ld);
2836 }
2837 AArch64reg regs[kFourRegister];
2838 regs[kFirstReg] = static_cast<AArch64reg>(pLoc.reg0);
2839 regs[kSecondReg] = static_cast<AArch64reg>(pLoc.reg1);
2840 regs[kThirdReg] = static_cast<AArch64reg>(pLoc.reg2);
2841 regs[kFourthReg] = static_cast<AArch64reg>(pLoc.reg3);
2842 RegOperand *dest;
2843 for (uint32 i = 0; i < numRegs; i++) {
2844 AArch64reg preg;
2845 MOperator mop2;
2846 if (fpParm) {
2847 preg = regs[i];
2848 mop2 = (loadSize == k4ByteSize) ? MOP_xvmovs : MOP_xvmovd;
2849 } else {
2850 preg = (i == 0 ? R0 : R1);
2851 mop2 = (loadSize == k4ByteSize) ? MOP_wmovrr : MOP_xmovrr;
2852 }
2853 dest = &GetOrCreatePhysicalRegisterOperand(preg, (loadSize * kBitsPerByte), regType);
2854 Insn &mov = GetInsnBuilder()->BuildInsn(mop2, *dest, *(result[i]));
2855 GetCurBB()->AppendInsn(mov);
2856 }
2857 /* Create artificial dependency to extend the live range */
2858 for (uint32 i = 0; i < numRegs; i++) {
2859 AArch64reg preg;
2860 MOperator mop3;
2861 if (fpParm) {
2862 preg = regs[i];
2863 mop3 = MOP_pseudo_ret_float;
2864 } else {
2865 preg = (i == 0 ? R0 : R1);
2866 mop3 = MOP_pseudo_ret_int;
2867 }
2868 dest = &GetOrCreatePhysicalRegisterOperand(preg, loadSize * kBitsPerByte, regType);
2869 Insn &pseudo = GetInsnBuilder()->BuildInsn(mop3, *dest);
2870 GetCurBB()->AppendInsn(pseudo);
2871 }
2872 return;
2873 } else if (x->GetOpCode() == OP_iread) {
2874 IreadNode *iread = static_cast<IreadNode *>(x);
2875 RegOperand *rhsAddrOpnd = static_cast<RegOperand *>(HandleExpr(*iread, *iread->Opnd(0)));
2876 rhsAddrOpnd = &LoadIntoRegister(*rhsAddrOpnd, iread->Opnd(0)->GetPrimType());
2877 MIRPtrType *ptrType =
2878 static_cast<MIRPtrType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(iread->GetTyIdx()));
2879 MIRType *mirType = static_cast<MIRStructType *>(ptrType->GetPointedType());
2880 bool isRefField = false;
2881 if (iread->GetFieldID() != 0) {
2882 MIRStructType *structType = static_cast<MIRStructType *>(mirType);
2883 mirType = structType->GetFieldType(iread->GetFieldID());
2884 offset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, iread->GetFieldID()).first);
2885 isRefField = GetBecommon().IsRefField(*structType, iread->GetFieldID());
2886 }
2887 uint32 typeSize = GetBecommon().GetTypeSize(mirType->GetTypeIndex());
2888 /* generate move to regs. */
2889 RegOperand *result[kTwoRegister]; /* maximum 16 bytes, 2 registers */
2890 uint32 loadSize;
2891 if (CGOptions::IsBigEndian()) {
2892 loadSize = k8ByteSize;
2893 } else {
2894 loadSize = (typeSize <= k4ByteSize) ? k4ByteSize : k8ByteSize;
2895 }
2896 uint32 numRegs = (typeSize <= k8ByteSize) ? kOneRegister : kTwoRegister;
2897 for (uint32 i = 0; i < numRegs; i++) {
2898 OfstOperand *rhsOffOpnd = &GetOrCreateOfstOpnd(offset + i * loadSize, loadSize * kBitsPerByte);
2899 Operand &rhsmemopnd = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, loadSize * kBitsPerByte, rhsAddrOpnd,
2900 nullptr, rhsOffOpnd, nullptr);
2901 result[i] = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, loadSize));
2902 MOperator mop1 = PickLdInsn(loadSize * kBitsPerByte, PTY_u32);
2903 Insn &ld = GetInsnBuilder()->BuildInsn(mop1, *(result[i]), rhsmemopnd);
2904 ld.MarkAsAccessRefField(isRefField);
2905 GetCurBB()->AppendInsn(ld);
2906 }
2907 RegOperand *dest;
2908 for (uint32 i = 0; i < numRegs; i++) {
2909 AArch64reg preg = (i == 0 ? R0 : R1);
2910 dest = &GetOrCreatePhysicalRegisterOperand(preg, loadSize * kBitsPerByte, kRegTyInt);
2911 Insn &mov = GetInsnBuilder()->BuildInsn(MOP_xmovrr, *dest, *(result[i]));
2912 GetCurBB()->AppendInsn(mov);
2913 }
2914 /* Create artificial dependency to extend the live range */
2915 for (uint32 i = 0; i < numRegs; i++) {
2916 AArch64reg preg = (i == 0 ? R0 : R1);
2917 dest = &GetOrCreatePhysicalRegisterOperand(preg, loadSize * kBitsPerByte, kRegTyInt);
2918 Insn &pseudo = GetInsnBuilder()->BuildInsn(MOP_pseudo_ret_int, *dest);
2919 GetCurBB()->AppendInsn(pseudo);
2920 }
2921 return;
2922 } else { // dummy return of 0 inserted by front-end at absence of return
2923 DEBUG_ASSERT(x->GetOpCode() == OP_constval, "SelectReturnSendOfStructInRegs: unexpected return operand");
2924 uint32 typeSize = GetPrimTypeSize(x->GetPrimType());
2925 RegOperand &dest = GetOrCreatePhysicalRegisterOperand(R0, typeSize * kBitsPerByte, kRegTyInt);
2926 ImmOperand &src = CreateImmOperand(0, k16BitSize, false);
2927 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wmovri32, dest, src));
2928 return;
2929 }
2930 }
2931
SelectDread(const BaseNode & parent,DreadNode & expr)2932 Operand *AArch64CGFunc::SelectDread(const BaseNode &parent, DreadNode &expr)
2933 {
2934 MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(expr.GetStIdx());
2935 auto itr = stIdx2OverflowResult.find(expr.GetStIdx());
2936 if (itr != stIdx2OverflowResult.end()) {
2937 /* add_with_overflow / sub_with_overflow:
2938 * reg1: param1
2939 * reg2: param2
2940 * adds/subs reg3, reg1, reg2
2941 * cset reg4, vs
2942 * result is saved in std::pair<RegOperand*, RegOperand*>(reg3, reg4)
2943 */
2944 if (expr.GetFieldID() == 1) {
2945 return itr->second.first;
2946 } else {
2947 DEBUG_ASSERT(expr.GetFieldID() == 2, "only has 2 fileds for intrinsic overflow call result");
2948 return itr->second.second;
2949 }
2950 }
2951 if (symbol->IsEhIndex()) {
2952 CHECK_FATAL(false, "should not go here");
2953 MIRType *type = GlobalTables::GetTypeTable().GetTypeFromTyIdx(static_cast<TyIdx>(PTY_i32));
2954 /* use the second register return by __builtin_eh_return(). */
2955 AArch64CallConvImpl retLocator(GetBecommon());
2956 CCLocInfo retMech;
2957 retLocator.InitReturnInfo(*type, retMech);
2958 retLocator.SetupSecondRetReg(*type, retMech);
2959 return &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(retMech.GetReg1()), k64BitSize, kRegTyInt);
2960 }
2961
2962 PrimType symType = symbol->GetType()->GetPrimType();
2963 uint32 offset = 0;
2964 bool parmCopy = false;
2965 if (expr.GetFieldID() != 0) {
2966 MIRStructType *structType = static_cast<MIRStructType *>(symbol->GetType());
2967 DEBUG_ASSERT(structType != nullptr, "SelectDread: non-zero fieldID for non-structure");
2968 symType = structType->GetFieldType(expr.GetFieldID())->GetPrimType();
2969 offset = static_cast<uint32>(GetBecommon().GetFieldOffset(*structType, expr.GetFieldID()).first);
2970 parmCopy = IsParamStructCopy(*symbol);
2971 }
2972
2973 uint32 dataSize = GetPrimTypeBitSize(symType);
2974 uint32 aggSize = 0;
2975 if (symType == PTY_agg) {
2976 if (expr.GetPrimType() == PTY_agg) {
2977 aggSize = static_cast<uint32>(GetBecommon().GetTypeSize(symbol->GetType()->GetTypeIndex().GetIdx()));
2978 dataSize = ((expr.GetFieldID() == 0) ? GetPointerSize() : aggSize) << k8BitShift;
2979 } else {
2980 dataSize = GetPrimTypeBitSize(expr.GetPrimType());
2981 }
2982 }
2983 MemOperand *memOpnd = nullptr;
2984 if (aggSize > k8ByteSize) {
2985 if (parent.op == OP_eval) {
2986 if (symbol->GetAttr(ATTR_volatile)) {
2987 /* Need to generate loads for the upper parts of the struct. */
2988 Operand &dest = GetZeroOpnd(k64BitSize);
2989 uint32 numLoads = static_cast<uint32>(RoundUp(aggSize, k64BitSize) / k64BitSize);
2990 for (uint32 o = 0; o < numLoads; ++o) {
2991 if (parmCopy) {
2992 memOpnd = &LoadStructCopyBase(*symbol, offset + o * GetPointerSize(), GetPointerSize());
2993 } else {
2994 memOpnd = &GetOrCreateMemOpnd(*symbol, offset + o * GetPointerSize(), GetPointerSize());
2995 }
2996 if (IsImmediateOffsetOutOfRange(*memOpnd, GetPointerSize())) {
2997 memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, GetPointerSize());
2998 }
2999 SelectCopy(dest, PTY_u64, *memOpnd, PTY_u64);
3000 }
3001 } else {
3002 /* No side-effects. No need to generate anything for eval. */
3003 }
3004 } else {
3005 if (expr.GetFieldID() != 0) {
3006 CHECK_FATAL(false, "SelectDread: Illegal agg size");
3007 }
3008 }
3009 }
3010 if (parmCopy) {
3011 memOpnd = &LoadStructCopyBase(*symbol, offset, static_cast<int>(dataSize));
3012 } else {
3013 memOpnd = &GetOrCreateMemOpnd(*symbol, offset, dataSize);
3014 }
3015 if ((memOpnd->GetMemVaryType() == kNotVary) && IsImmediateOffsetOutOfRange(*memOpnd, dataSize)) {
3016 memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, dataSize);
3017 }
3018
3019 PrimType resultType = expr.GetPrimType();
3020 RegOperand &resOpnd = GetOrCreateResOperand(parent, symType);
3021 /* a local register variable defined with a specified register */
3022 if (symbol->GetAsmAttr() != UStrIdx(0) && symbol->GetStorageClass() != kScPstatic &&
3023 symbol->GetStorageClass() != kScFstatic) {
3024 std::string regDesp = GlobalTables::GetUStrTable().GetStringFromStrIdx(symbol->GetAsmAttr());
3025 RegOperand &specifiedOpnd = GetOrCreatePhysicalRegisterOperand(regDesp);
3026 return &specifiedOpnd;
3027 }
3028 memOpnd = memOpnd->IsOffsetMisaligned(dataSize) ? &ConstraintOffsetToSafeRegion(dataSize, *memOpnd) : memOpnd;
3029 SelectCopy(resOpnd, resultType, *memOpnd, symType);
3030 return &resOpnd;
3031 }
3032
SelectRegread(RegreadNode & expr)3033 RegOperand *AArch64CGFunc::SelectRegread(RegreadNode &expr)
3034 {
3035 PregIdx pregIdx = expr.GetRegIdx();
3036 if (IsSpecialPseudoRegister(pregIdx)) {
3037 /* if it is one of special registers */
3038 return &GetOrCreateSpecialRegisterOperand(-pregIdx, expr.GetPrimType());
3039 }
3040 RegOperand ® = GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx));
3041 if (GetOpndFromPregIdx(pregIdx) == nullptr) {
3042 SetPregIdx2Opnd(pregIdx, reg);
3043 }
3044 if (expr.GetPrimType() == PTY_ref) {
3045 reg.SetIsReference(true);
3046 AddReferenceReg(reg.GetRegisterNumber());
3047 }
3048 if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) {
3049 MemOperand *src = GetPseudoRegisterSpillMemoryOperand(pregIdx);
3050 MIRPreg *preg = GetFunction().GetPregTab()->PregFromPregIdx(pregIdx);
3051 PrimType stype = preg->GetPrimType();
3052 uint32 srcBitLength = GetPrimTypeSize(stype) * kBitsPerByte;
3053 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(srcBitLength, stype), reg, *src));
3054 }
3055 return ®
3056 }
3057
SelectAddrof(Operand & result,StImmOperand & stImm,FieldID field)3058 void AArch64CGFunc::SelectAddrof(Operand &result, StImmOperand &stImm, FieldID field)
3059 {
3060 const MIRSymbol *symbol = stImm.GetSymbol();
3061 if (symbol->GetStorageClass() == kScAuto) {
3062 SetStackProtectInfo(kAddrofStack);
3063 }
3064 if ((symbol->GetStorageClass() == kScAuto) || (symbol->GetStorageClass() == kScFormal)) {
3065 if (!CGOptions::IsQuiet()) {
3066 maple::LogInfo::MapleLogger(kLlErr)
3067 << "Warning: we expect AddrOf with StImmOperand is not used for local variables";
3068 }
3069 AArch64SymbolAlloc *symLoc =
3070 static_cast<AArch64SymbolAlloc *>(GetMemlayout()->GetSymAllocInfo(symbol->GetStIndex()));
3071 ImmOperand *offset = nullptr;
3072 if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsArgsStkPassed) {
3073 offset = &CreateImmOperand(GetBaseOffset(*symLoc) + stImm.GetOffset(), k64BitSize, false, kUnAdjustVary);
3074 } else if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsRefLocals) {
3075 auto it = immOpndsRequiringOffsetAdjustmentForRefloc.find(symLoc);
3076 if (it != immOpndsRequiringOffsetAdjustmentForRefloc.end()) {
3077 offset = (*it).second;
3078 } else {
3079 offset = &CreateImmOperand(GetBaseOffset(*symLoc) + stImm.GetOffset(), k64BitSize, false);
3080 immOpndsRequiringOffsetAdjustmentForRefloc[symLoc] = offset;
3081 }
3082 } else if (mirModule.IsJavaModule()) {
3083 auto it = immOpndsRequiringOffsetAdjustment.find(symLoc);
3084 if ((it != immOpndsRequiringOffsetAdjustment.end()) && (symbol->GetType()->GetPrimType() != PTY_agg)) {
3085 offset = (*it).second;
3086 } else {
3087 offset = &CreateImmOperand(GetBaseOffset(*symLoc) + stImm.GetOffset(), k64BitSize, false);
3088 if (symbol->GetType()->GetKind() != kTypeClass) {
3089 immOpndsRequiringOffsetAdjustment[symLoc] = offset;
3090 }
3091 }
3092 } else {
3093 /* Do not cache modified symbol location */
3094 offset = &CreateImmOperand(GetBaseOffset(*symLoc) + stImm.GetOffset(), k64BitSize, false);
3095 }
3096
3097 SelectAdd(result, *GetBaseReg(*symLoc), *offset, PTY_u64);
3098 if (GetCG()->GenerateVerboseCG()) {
3099 /* Add a comment */
3100 Insn *insn = GetCurBB()->GetLastInsn();
3101 std::string comm = "local/formal var: ";
3102 comm += symbol->GetName();
3103 insn->SetComment(comm);
3104 }
3105 } else if (symbol->IsThreadLocal()) {
3106 SelectAddrofThreadLocal(result, stImm);
3107 return;
3108 } else {
3109 Operand *srcOpnd = &result;
3110 if (!IsAfterRegAlloc()) {
3111 // Create a new vreg/preg for the upper bits of the address
3112 PregIdx pregIdx = GetFunction().GetPregTab()->CreatePreg(PTY_a64);
3113 MIRPreg *tmpPreg = GetFunction().GetPregTab()->PregFromPregIdx(pregIdx);
3114 regno_t vRegNO = NewVReg(kRegTyInt, GetPrimTypeSize(PTY_a64));
3115 RegOperand &tmpreg = GetOrCreateVirtualRegisterOperand(vRegNO);
3116
3117 // Register this vreg mapping
3118 RegisterVregMapping(vRegNO, pregIdx);
3119
3120 // Store rematerialization info in the preg
3121 tmpPreg->SetOp(OP_addrof);
3122 tmpPreg->rematInfo.sym = symbol;
3123 tmpPreg->fieldID = field;
3124 tmpPreg->addrUpper = true;
3125
3126 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrp, tmpreg, stImm));
3127 srcOpnd = &tmpreg;
3128 } else {
3129 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrp, result, stImm));
3130 }
3131 if (CGOptions::IsPIC() && symbol->NeedPIC()) {
3132 /* ldr x0, [x0, #:got_lo12:Ljava_2Flang_2FSystem_3B_7Cout] */
3133 OfstOperand &offset = CreateOfstOpnd(*stImm.GetSymbol(), stImm.GetOffset(), stImm.GetRelocs());
3134
3135 auto size = GetPointerSize() * kBitsPerByte;
3136 MemOperand &memOpnd = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, size, static_cast<RegOperand *>(srcOpnd),
3137 nullptr, &offset, nullptr);
3138 GetCurBB()->AppendInsn(
3139 GetInsnBuilder()->BuildInsn(size == k64BitSize ? MOP_xldr : MOP_wldr, result, memOpnd));
3140
3141 if (stImm.GetOffset() > 0) {
3142 ImmOperand &immOpnd = CreateImmOperand(stImm.GetOffset(), result.GetSize(), false);
3143 SelectAdd(result, result, immOpnd, PTY_u64);
3144 }
3145 } else {
3146 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrpl12, result, *srcOpnd, stImm));
3147 }
3148 }
3149 }
3150
SelectAddrof(Operand & result,MemOperand & memOpnd,FieldID field)3151 void AArch64CGFunc::SelectAddrof(Operand &result, MemOperand &memOpnd, FieldID field)
3152 {
3153 const MIRSymbol *symbol = memOpnd.GetSymbol();
3154 if (symbol->GetStorageClass() == kScAuto) {
3155 auto *offsetOpnd = static_cast<OfstOperand *>(memOpnd.GetOffsetImmediate());
3156 Operand &immOpnd = CreateImmOperand(offsetOpnd->GetOffsetValue(), PTY_u32, false);
3157 DEBUG_ASSERT(memOpnd.GetBaseRegister() != nullptr, "nullptr check");
3158 SelectAdd(result, *memOpnd.GetBaseRegister(), immOpnd, PTY_u32);
3159 SetStackProtectInfo(kAddrofStack);
3160 } else if (!IsAfterRegAlloc()) {
3161 // Create a new vreg/preg for the upper bits of the address
3162 PregIdx pregIdx = GetFunction().GetPregTab()->CreatePreg(PTY_a64);
3163 MIRPreg *tmpPreg = GetFunction().GetPregTab()->PregFromPregIdx(pregIdx);
3164 regno_t vRegNO = NewVReg(kRegTyInt, GetPrimTypeSize(PTY_a64));
3165 RegOperand &tmpreg = GetOrCreateVirtualRegisterOperand(vRegNO);
3166
3167 // Register this vreg mapping
3168 RegisterVregMapping(vRegNO, pregIdx);
3169
3170 // Store rematerialization info in the preg
3171 tmpPreg->SetOp(OP_addrof);
3172 tmpPreg->rematInfo.sym = symbol;
3173 tmpPreg->fieldID = field;
3174 tmpPreg->addrUpper = true;
3175
3176 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrp, tmpreg, memOpnd));
3177 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrpl12, result, tmpreg, memOpnd));
3178 } else {
3179 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrp, result, memOpnd));
3180 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrpl12, result, result, memOpnd));
3181 }
3182 }
3183
SelectAddrof(AddrofNode & expr,const BaseNode & parent,bool isAddrofoff)3184 Operand *AArch64CGFunc::SelectAddrof(AddrofNode &expr, const BaseNode &parent, bool isAddrofoff)
3185 {
3186 MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(expr.GetStIdx());
3187 int32 offset = 0;
3188 AddrofoffNode &addrofoffExpr = static_cast<AddrofoffNode &>(static_cast<BaseNode &>(expr));
3189 if (isAddrofoff) {
3190 offset = addrofoffExpr.offset;
3191 } else {
3192 if (expr.GetFieldID() != 0) {
3193 MIRStructType *structType = static_cast<MIRStructType *>(symbol->GetType());
3194 /* with array of structs, it is possible to have nullptr */
3195 if (structType != nullptr) {
3196 offset = GetBecommon().GetFieldOffset(*structType, expr.GetFieldID()).first;
3197 }
3198 }
3199 }
3200 if ((symbol->GetStorageClass() == kScFormal) && (symbol->GetSKind() == kStVar) &&
3201 ((!isAddrofoff && expr.GetFieldID() != 0) ||
3202 (GetBecommon().GetTypeSize(symbol->GetType()->GetTypeIndex().GetIdx()) > k16ByteSize))) {
3203 /*
3204 * Struct param is copied on the stack by caller if struct size > 16.
3205 * Else if size < 16 then struct param is copied into one or two registers.
3206 */
3207 RegOperand *stackAddr = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
3208 /* load the base address of the struct copy from stack. */
3209 SelectAddrof(*stackAddr, CreateStImmOperand(*symbol, 0, 0));
3210 Operand *structAddr;
3211 if (GetBecommon().GetTypeSize(symbol->GetType()->GetTypeIndex().GetIdx()) <= k16ByteSize) {
3212 isAggParamInReg = true;
3213 structAddr = stackAddr;
3214 } else {
3215 OfstOperand *offopnd = &CreateOfstOpnd(0, k32BitSize);
3216 MemOperand *mo = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, GetPointerSize() * kBitsPerByte, stackAddr,
3217 nullptr, offopnd, nullptr);
3218 structAddr = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
3219 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xldr, *structAddr, *mo));
3220 }
3221 if (offset == 0) {
3222 return structAddr;
3223 } else {
3224 /* add the struct offset to the base address */
3225 Operand *result = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
3226 ImmOperand *imm = &CreateImmOperand(PTY_a64, offset);
3227 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *result, *structAddr, *imm));
3228 return result;
3229 }
3230 }
3231 PrimType ptype = expr.GetPrimType();
3232 Operand &result = GetOrCreateResOperand(parent, ptype);
3233 if (symbol->IsReflectionClassInfo() && !symbol->IsReflectionArrayClassInfo() && !GetCG()->IsLibcore()) {
3234 /*
3235 * Turn addrof __cinf_X into a load of _PTR__cinf_X
3236 * adrp x1, _PTR__cinf_Ljava_2Flang_2FSystem_3B
3237 * ldr x1, [x1, #:lo12:_PTR__cinf_Ljava_2Flang_2FSystem_3B]
3238 */
3239 std::string ptrName = namemangler::kPtrPrefixStr + symbol->GetName();
3240 MIRType *ptrType = GlobalTables::GetTypeTable().GetPtr();
3241 symbol = GetMirModule().GetMIRBuilder()->GetOrCreateGlobalDecl(ptrName, *ptrType);
3242 symbol->SetStorageClass(kScFstatic);
3243
3244 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_adrp_ldr, result, CreateStImmOperand(*symbol, 0, 0)));
3245 /* make it un rematerializable. */
3246 MIRPreg *preg = GetPseudoRegFromVirtualRegNO(static_cast<RegOperand &>(result).GetRegisterNumber());
3247 if (preg) {
3248 preg->SetOp(OP_undef);
3249 }
3250 return &result;
3251 }
3252
3253 SelectAddrof(result, CreateStImmOperand(*symbol, offset, 0), isAddrofoff ? 0 : expr.GetFieldID());
3254 return &result;
3255 }
3256
SelectAddrofoff(AddrofoffNode & expr,const BaseNode & parent)3257 Operand *AArch64CGFunc::SelectAddrofoff(AddrofoffNode &expr, const BaseNode &parent)
3258 {
3259 return SelectAddrof(static_cast<AddrofNode &>(static_cast<BaseNode &>(expr)), parent, true);
3260 }
3261
SelectAddrofFunc(AddroffuncNode & expr,const BaseNode & parent)3262 Operand &AArch64CGFunc::SelectAddrofFunc(AddroffuncNode &expr, const BaseNode &parent)
3263 {
3264 uint32 instrSize = static_cast<uint32>(expr.SizeOfInstr());
3265 PrimType primType = (instrSize == k8ByteSize)
3266 ? PTY_u64
3267 : (instrSize == k4ByteSize) ? PTY_u32 : (instrSize == k2ByteSize) ? PTY_u16 : PTY_u8;
3268 Operand &operand = GetOrCreateResOperand(parent, primType);
3269 MIRFunction *mirFunction = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(expr.GetPUIdx());
3270 SelectAddrof(operand, CreateStImmOperand(*mirFunction->GetFuncSymbol(), 0, 0));
3271 return operand;
3272 }
3273
3274 /* For an entire aggregate that can fit inside a single 8 byte register. */
GetDestTypeFromAggSize(uint32 bitSize) const3275 PrimType AArch64CGFunc::GetDestTypeFromAggSize(uint32 bitSize) const
3276 {
3277 PrimType primType;
3278 switch (bitSize) {
3279 case k8BitSize: {
3280 primType = PTY_u8;
3281 break;
3282 }
3283 case k16BitSize: {
3284 primType = PTY_u16;
3285 break;
3286 }
3287 case k32BitSize: {
3288 primType = PTY_u32;
3289 break;
3290 }
3291 case k64BitSize: {
3292 primType = PTY_u64;
3293 break;
3294 }
3295 default:
3296 CHECK_FATAL(false, "aggregate of unhandled size");
3297 }
3298 return primType;
3299 }
3300
SelectAddrofLabel(AddroflabelNode & expr,const BaseNode & parent)3301 Operand &AArch64CGFunc::SelectAddrofLabel(AddroflabelNode &expr, const BaseNode &parent)
3302 {
3303 /* adrp reg, label-id */
3304 uint32 instrSize = static_cast<uint32>(expr.SizeOfInstr());
3305 PrimType primType = (instrSize == k8ByteSize)
3306 ? PTY_u64
3307 : (instrSize == k4ByteSize) ? PTY_u32 : (instrSize == k2ByteSize) ? PTY_u16 : PTY_u8;
3308 Operand &dst = GetOrCreateResOperand(parent, primType);
3309 Operand &immOpnd = CreateImmOperand(expr.GetOffset(), k64BitSize, false);
3310 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_adrp_label, dst, immOpnd));
3311 return dst;
3312 }
3313
SelectIreadoff(const BaseNode & parent,IreadoffNode & ireadoff)3314 Operand *AArch64CGFunc::SelectIreadoff(const BaseNode &parent, IreadoffNode &ireadoff)
3315 {
3316 auto offset = ireadoff.GetOffset();
3317 auto primType = ireadoff.GetPrimType();
3318 auto bitSize = GetPrimTypeBitSize(primType);
3319 auto *baseAddr = ireadoff.Opnd(0);
3320 auto *result = &CreateRegisterOperandOfType(primType);
3321 auto *addrOpnd = HandleExpr(ireadoff, *baseAddr);
3322 auto &memOpnd = CreateMemOpnd(LoadIntoRegister(*addrOpnd, PTY_a64), offset, bitSize);
3323 auto mop = PickLdInsn(bitSize, primType);
3324 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mop, *result, memOpnd));
3325 return result;
3326 }
3327
GenLmbcParamLoad(int32 offset,uint32 byteSize,RegType regType,PrimType primType,AArch64reg baseRegno)3328 RegOperand *AArch64CGFunc::GenLmbcParamLoad(int32 offset, uint32 byteSize, RegType regType, PrimType primType,
3329 AArch64reg baseRegno)
3330 {
3331 MemOperand *memOpnd = GenLmbcFpMemOperand(offset, byteSize, baseRegno);
3332 RegOperand *result = &GetOrCreateVirtualRegisterOperand(NewVReg(regType, byteSize));
3333 MOperator mOp = PickLdInsn(byteSize * kBitsPerByte, primType);
3334 Insn &load = GetInsnBuilder()->BuildInsn(mOp, *result, *memOpnd);
3335 GetCurBB()->AppendInsn(load);
3336 return result;
3337 }
3338
LmbcStructReturnLoad(int32 offset)3339 RegOperand *AArch64CGFunc::LmbcStructReturnLoad(int32 offset)
3340 {
3341 RegOperand *result = nullptr;
3342 MIRFunction &func = GetFunction();
3343 CHECK_FATAL(func.IsReturnStruct(), "LmbcStructReturnLoad: not struct return");
3344 MIRType *ty = func.GetReturnType();
3345 uint32 sz = GetBecommon().GetTypeSize(ty->GetTypeIndex());
3346 uint32 fpSize;
3347 uint32 numFpRegs = FloatParamRegRequired(static_cast<MIRStructType *>(ty), fpSize);
3348 if (numFpRegs > 0) {
3349 PrimType pType = (fpSize <= k4ByteSize) ? PTY_f32 : PTY_f64;
3350 for (int32 i = (numFpRegs - kOneRegister); i > 0; --i) {
3351 result = GenLmbcParamLoad(offset + (i * static_cast<int32>(fpSize)), fpSize, kRegTyFloat, pType);
3352 AArch64reg regNo = static_cast<AArch64reg>(V0 + static_cast<uint32>(i));
3353 RegOperand *reg = &GetOrCreatePhysicalRegisterOperand(regNo, fpSize * kBitsPerByte, kRegTyFloat);
3354 SelectCopy(*reg, pType, *result, pType);
3355 Insn &pseudo = GetInsnBuilder()->BuildInsn(MOP_pseudo_ret_float, *reg);
3356 GetCurBB()->AppendInsn(pseudo);
3357 }
3358 result = GenLmbcParamLoad(offset, fpSize, kRegTyFloat, pType);
3359 } else if (sz <= k4ByteSize) {
3360 result = GenLmbcParamLoad(offset, k4ByteSize, kRegTyInt, PTY_u32);
3361 } else if (sz <= k8ByteSize) {
3362 result = GenLmbcParamLoad(offset, k8ByteSize, kRegTyInt, PTY_i64);
3363 } else if (sz <= k16ByteSize) {
3364 result = GenLmbcParamLoad(offset + k8ByteSizeInt, k8ByteSize, kRegTyInt, PTY_i64);
3365 RegOperand *r1 = &GetOrCreatePhysicalRegisterOperand(R1, k8ByteSize * kBitsPerByte, kRegTyInt);
3366 SelectCopy(*r1, PTY_i64, *result, PTY_i64);
3367 Insn &pseudo = GetInsnBuilder()->BuildInsn(MOP_pseudo_ret_int, *r1);
3368 GetCurBB()->AppendInsn(pseudo);
3369 result = GenLmbcParamLoad(offset, k8ByteSize, kRegTyInt, PTY_i64);
3370 }
3371 return result;
3372 }
3373
SelectIreadfpoff(const BaseNode & parent,IreadFPoffNode & ireadoff)3374 Operand *AArch64CGFunc::SelectIreadfpoff(const BaseNode &parent, IreadFPoffNode &ireadoff)
3375 {
3376 uint32 offset = ireadoff.GetOffset();
3377 PrimType primType = ireadoff.GetPrimType();
3378 uint32 bytelen = GetPrimTypeSize(primType);
3379 uint32 bitlen = bytelen * kBitsPerByte;
3380 RegType regty = GetRegTyFromPrimTy(primType);
3381 RegOperand *result = nullptr;
3382 if (offset >= 0) {
3383 LmbcFormalParamInfo *info = GetLmbcFormalParamInfo(static_cast<uint32>(offset));
3384 if (info->GetPrimType() == PTY_agg) {
3385 if (info->IsOnStack()) {
3386 result = GenLmbcParamLoad(info->GetOnStackOffset(), GetPrimTypeSize(PTY_a64), kRegTyInt, PTY_a64);
3387 regno_t baseRegno = result->GetRegisterNumber();
3388 result = GenLmbcParamLoad(offset - static_cast<int32>(info->GetOffset()), bytelen, regty, primType,
3389 (AArch64reg)baseRegno);
3390 } else if (primType == PTY_agg) {
3391 CHECK_FATAL(parent.GetOpCode() == OP_regassign, "SelectIreadfpoff of agg");
3392 result = LmbcStructReturnLoad(offset);
3393 } else {
3394 result = GenLmbcParamLoad(offset, bytelen, regty, primType);
3395 }
3396 } else {
3397 CHECK_FATAL(primType == info->GetPrimType(), "Incorrect primtype");
3398 CHECK_FATAL(offset == info->GetOffset(), "Incorrect offset");
3399 if (info->GetRegNO() == 0 || !info->HasRegassign()) {
3400 result = GenLmbcParamLoad(offset, bytelen, regty, primType);
3401 } else {
3402 result = &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(info->GetRegNO()), bitlen, regty);
3403 }
3404 }
3405 } else {
3406 if (primType == PTY_agg) {
3407 CHECK_FATAL(parent.GetOpCode() == OP_regassign, "SelectIreadfpoff of agg");
3408 result = LmbcStructReturnLoad(offset);
3409 } else {
3410 result = GenLmbcParamLoad(offset, bytelen, regty, primType);
3411 }
3412 }
3413 return result;
3414 }
3415
SelectIread(const BaseNode & parent,IreadNode & expr,int extraOffset,PrimType finalBitFieldDestType)3416 Operand *AArch64CGFunc::SelectIread(const BaseNode &parent, IreadNode &expr, int extraOffset,
3417 PrimType finalBitFieldDestType)
3418 {
3419 int32 offset = 0;
3420 MIRType *type = GlobalTables::GetTypeTable().GetTypeFromTyIdx(expr.GetTyIdx());
3421 MIRPtrType *pointerType = static_cast<MIRPtrType *>(type);
3422 DEBUG_ASSERT(pointerType != nullptr, "expect a pointer type at iread node");
3423 MIRType *pointedType = nullptr;
3424 bool isRefField = false;
3425 AArch64isa::MemoryOrdering memOrd = AArch64isa::kMoNone;
3426
3427 if (expr.GetFieldID() != 0) {
3428 MIRType *pointedTy = GlobalTables::GetTypeTable().GetTypeFromTyIdx(pointerType->GetPointedTyIdx());
3429 MIRStructType *structType = nullptr;
3430 if (pointedTy->GetKind() != kTypeJArray) {
3431 structType = static_cast<MIRStructType *>(pointedTy);
3432 } else {
3433 /* it's a Jarray type. using it's parent's field info: java.lang.Object */
3434 structType = static_cast<MIRJarrayType *>(pointedTy)->GetParentType();
3435 }
3436
3437 DEBUG_ASSERT(structType != nullptr, "SelectIread: non-zero fieldID for non-structure");
3438 pointedType = structType->GetFieldType(expr.GetFieldID());
3439 offset = GetBecommon().GetFieldOffset(*structType, expr.GetFieldID()).first;
3440 isRefField = GetBecommon().IsRefField(*structType, expr.GetFieldID());
3441 } else {
3442 pointedType = GetPointedToType(*pointerType);
3443 if (GetFunction().IsJava() && (pointedType->GetKind() == kTypePointer)) {
3444 MIRType *nextPointedType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(
3445 static_cast<MIRPtrType *>(pointedType)->GetPointedTyIdx());
3446 if (nextPointedType->GetKind() != kTypeScalar) {
3447 isRefField = true; /* read from an object array, or an high-dimentional array */
3448 }
3449 }
3450 }
3451
3452 RegType regType = GetRegTyFromPrimTy(expr.GetPrimType());
3453 uint32 regSize = GetPrimTypeSize(expr.GetPrimType());
3454 if (expr.GetFieldID() == 0 && pointedType->GetPrimType() == PTY_agg) {
3455 /* Maple IR can passing small struct to be loaded into a single register. */
3456 if (regType == kRegTyFloat) {
3457 /* regsize is correct */
3458 } else {
3459 uint32 sz = GetBecommon().GetTypeSize(pointedType->GetTypeIndex().GetIdx());
3460 regSize = (sz <= k4ByteSize) ? k4ByteSize : k8ByteSize;
3461 }
3462 } else if (regSize < k4ByteSize) {
3463 regSize = k4ByteSize; /* 32-bit */
3464 }
3465 Operand *result = nullptr;
3466 if (parent.GetOpCode() == OP_eval) {
3467 /* regSize << 3, that is regSize * 8, change bytes to bits */
3468 result = &GetZeroOpnd(regSize << 3);
3469 } else {
3470 result = &GetOrCreateResOperand(parent, expr.GetPrimType());
3471 }
3472
3473 PrimType destType = pointedType->GetPrimType();
3474
3475 uint32 bitSize = 0;
3476 if ((pointedType->GetKind() == kTypeStructIncomplete) || (pointedType->GetKind() == kTypeClassIncomplete) ||
3477 (pointedType->GetKind() == kTypeInterfaceIncomplete)) {
3478 bitSize = GetPrimTypeBitSize(expr.GetPrimType());
3479 maple::LogInfo::MapleLogger(kLlErr) << "Warning: objsize is zero! \n";
3480 } else {
3481 if (pointedType->IsStructType()) {
3482 MIRStructType *structType = static_cast<MIRStructType *>(pointedType);
3483 /* size << 3, that is size * 8, change bytes to bits */
3484 bitSize = std::min(structType->GetSize(), static_cast<size_t>(GetPointerSize())) << 3;
3485 } else {
3486 bitSize = GetPrimTypeBitSize(destType);
3487 }
3488 if (regType == kRegTyFloat) {
3489 destType = expr.GetPrimType();
3490 bitSize = GetPrimTypeBitSize(destType);
3491 } else if (destType == PTY_agg) {
3492 switch (bitSize) {
3493 case k8BitSize:
3494 destType = PTY_u8;
3495 break;
3496 case k16BitSize:
3497 destType = PTY_u16;
3498 break;
3499 case k32BitSize:
3500 destType = PTY_u32;
3501 break;
3502 case k64BitSize:
3503 destType = PTY_u64;
3504 break;
3505 default:
3506 destType = PTY_u64; // when eval agg . a way to round up
3507 break;
3508 }
3509 }
3510 }
3511
3512 MemOperand *memOpnd =
3513 CreateMemOpndOrNull(destType, expr, *expr.Opnd(0), static_cast<int64>(offset) + extraOffset, memOrd);
3514 if (aggParamReg != nullptr) {
3515 isAggParamInReg = false;
3516 return aggParamReg;
3517 }
3518 DEBUG_ASSERT(memOpnd != nullptr, "memOpnd should not be nullptr");
3519 if (isVolLoad && (memOpnd->GetAddrMode() == MemOperand::kAddrModeBOi)) {
3520 memOrd = AArch64isa::kMoAcquire;
3521 isVolLoad = false;
3522 }
3523
3524 memOpnd = memOpnd->IsOffsetMisaligned(bitSize) ? &ConstraintOffsetToSafeRegion(bitSize, *memOpnd) : memOpnd;
3525 if (memOrd == AArch64isa::kMoNone) {
3526 MOperator mOp = 0;
3527 if (finalBitFieldDestType == kPtyInvalid) {
3528 mOp = PickLdInsn(bitSize, destType);
3529 } else {
3530 mOp = PickLdInsn(GetPrimTypeBitSize(finalBitFieldDestType), finalBitFieldDestType);
3531 }
3532 if ((memOpnd->GetMemVaryType() == kNotVary) && !IsOperandImmValid(mOp, memOpnd, 1)) {
3533 memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, bitSize);
3534 }
3535 Insn &insn = GetInsnBuilder()->BuildInsn(mOp, *result, *memOpnd);
3536 if (parent.GetOpCode() == OP_eval && result->IsRegister() &&
3537 static_cast<RegOperand *>(result)->GetRegisterNumber() == RZR) {
3538 insn.SetComment("null-check");
3539 }
3540 GetCurBB()->AppendInsn(insn);
3541
3542 if (parent.op != OP_eval) {
3543 const InsnDesc *md = &AArch64CG::kMd[insn.GetMachineOpcode()];
3544 auto *prop = md->GetOpndDes(0);
3545 if ((prop->GetSize()) < insn.GetOperand(0).GetSize()) {
3546 switch (destType) {
3547 case PTY_i8:
3548 mOp = MOP_xsxtb64;
3549 break;
3550 case PTY_i16:
3551 mOp = MOP_xsxth64;
3552 break;
3553 case PTY_i32:
3554 mOp = MOP_xsxtw64;
3555 break;
3556 case PTY_u8:
3557 mOp = MOP_xuxtb32;
3558 break;
3559 case PTY_u16:
3560 mOp = MOP_xuxth32;
3561 break;
3562 case PTY_u32:
3563 mOp = MOP_xuxtw64;
3564 break;
3565 default:
3566 break;
3567 }
3568 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, insn.GetOperand(0), insn.GetOperand(0)));
3569 }
3570 }
3571 } else {
3572 if ((memOpnd->GetMemVaryType() == kNotVary) && IsImmediateOffsetOutOfRange(*memOpnd, bitSize)) {
3573 memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, bitSize);
3574 }
3575 AArch64CGFunc::SelectLoadAcquire(*result, destType, *memOpnd, destType, memOrd, false);
3576 }
3577 GetCurBB()->GetLastInsn()->MarkAsAccessRefField(isRefField);
3578 return result;
3579 }
3580
SelectIntConst(const MIRIntConst & intConst)3581 Operand *AArch64CGFunc::SelectIntConst(const MIRIntConst &intConst)
3582 {
3583 return &CreateImmOperand(intConst.GetExtValue(), GetPrimTypeSize(intConst.GetType().GetPrimType()) * kBitsPerByte,
3584 false);
3585 }
3586
3587 template <typename T>
SelectLiteral(T * c,MIRFunction * func,uint32 labelIdx,AArch64CGFunc * cgFunc)3588 Operand *SelectLiteral(T *c, MIRFunction *func, uint32 labelIdx, AArch64CGFunc *cgFunc)
3589 {
3590 MIRSymbol *st = func->GetSymTab()->CreateSymbol(kScopeLocal);
3591 std::string lblStr(".LB_");
3592 MIRSymbol *funcSt = GlobalTables::GetGsymTable().GetSymbolFromStidx(func->GetStIdx().Idx());
3593 std::string funcName = funcSt->GetName();
3594 lblStr += funcName;
3595 lblStr += std::to_string(labelIdx);
3596 st->SetNameStrIdx(lblStr);
3597 st->SetStorageClass(kScPstatic);
3598 st->SetSKind(kStConst);
3599 st->SetKonst(c);
3600 cgFunc->SetLocalSymLabelIndex(*st, labelIdx);
3601 PrimType primType = c->GetType().GetPrimType();
3602 st->SetTyIdx(TyIdx(primType));
3603 uint32 typeBitSize = GetPrimTypeBitSize(primType);
3604
3605 if ((T::GetPrimType() == PTY_f32 || T::GetPrimType() == PTY_f64)) {
3606 return static_cast<Operand *>(&cgFunc->GetOrCreateMemOpnd(*st, 0, typeBitSize));
3607 } else {
3608 CHECK_FATAL(false, "Unsupported const type");
3609 }
3610 return nullptr;
3611 }
3612
HandleFmovImm(PrimType stype,int64 val,MIRConst & mirConst,const BaseNode & parent)3613 Operand *AArch64CGFunc::HandleFmovImm(PrimType stype, int64 val, MIRConst &mirConst, const BaseNode &parent)
3614 {
3615 Operand *result;
3616 bool is64Bits = (GetPrimTypeBitSize(stype) == k64BitSize);
3617 uint64 canRepreset = is64Bits ? (val & 0xffffffffffff) : (val & 0x7ffff);
3618 uint32 val1 = is64Bits ? (val >> 61) & 0x3 : (val >> 29) & 0x3;
3619 uint32 val2 = is64Bits ? (val >> 54) & 0xff : (val >> 25) & 0x1f;
3620 bool isSame = is64Bits ? ((val2 == 0) || (val2 == 0xff)) : ((val2 == 0) || (val2 == 0x1f));
3621 canRepreset = (canRepreset == 0) && ((val1 & 0x1) ^ ((val1 & 0x2) >> 1)) && isSame;
3622 if (canRepreset) {
3623 uint64 temp1 = is64Bits ? (val >> 63) << 7 : (val >> 31) << 7;
3624 uint64 temp2 = is64Bits ? val >> 48 : val >> 19;
3625 int64 imm8 = (temp2 & 0x7f) | temp1;
3626 Operand *newOpnd0 = &CreateImmOperand(imm8, k8BitSize, true, kNotVary, true);
3627 result = &GetOrCreateResOperand(parent, stype);
3628 MOperator mopFmov = (is64Bits ? MOP_xdfmovri : MOP_wsfmovri);
3629 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopFmov, *result, *newOpnd0));
3630 } else {
3631 if (is64Bits) { // For DoubleConst, use ldr .literal
3632 uint32 labelIdxTmp = GetLabelIdx();
3633 result = SelectLiteral(static_cast<MIRDoubleConst *>(&mirConst), &GetFunction(), labelIdxTmp++, this);
3634 SetLabelIdx(labelIdxTmp);
3635 return result;
3636 }
3637 Operand *newOpnd0 = &CreateImmOperand(val, GetPrimTypeSize(stype) * kBitsPerByte, false);
3638 PrimType itype = (stype == PTY_f32) ? PTY_i32 : PTY_i64;
3639 RegOperand ®Opnd = LoadIntoRegister(*newOpnd0, itype);
3640
3641 result = &GetOrCreateResOperand(parent, stype);
3642 MOperator mopFmov = (is64Bits ? MOP_xvmovdr : MOP_xvmovsr);
3643 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopFmov, *result, regOpnd));
3644 }
3645 return result;
3646 }
3647
SelectFloatConst(MIRFloatConst & floatConst,const BaseNode & parent)3648 Operand *AArch64CGFunc::SelectFloatConst(MIRFloatConst &floatConst, const BaseNode &parent)
3649 {
3650 PrimType stype = floatConst.GetType().GetPrimType();
3651 int32 val = floatConst.GetIntValue();
3652 /* according to aarch64 encoding format, convert int to float expression */
3653 Operand *result;
3654 result = HandleFmovImm(stype, val, floatConst, parent);
3655 return result;
3656 }
3657
SelectDoubleConst(MIRDoubleConst & doubleConst,const BaseNode & parent)3658 Operand *AArch64CGFunc::SelectDoubleConst(MIRDoubleConst &doubleConst, const BaseNode &parent)
3659 {
3660 PrimType stype = doubleConst.GetType().GetPrimType();
3661 int64 val = doubleConst.GetIntValue();
3662 /* according to aarch64 encoding format, convert int to float expression */
3663 Operand *result;
3664 result = HandleFmovImm(stype, val, doubleConst, parent);
3665 return result;
3666 }
3667
3668 template <typename T>
SelectStrLiteral(T & c,AArch64CGFunc & cgFunc)3669 Operand *SelectStrLiteral(T &c, AArch64CGFunc &cgFunc)
3670 {
3671 std::string labelStr;
3672 if (c.GetKind() == kConstStrConst) {
3673 labelStr += ".LUstr_";
3674 } else if (c.GetKind() == kConstStr16Const) {
3675 labelStr += ".LUstr16_";
3676 } else {
3677 CHECK_FATAL(false, "Unsupported literal type");
3678 }
3679 labelStr += std::to_string(c.GetValue());
3680
3681 MIRSymbol *labelSym =
3682 GlobalTables::GetGsymTable().GetSymbolFromStrIdx(GlobalTables::GetStrTable().GetStrIdxFromName(labelStr));
3683 if (labelSym == nullptr) {
3684 labelSym = cgFunc.GetMirModule().GetMIRBuilder()->CreateGlobalDecl(labelStr, c.GetType());
3685 labelSym->SetStorageClass(kScFstatic);
3686 labelSym->SetSKind(kStConst);
3687 /* c may be local, we need a global node here */
3688 labelSym->SetKonst(cgFunc.NewMirConst(c));
3689 }
3690
3691 if (c.GetPrimType() == PTY_ptr) {
3692 StImmOperand &stOpnd = cgFunc.CreateStImmOperand(*labelSym, 0, 0);
3693 RegOperand &addrOpnd = cgFunc.CreateRegisterOperandOfType(PTY_a64);
3694 cgFunc.SelectAddrof(addrOpnd, stOpnd);
3695 return &addrOpnd;
3696 }
3697 CHECK_FATAL(false, "Unsupported const string type");
3698 return nullptr;
3699 }
3700
SelectStrConst(MIRStrConst & strConst)3701 Operand *AArch64CGFunc::SelectStrConst(MIRStrConst &strConst)
3702 {
3703 return SelectStrLiteral(strConst, *this);
3704 }
3705
SelectStr16Const(MIRStr16Const & str16Const)3706 Operand *AArch64CGFunc::SelectStr16Const(MIRStr16Const &str16Const)
3707 {
3708 return SelectStrLiteral(str16Const, *this);
3709 }
3710
AppendInstructionTo(Insn & i,CGFunc & f)3711 static inline void AppendInstructionTo(Insn &i, CGFunc &f)
3712 {
3713 f.GetCurBB()->AppendInsn(i);
3714 }
3715
3716 /*
3717 * Returns the number of leading 0-bits in x, starting at the most significant bit position.
3718 * If x is 0, the result is -1.
3719 */
GetHead0BitNum(int64 val)3720 static int32 GetHead0BitNum(int64 val)
3721 {
3722 uint32 bitNum = 0;
3723 for (; bitNum < k64BitSize; bitNum++) {
3724 if ((0x8000000000000000ULL >> static_cast<uint32>(bitNum)) & static_cast<uint64>(val)) {
3725 break;
3726 }
3727 }
3728 if (bitNum == k64BitSize) {
3729 return -1;
3730 }
3731 return bitNum;
3732 }
3733
3734 /*
3735 * Returns the number of trailing 0-bits in x, starting at the least significant bit position.
3736 * If x is 0, the result is -1.
3737 */
GetTail0BitNum(int64 val)3738 static int32 GetTail0BitNum(int64 val)
3739 {
3740 uint32 bitNum = 0;
3741 for (; bitNum < k64BitSize; bitNum++) {
3742 if ((static_cast<uint64>(1) << static_cast<uint32>(bitNum)) & static_cast<uint64>(val)) {
3743 break;
3744 }
3745 }
3746 if (bitNum == k64BitSize) {
3747 return -1;
3748 }
3749 return bitNum;
3750 }
3751
3752 /*
3753 * If the input integer is power of 2, return log2(input)
3754 * else return -1
3755 */
GetLog2(uint64 val)3756 static inline int32 GetLog2(uint64 val)
3757 {
3758 if (__builtin_popcountll(val) == 1) {
3759 return __builtin_ffsll(static_cast<int64>(val)) - 1;
3760 }
3761 return -1;
3762 }
3763
PickJmpInsn(Opcode brOp,Opcode cmpOp,bool isFloat,bool isSigned) const3764 MOperator AArch64CGFunc::PickJmpInsn(Opcode brOp, Opcode cmpOp, bool isFloat, bool isSigned) const
3765 {
3766 switch (cmpOp) {
3767 case OP_ne:
3768 return (brOp == OP_brtrue) ? MOP_bne : MOP_beq;
3769 case OP_eq:
3770 return (brOp == OP_brtrue) ? MOP_beq : MOP_bne;
3771 case OP_lt:
3772 return (brOp == OP_brtrue) ? (isSigned ? MOP_blt : MOP_blo)
3773 : (isFloat ? MOP_bpl : (isSigned ? MOP_bge : MOP_bhs));
3774 case OP_le:
3775 return (brOp == OP_brtrue) ? (isSigned ? MOP_ble : MOP_bls)
3776 : (isFloat ? MOP_bhi : (isSigned ? MOP_bgt : MOP_bhi));
3777 case OP_gt:
3778 return (brOp == OP_brtrue) ? (isFloat ? MOP_bgt : (isSigned ? MOP_bgt : MOP_bhi))
3779 : (isSigned ? MOP_ble : MOP_bls);
3780 case OP_ge:
3781 return (brOp == OP_brtrue) ? (isFloat ? MOP_bpl : (isSigned ? MOP_bge : MOP_bhs))
3782 : (isSigned ? MOP_blt : MOP_blo);
3783 default:
3784 CHECK_FATAL(false, "PickJmpInsn error");
3785 }
3786 }
3787
GenerateCompareWithZeroInstruction(Opcode jmpOp,Opcode cmpOp,bool is64Bits,PrimType primType,LabelOperand & targetOpnd,Operand & opnd0)3788 bool AArch64CGFunc::GenerateCompareWithZeroInstruction(Opcode jmpOp, Opcode cmpOp, bool is64Bits, PrimType primType,
3789 LabelOperand &targetOpnd, Operand &opnd0)
3790 {
3791 bool finish = true;
3792 MOperator mOpCode = MOP_undef;
3793 switch (cmpOp) {
3794 case OP_ne: {
3795 if (jmpOp == OP_brtrue) {
3796 mOpCode = is64Bits ? MOP_xcbnz : MOP_wcbnz;
3797 } else {
3798 mOpCode = is64Bits ? MOP_xcbz : MOP_wcbz;
3799 }
3800 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, opnd0, targetOpnd));
3801 break;
3802 }
3803 case OP_eq: {
3804 if (jmpOp == OP_brtrue) {
3805 mOpCode = is64Bits ? MOP_xcbz : MOP_wcbz;
3806 } else {
3807 mOpCode = is64Bits ? MOP_xcbnz : MOP_wcbnz;
3808 }
3809 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, opnd0, targetOpnd));
3810 break;
3811 }
3812 /*
3813 * TBZ/TBNZ instruction have a range of +/-32KB, need to check if the jump target is reachable in a later
3814 * phase. If the branch target is not reachable, then we change tbz/tbnz into combination of ubfx and
3815 * cbz/cbnz, which will clobber one extra register. With LSRA under O2, we can use of the reserved registers
3816 * for that purpose.
3817 */
3818 case OP_lt: {
3819 if (primType == PTY_u64 || primType == PTY_u32) {
3820 return false;
3821 }
3822 ImmOperand &signBit =
3823 CreateImmOperand(is64Bits ? kHighestBitOf64Bits : kHighestBitOf32Bits, k8BitSize, false);
3824 if (jmpOp == OP_brtrue) {
3825 mOpCode = is64Bits ? MOP_xtbnz : MOP_wtbnz;
3826 } else {
3827 mOpCode = is64Bits ? MOP_xtbz : MOP_wtbz;
3828 }
3829 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, opnd0, signBit, targetOpnd));
3830 break;
3831 }
3832 case OP_ge: {
3833 if (primType == PTY_u64 || primType == PTY_u32) {
3834 return false;
3835 }
3836 ImmOperand &signBit =
3837 CreateImmOperand(is64Bits ? kHighestBitOf64Bits : kHighestBitOf32Bits, k8BitSize, false);
3838 if (jmpOp == OP_brtrue) {
3839 mOpCode = is64Bits ? MOP_xtbz : MOP_wtbz;
3840 } else {
3841 mOpCode = is64Bits ? MOP_xtbnz : MOP_wtbnz;
3842 }
3843 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, opnd0, signBit, targetOpnd));
3844 break;
3845 }
3846 default:
3847 finish = false;
3848 break;
3849 }
3850 return finish;
3851 }
3852
SelectIgoto(Operand * opnd0)3853 void AArch64CGFunc::SelectIgoto(Operand *opnd0)
3854 {
3855 Operand *srcOpnd = opnd0;
3856 if (opnd0->GetKind() == Operand::kOpdMem) {
3857 Operand *dst = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
3858 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xldr, *dst, *opnd0));
3859 srcOpnd = dst;
3860 }
3861 GetCurBB()->SetKind(BB::kBBIgoto);
3862 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xbr, *srcOpnd));
3863 }
3864
SelectCondGoto(LabelOperand & targetOpnd,Opcode jmpOp,Opcode cmpOp,Operand & origOpnd0,Operand & origOpnd1,PrimType primType,bool signedCond)3865 void AArch64CGFunc::SelectCondGoto(LabelOperand &targetOpnd, Opcode jmpOp, Opcode cmpOp, Operand &origOpnd0,
3866 Operand &origOpnd1, PrimType primType, bool signedCond)
3867 {
3868 Operand *opnd0 = &origOpnd0;
3869 Operand *opnd1 = &origOpnd1;
3870 opnd0 = &LoadIntoRegister(origOpnd0, primType);
3871
3872 bool is64Bits = GetPrimTypeBitSize(primType) == k64BitSize;
3873 bool isFloat = IsPrimitiveFloat(primType);
3874 Operand &rflag = GetOrCreateRflag();
3875 if (isFloat) {
3876 opnd1 = &LoadIntoRegister(origOpnd1, primType);
3877 MOperator mOp =
3878 is64Bits ? MOP_dcmperr : ((GetPrimTypeBitSize(primType) == k32BitSize) ? MOP_scmperr : MOP_hcmperr);
3879 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, rflag, *opnd0, *opnd1));
3880 } else {
3881 bool isImm = ((origOpnd1.GetKind() == Operand::kOpdImmediate) || (origOpnd1.GetKind() == Operand::kOpdOffset));
3882 if ((origOpnd1.GetKind() != Operand::kOpdRegister) && !isImm) {
3883 opnd1 = &SelectCopy(origOpnd1, primType, primType);
3884 }
3885 MOperator mOp = is64Bits ? MOP_xcmprr : MOP_wcmprr;
3886
3887 if (isImm) {
3888 /* Special cases, i.e., comparing with zero
3889 * Do not perform optimization for C, unlike Java which has no unsigned int.
3890 */
3891 if (static_cast<ImmOperand *>(opnd1)->IsZero() &&
3892 (Globals::GetInstance()->GetOptimLevel() > CGOptions::kLevel0)) {
3893 bool finish = GenerateCompareWithZeroInstruction(jmpOp, cmpOp, is64Bits, primType, targetOpnd, *opnd0);
3894 if (finish) {
3895 return;
3896 }
3897 }
3898
3899 /*
3900 * aarch64 assembly takes up to 24-bits immediate, generating
3901 * either cmp or cmp with shift 12 encoding
3902 */
3903 ImmOperand *immOpnd = static_cast<ImmOperand *>(opnd1);
3904 if (immOpnd->IsInBitSize(kMaxImmVal12Bits, 0) || immOpnd->IsInBitSize(kMaxImmVal12Bits, kMaxImmVal12Bits)) {
3905 mOp = is64Bits ? MOP_xcmpri : MOP_wcmpri;
3906 } else {
3907 opnd1 = &SelectCopy(*opnd1, primType, primType);
3908 }
3909 }
3910 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, rflag, *opnd0, *opnd1));
3911 }
3912
3913 bool isSigned = IsPrimitiveInteger(primType) ? IsSignedInteger(primType) : (signedCond ? true : false);
3914 MOperator jmpOperator = PickJmpInsn(jmpOp, cmpOp, isFloat, isSigned);
3915 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(jmpOperator, rflag, targetOpnd));
3916 }
3917
3918 /*
3919 * brtrue @label0 (ge u8 i32 (
3920 * cmp i32 i64 (dread i64 %Reg2_J, dread i64 %Reg4_J),
3921 * constval i32 0))
3922 * ===>
3923 * cmp r1, r2
3924 * bge Cond, label0
3925 */
SelectCondSpecialCase1(CondGotoNode & stmt,BaseNode & expr)3926 void AArch64CGFunc::SelectCondSpecialCase1(CondGotoNode &stmt, BaseNode &expr)
3927 {
3928 DEBUG_ASSERT(expr.GetOpCode() == OP_cmp, "unexpect opcode");
3929 Operand *opnd0 = HandleExpr(expr, *expr.Opnd(0));
3930 Operand *opnd1 = HandleExpr(expr, *expr.Opnd(1));
3931 CompareNode *node = static_cast<CompareNode *>(&expr);
3932 bool isFloat = IsPrimitiveFloat(node->GetOpndType());
3933 opnd0 = &LoadIntoRegister(*opnd0, node->GetOpndType());
3934 /*
3935 * most of FP constants are passed as MemOperand
3936 * except 0.0 which is passed as kOpdFPImmediate
3937 */
3938 Operand::OperandType opnd1Type = opnd1->GetKind();
3939 if ((opnd1Type != Operand::kOpdImmediate) && (opnd1Type != Operand::kOpdFPImmediate) &&
3940 (opnd1Type != Operand::kOpdOffset)) {
3941 opnd1 = &LoadIntoRegister(*opnd1, node->GetOpndType());
3942 }
3943 SelectAArch64Cmp(*opnd0, *opnd1, !isFloat, GetPrimTypeBitSize(node->GetOpndType()));
3944 /* handle condgoto now. */
3945 LabelIdx labelIdx = stmt.GetOffset();
3946 BaseNode *condNode = stmt.Opnd(0);
3947 LabelOperand &targetOpnd = GetOrCreateLabelOperand(labelIdx);
3948 Opcode cmpOp = condNode->GetOpCode();
3949 PrimType pType = static_cast<CompareNode *>(condNode)->GetOpndType();
3950 isFloat = IsPrimitiveFloat(pType);
3951 Operand &rflag = GetOrCreateRflag();
3952 bool isSigned =
3953 IsPrimitiveInteger(pType) ? IsSignedInteger(pType) : (IsSignedInteger(condNode->GetPrimType()) ? true : false);
3954 MOperator jmpOp = PickJmpInsn(stmt.GetOpCode(), cmpOp, isFloat, isSigned);
3955 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(jmpOp, rflag, targetOpnd));
3956 }
3957
3958 /*
3959 * Special case:
3960 * brfalse(ge (cmpg (op0, op1), 0) ==>
3961 * fcmp op1, op2
3962 * blo
3963 */
SelectCondSpecialCase2(const CondGotoNode & stmt,BaseNode & expr)3964 void AArch64CGFunc::SelectCondSpecialCase2(const CondGotoNode &stmt, BaseNode &expr)
3965 {
3966 auto &cmpNode = static_cast<CompareNode &>(expr);
3967 Operand *opnd0 = HandleExpr(cmpNode, *cmpNode.Opnd(0));
3968 Operand *opnd1 = HandleExpr(cmpNode, *cmpNode.Opnd(1));
3969 PrimType operandType = cmpNode.GetOpndType();
3970 opnd0 = opnd0->IsRegister() ? static_cast<RegOperand *>(opnd0) : &SelectCopy(*opnd0, operandType, operandType);
3971 Operand::OperandType opnd1Type = opnd1->GetKind();
3972 if ((opnd1Type != Operand::kOpdImmediate) && (opnd1Type != Operand::kOpdFPImmediate) &&
3973 (opnd1Type != Operand::kOpdOffset)) {
3974 opnd1 = opnd1->IsRegister() ? static_cast<RegOperand *>(opnd1) : &SelectCopy(*opnd1, operandType, operandType);
3975 }
3976 #ifdef DEBUG
3977 bool isFloat = IsPrimitiveFloat(operandType);
3978 if (!isFloat) {
3979 DEBUG_ASSERT(false, "incorrect operand types");
3980 }
3981 #endif
3982 SelectTargetFPCmpQuiet(*opnd0, *opnd1, GetPrimTypeBitSize(operandType));
3983 Operand &rFlag = GetOrCreateRflag();
3984 LabelIdx tempLabelIdx = stmt.GetOffset();
3985 LabelOperand &targetOpnd = GetOrCreateLabelOperand(tempLabelIdx);
3986 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_blo, rFlag, targetOpnd));
3987 }
3988
SelectCondGoto(CondGotoNode & stmt,Operand & opnd0,Operand & opnd1)3989 void AArch64CGFunc::SelectCondGoto(CondGotoNode &stmt, Operand &opnd0, Operand &opnd1)
3990 {
3991 /*
3992 * handle brfalse/brtrue op, opnd0 can be a compare node or non-compare node
3993 * such as a dread for example
3994 */
3995 LabelIdx labelIdx = stmt.GetOffset();
3996 BaseNode *condNode = stmt.Opnd(0);
3997 LabelOperand &targetOpnd = GetOrCreateLabelOperand(labelIdx);
3998 Opcode cmpOp;
3999
4000 if (opnd0.IsRegister() && (static_cast<RegOperand *>(&opnd0)->GetValidBitsNum() == 1) &&
4001 (condNode->GetOpCode() == OP_lior)) {
4002 ImmOperand &condBit = CreateImmOperand(0, k8BitSize, false);
4003 if (stmt.GetOpCode() == OP_brtrue) {
4004 GetCurBB()->AppendInsn(
4005 GetInsnBuilder()->BuildInsn(MOP_wtbnz, static_cast<RegOperand &>(opnd0), condBit, targetOpnd));
4006 } else {
4007 GetCurBB()->AppendInsn(
4008 GetInsnBuilder()->BuildInsn(MOP_wtbz, static_cast<RegOperand &>(opnd0), condBit, targetOpnd));
4009 }
4010 return;
4011 }
4012
4013 PrimType pType;
4014 if (kOpcodeInfo.IsCompare(condNode->GetOpCode())) {
4015 cmpOp = condNode->GetOpCode();
4016 pType = static_cast<CompareNode *>(condNode)->GetOpndType();
4017 } else {
4018 /* not a compare node; dread for example, take its pType */
4019 cmpOp = OP_ne;
4020 pType = condNode->GetPrimType();
4021 }
4022 bool signedCond = IsSignedInteger(pType) || IsPrimitiveFloat(pType);
4023 SelectCondGoto(targetOpnd, stmt.GetOpCode(), cmpOp, opnd0, opnd1, pType, signedCond);
4024 }
4025
SelectGoto(GotoNode & stmt)4026 void AArch64CGFunc::SelectGoto(GotoNode &stmt)
4027 {
4028 Operand &targetOpnd = GetOrCreateLabelOperand(stmt.GetOffset());
4029 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xuncond, targetOpnd));
4030 GetCurBB()->SetKind(BB::kBBGoto);
4031 }
4032
SelectAdd(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4033 Operand *AArch64CGFunc::SelectAdd(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4034 {
4035 PrimType dtype = node.GetPrimType();
4036 bool isSigned = IsSignedInteger(dtype);
4037 uint32 dsize = GetPrimTypeBitSize(dtype);
4038 bool is64Bits = (dsize == k64BitSize);
4039 bool isFloat = IsPrimitiveFloat(dtype);
4040 RegOperand *resOpnd = nullptr;
4041 if (!IsPrimitiveVector(dtype)) {
4042 /* promoted type */
4043 PrimType primType =
4044 isFloat ? dtype : ((is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32)));
4045 if (parent.GetOpCode() == OP_regassign) {
4046 auto ®AssignNode = static_cast<const RegassignNode &>(parent);
4047 PregIdx pregIdx = regAssignNode.GetRegIdx();
4048 if (IsSpecialPseudoRegister(pregIdx)) {
4049 resOpnd = &GetOrCreateSpecialRegisterOperand(-pregIdx, dtype);
4050 } else {
4051 resOpnd = &GetOrCreateVirtualRegisterOperand(GetVirtualRegNOFromPseudoRegIdx(pregIdx));
4052 }
4053 } else {
4054 resOpnd = &CreateRegisterOperandOfType(primType);
4055 }
4056 SelectAdd(*resOpnd, opnd0, opnd1, primType);
4057 } else {
4058 /* vector operands */
4059 resOpnd =
4060 SelectVectorBinOp(dtype, &opnd0, node.Opnd(0)->GetPrimType(), &opnd1, node.Opnd(1)->GetPrimType(), OP_add);
4061 }
4062 return resOpnd;
4063 }
4064
SelectAdd(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)4065 void AArch64CGFunc::SelectAdd(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
4066 {
4067 Operand::OperandType opnd0Type = opnd0.GetKind();
4068 Operand::OperandType opnd1Type = opnd1.GetKind();
4069 uint32 dsize = GetPrimTypeBitSize(primType);
4070 bool is64Bits = (dsize == k64BitSize);
4071 if (opnd0Type != Operand::kOpdRegister) {
4072 /* add #imm, #imm */
4073 if (opnd1Type != Operand::kOpdRegister) {
4074 SelectAdd(resOpnd, SelectCopy(opnd0, primType, primType), opnd1, primType);
4075 return;
4076 }
4077 /* add #imm, reg */
4078 SelectAdd(resOpnd, opnd1, opnd0, primType); /* commutative */
4079 return;
4080 }
4081 /* add reg, reg */
4082 if (opnd1Type == Operand::kOpdRegister) {
4083 DEBUG_ASSERT(IsPrimitiveFloat(primType) || IsPrimitiveInteger(primType), "NYI add");
4084 MOperator mOp =
4085 IsPrimitiveFloat(primType) ? (is64Bits ? MOP_dadd : MOP_sadd) : (is64Bits ? MOP_xaddrrr : MOP_waddrrr);
4086 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, opnd1));
4087 return;
4088 } else if (!((opnd1Type == Operand::kOpdImmediate) || (opnd1Type == Operand::kOpdOffset))) {
4089 /* add reg, otheregType */
4090 SelectAdd(resOpnd, opnd0, SelectCopy(opnd1, primType, primType), primType);
4091 return;
4092 } else {
4093 /* add reg, #imm */
4094 ImmOperand *immOpnd = static_cast<ImmOperand *>(&opnd1);
4095 if (immOpnd->IsNegative()) {
4096 immOpnd->Negate();
4097 SelectSub(resOpnd, opnd0, *immOpnd, primType);
4098 return;
4099 }
4100 if (immOpnd->IsInBitSize(kMaxImmVal24Bits, 0)) {
4101 /*
4102 * ADD Wd|WSP, Wn|WSP, #imm{, shift} ; 32-bit general registers
4103 * ADD Xd|SP, Xn|SP, #imm{, shift} ; 64-bit general registers
4104 * imm : 0 ~ 4095, shift: none, LSL #0, or LSL #12
4105 * aarch64 assembly takes up to 24-bits, if the lower 12 bits is all 0
4106 */
4107 MOperator mOpCode = MOP_undef;
4108 Operand *newOpnd0 = &opnd0;
4109 if (!(immOpnd->IsInBitSize(kMaxImmVal12Bits, 0) ||
4110 immOpnd->IsInBitSize(kMaxImmVal12Bits, kMaxImmVal12Bits))) {
4111 /* process higher 12 bits */
4112 ImmOperand &immOpnd2 =
4113 CreateImmOperand(static_cast<int64>(static_cast<uint64>(immOpnd->GetValue()) >> kMaxImmVal12Bits),
4114 immOpnd->GetSize(), immOpnd->IsSignedValue());
4115 mOpCode = is64Bits ? MOP_xaddrri24 : MOP_waddrri24;
4116 Operand *tmpRes = IsAfterRegAlloc() ? &resOpnd : &CreateRegisterOperandOfType(primType);
4117 BitShiftOperand &shiftopnd = CreateBitShiftOperand(BitShiftOperand::kLSL, kShiftAmount12, k64BitSize);
4118 Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, *tmpRes, opnd0, immOpnd2, shiftopnd);
4119 GetCurBB()->AppendInsn(newInsn);
4120 immOpnd->ModuloByPow2(static_cast<int32>(kMaxImmVal12Bits));
4121 newOpnd0 = tmpRes;
4122 }
4123 /* process lower 12 bits */
4124 mOpCode = is64Bits ? MOP_xaddrri12 : MOP_waddrri12;
4125 Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, *newOpnd0, *immOpnd);
4126 GetCurBB()->AppendInsn(newInsn);
4127 return;
4128 }
4129 /* load into register */
4130 int64 immVal = immOpnd->GetValue();
4131 int32 tail0bitNum = GetTail0BitNum(immVal);
4132 int32 head0bitNum = GetHead0BitNum(immVal);
4133 const int32 bitNum = (k64BitSizeInt - head0bitNum) - tail0bitNum;
4134 RegOperand ®Opnd = CreateRegisterOperandOfType(primType);
4135 if (isAfterRegAlloc) {
4136 RegType regty = GetRegTyFromPrimTy(primType);
4137 uint32 bytelen = GetPrimTypeSize(primType);
4138 regOpnd = GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(R16), bytelen, regty);
4139 }
4140 regno_t regNO0 = static_cast<RegOperand &>(opnd0).GetRegisterNumber();
4141 /* addrrrs do not support sp */
4142 if (bitNum <= k16ValidBit && regNO0 != RSP) {
4143 int64 newImm = (static_cast<uint64>(immVal) >> static_cast<uint32>(tail0bitNum)) & 0xFFFF;
4144 ImmOperand &immOpnd1 = CreateImmOperand(newImm, k16BitSize, false);
4145 SelectCopyImm(regOpnd, immOpnd1, primType);
4146 uint32 mopBadd = is64Bits ? MOP_xaddrrrs : MOP_waddrrrs;
4147 int32 bitLen = is64Bits ? kBitLenOfShift64Bits : kBitLenOfShift32Bits;
4148 BitShiftOperand &bitShiftOpnd =
4149 CreateBitShiftOperand(BitShiftOperand::kLSL, static_cast<uint32>(tail0bitNum), bitLen);
4150 Insn &newInsn = GetInsnBuilder()->BuildInsn(mopBadd, resOpnd, opnd0, regOpnd, bitShiftOpnd);
4151 GetCurBB()->AppendInsn(newInsn);
4152 return;
4153 }
4154
4155 SelectCopyImm(regOpnd, *immOpnd, primType);
4156 MOperator mOpCode = is64Bits ? MOP_xaddrrr : MOP_waddrrr;
4157 Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, opnd0, regOpnd);
4158 GetCurBB()->AppendInsn(newInsn);
4159 }
4160 }
4161
SelectMadd(BinaryNode & node,Operand & opndM0,Operand & opndM1,Operand & opnd1,const BaseNode & parent)4162 Operand *AArch64CGFunc::SelectMadd(BinaryNode &node, Operand &opndM0, Operand &opndM1, Operand &opnd1,
4163 const BaseNode &parent)
4164 {
4165 PrimType dtype = node.GetPrimType();
4166 bool isSigned = IsSignedInteger(dtype);
4167 uint32 dsize = GetPrimTypeBitSize(dtype);
4168 bool is64Bits = (dsize == k64BitSize);
4169 /* promoted type */
4170 PrimType primType = is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32);
4171 RegOperand &resOpnd = GetOrCreateResOperand(parent, primType);
4172 SelectMadd(resOpnd, opndM0, opndM1, opnd1, primType);
4173 return &resOpnd;
4174 }
4175
SelectMadd(Operand & resOpnd,Operand & opndM0,Operand & opndM1,Operand & opnd1,PrimType primType)4176 void AArch64CGFunc::SelectMadd(Operand &resOpnd, Operand &opndM0, Operand &opndM1, Operand &opnd1, PrimType primType)
4177 {
4178 Operand::OperandType opndM0Type = opndM0.GetKind();
4179 Operand::OperandType opndM1Type = opndM1.GetKind();
4180 Operand::OperandType opnd1Type = opnd1.GetKind();
4181 uint32 dsize = GetPrimTypeBitSize(primType);
4182 bool is64Bits = (dsize == k64BitSize);
4183
4184 if (opndM0Type != Operand::kOpdRegister) {
4185 SelectMadd(resOpnd, SelectCopy(opndM0, primType, primType), opndM1, opnd1, primType);
4186 return;
4187 } else if (opndM1Type != Operand::kOpdRegister) {
4188 SelectMadd(resOpnd, opndM0, SelectCopy(opndM1, primType, primType), opnd1, primType);
4189 return;
4190 } else if (opnd1Type != Operand::kOpdRegister) {
4191 SelectMadd(resOpnd, opndM0, opndM1, SelectCopy(opnd1, primType, primType), primType);
4192 return;
4193 }
4194
4195 DEBUG_ASSERT(IsPrimitiveInteger(primType), "NYI MAdd");
4196 MOperator mOp = is64Bits ? MOP_xmaddrrrr : MOP_wmaddrrrr;
4197 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opndM0, opndM1, opnd1));
4198 }
4199
SelectCGArrayElemAdd(BinaryNode & node,const BaseNode & parent)4200 Operand &AArch64CGFunc::SelectCGArrayElemAdd(BinaryNode &node, const BaseNode &parent)
4201 {
4202 BaseNode *opnd0 = node.Opnd(0);
4203 BaseNode *opnd1 = node.Opnd(1);
4204 DEBUG_ASSERT(opnd1->GetOpCode() == OP_constval, "Internal error, opnd1->op should be OP_constval.");
4205
4206 switch (opnd0->op) {
4207 case OP_regread: {
4208 RegreadNode *regreadNode = static_cast<RegreadNode *>(opnd0);
4209 return *SelectRegread(*regreadNode);
4210 }
4211 case OP_addrof: {
4212 AddrofNode *addrofNode = static_cast<AddrofNode *>(opnd0);
4213 MIRSymbol &symbol = *mirModule.CurFunction()->GetLocalOrGlobalSymbol(addrofNode->GetStIdx());
4214 DEBUG_ASSERT(addrofNode->GetFieldID() == 0, "For debug SelectCGArrayElemAdd.");
4215
4216 Operand &result = GetOrCreateResOperand(parent, PTY_a64);
4217
4218 /* OP_constval */
4219 ConstvalNode *constvalNode = static_cast<ConstvalNode *>(opnd1);
4220 MIRConst *mirConst = constvalNode->GetConstVal();
4221 MIRIntConst *mirIntConst = static_cast<MIRIntConst *>(mirConst);
4222 SelectAddrof(result, CreateStImmOperand(symbol, mirIntConst->GetExtValue(), 0));
4223
4224 return result;
4225 }
4226 default:
4227 CHECK_FATAL(0, "Internal error, cannot handle opnd0.");
4228 }
4229 }
4230
SelectSub(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)4231 void AArch64CGFunc::SelectSub(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
4232 {
4233 Operand::OperandType opnd1Type = opnd1.GetKind();
4234 uint32 dsize = GetPrimTypeBitSize(primType);
4235 bool is64Bits = (dsize == k64BitSize);
4236 bool isFloat = IsPrimitiveFloat(primType);
4237 Operand *opnd0Bak = &LoadIntoRegister(opnd0, primType);
4238 if (opnd1Type == Operand::kOpdRegister) {
4239 MOperator mOp = isFloat ? (is64Bits ? MOP_dsub : MOP_ssub) : (is64Bits ? MOP_xsubrrr : MOP_wsubrrr);
4240 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, *opnd0Bak, opnd1));
4241 return;
4242 }
4243
4244 if ((opnd1Type != Operand::kOpdImmediate) && (opnd1Type != Operand::kOpdOffset)) {
4245 SelectSub(resOpnd, *opnd0Bak, SelectCopy(opnd1, primType, primType), primType);
4246 return;
4247 }
4248
4249 ImmOperand *immOpnd = static_cast<ImmOperand *>(&opnd1);
4250 if (immOpnd->IsNegative()) {
4251 immOpnd->Negate();
4252 SelectAdd(resOpnd, *opnd0Bak, *immOpnd, primType);
4253 return;
4254 }
4255
4256 int64 higher12BitVal = static_cast<int64>(static_cast<uint64>(immOpnd->GetValue()) >> kMaxImmVal12Bits);
4257 if (immOpnd->IsInBitSize(kMaxImmVal24Bits, 0) && higher12BitVal + 1 <= kMaxPimm8) {
4258 /*
4259 * SUB Wd|WSP, Wn|WSP, #imm{, shift} ; 32-bit general registers
4260 * SUB Xd|SP, Xn|SP, #imm{, shift} ; 64-bit general registers
4261 * imm : 0 ~ 4095, shift: none, LSL #0, or LSL #12
4262 * aarch64 assembly takes up to 24-bits, if the lower 12 bits is all 0
4263 * large offset is treated as sub (higher 12 bits + 4096) + add
4264 * it gives opportunities for combining add + ldr due to the characteristics of aarch64's load/store
4265 */
4266 MOperator mOpCode = MOP_undef;
4267 bool isSplitSub = false;
4268 if (!(immOpnd->IsInBitSize(kMaxImmVal12Bits, 0) || immOpnd->IsInBitSize(kMaxImmVal12Bits, kMaxImmVal12Bits))) {
4269 isSplitSub = true;
4270 /* process higher 12 bits */
4271 ImmOperand &immOpnd2 = CreateImmOperand(higher12BitVal + 1, immOpnd->GetSize(), immOpnd->IsSignedValue());
4272
4273 mOpCode = is64Bits ? MOP_xsubrri24 : MOP_wsubrri24;
4274 BitShiftOperand &shiftopnd = CreateBitShiftOperand(BitShiftOperand::kLSL, kShiftAmount12, k64BitSize);
4275 Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, *opnd0Bak, immOpnd2, shiftopnd);
4276 GetCurBB()->AppendInsn(newInsn);
4277 immOpnd->ModuloByPow2(static_cast<int64>(kMaxImmVal12Bits));
4278 immOpnd->SetValue(static_cast<int64>(kMax12UnsignedImm) - immOpnd->GetValue());
4279 opnd0Bak = &resOpnd;
4280 }
4281 /* process lower 12 bits */
4282 mOpCode = isSplitSub ? (is64Bits ? MOP_xaddrri12 : MOP_waddrri12) : (is64Bits ? MOP_xsubrri12 : MOP_wsubrri12);
4283 Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, *opnd0Bak, *immOpnd);
4284 GetCurBB()->AppendInsn(newInsn);
4285 return;
4286 }
4287
4288 /* load into register */
4289 int64 immVal = immOpnd->GetValue();
4290 int32 tail0bitNum = GetTail0BitNum(immVal);
4291 int32 head0bitNum = GetHead0BitNum(immVal);
4292 const int32 bitNum = (k64BitSizeInt - head0bitNum) - tail0bitNum;
4293 RegOperand ®Opnd = CreateRegisterOperandOfType(primType);
4294 if (isAfterRegAlloc) {
4295 RegType regty = GetRegTyFromPrimTy(primType);
4296 uint32 bytelen = GetPrimTypeSize(primType);
4297 regOpnd = GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(R16), bytelen, regty);
4298 }
4299
4300 if (bitNum <= k16ValidBit) {
4301 int64 newImm = (static_cast<uint64>(immVal) >> static_cast<uint32>(tail0bitNum)) & 0xFFFF;
4302 ImmOperand &immOpnd1 = CreateImmOperand(newImm, k16BitSize, false);
4303 SelectCopyImm(regOpnd, immOpnd1, primType);
4304 uint32 mopBsub = is64Bits ? MOP_xsubrrrs : MOP_wsubrrrs;
4305 int32 bitLen = is64Bits ? kBitLenOfShift64Bits : kBitLenOfShift32Bits;
4306 BitShiftOperand &bitShiftOpnd =
4307 CreateBitShiftOperand(BitShiftOperand::kLSL, static_cast<uint32>(tail0bitNum), bitLen);
4308 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBsub, resOpnd, *opnd0Bak, regOpnd, bitShiftOpnd));
4309 return;
4310 }
4311
4312 SelectCopyImm(regOpnd, *immOpnd, primType);
4313 MOperator mOpCode = is64Bits ? MOP_xsubrrr : MOP_wsubrrr;
4314 Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, *opnd0Bak, regOpnd);
4315 GetCurBB()->AppendInsn(newInsn);
4316 }
4317
SelectSub(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4318 Operand *AArch64CGFunc::SelectSub(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4319 {
4320 PrimType dtype = node.GetPrimType();
4321 bool isSigned = IsSignedInteger(dtype);
4322 uint32 dsize = GetPrimTypeBitSize(dtype);
4323 bool is64Bits = (dsize == k64BitSize);
4324 bool isFloat = IsPrimitiveFloat(dtype);
4325 RegOperand *resOpnd = nullptr;
4326 if (!IsPrimitiveVector(dtype)) {
4327 /* promoted type */
4328 PrimType primType =
4329 isFloat ? dtype : ((is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32)));
4330 resOpnd = &GetOrCreateResOperand(parent, primType);
4331 SelectSub(*resOpnd, opnd0, opnd1, primType);
4332 } else {
4333 /* vector operands */
4334 resOpnd =
4335 SelectVectorBinOp(dtype, &opnd0, node.Opnd(0)->GetPrimType(), &opnd1, node.Opnd(1)->GetPrimType(), OP_sub);
4336 }
4337 return resOpnd;
4338 }
4339
SelectMpy(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4340 Operand *AArch64CGFunc::SelectMpy(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4341 {
4342 PrimType dtype = node.GetPrimType();
4343 bool isSigned = IsSignedInteger(dtype);
4344 uint32 dsize = GetPrimTypeBitSize(dtype);
4345 bool is64Bits = (dsize == k64BitSize);
4346 bool isFloat = IsPrimitiveFloat(dtype);
4347 RegOperand *resOpnd = nullptr;
4348 if (!IsPrimitiveVector(dtype)) {
4349 /* promoted type */
4350 PrimType primType =
4351 isFloat ? dtype : ((is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32)));
4352 resOpnd = &GetOrCreateResOperand(parent, primType);
4353 SelectMpy(*resOpnd, opnd0, opnd1, primType);
4354 } else {
4355 resOpnd =
4356 SelectVectorBinOp(dtype, &opnd0, node.Opnd(0)->GetPrimType(), &opnd1, node.Opnd(1)->GetPrimType(), OP_mul);
4357 }
4358 return resOpnd;
4359 }
4360
SelectMpy(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)4361 void AArch64CGFunc::SelectMpy(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
4362 {
4363 Operand::OperandType opnd0Type = opnd0.GetKind();
4364 Operand::OperandType opnd1Type = opnd1.GetKind();
4365 uint32 dsize = GetPrimTypeBitSize(primType);
4366 bool is64Bits = (dsize == k64BitSize);
4367
4368 if (((opnd0Type == Operand::kOpdImmediate) || (opnd0Type == Operand::kOpdOffset) ||
4369 (opnd1Type == Operand::kOpdImmediate) || (opnd1Type == Operand::kOpdOffset)) &&
4370 IsPrimitiveInteger(primType)) {
4371 ImmOperand *imm = ((opnd0Type == Operand::kOpdImmediate) || (opnd0Type == Operand::kOpdOffset))
4372 ? static_cast<ImmOperand *>(&opnd0)
4373 : static_cast<ImmOperand *>(&opnd1);
4374 Operand *otherOp =
4375 ((opnd0Type == Operand::kOpdImmediate) || (opnd0Type == Operand::kOpdOffset)) ? &opnd1 : &opnd0;
4376 int64 immValue = llabs(imm->GetValue());
4377 if (immValue != 0 && (static_cast<uint64>(immValue) & (static_cast<uint64>(immValue) - 1)) == 0) {
4378 /* immValue is 1 << n */
4379 if (otherOp->GetKind() != Operand::kOpdRegister) {
4380 otherOp = &SelectCopy(*otherOp, primType, primType);
4381 }
4382 int64 shiftVal = __builtin_ffsll(immValue);
4383 ImmOperand &shiftNum = CreateImmOperand(shiftVal - 1, dsize, false);
4384 SelectShift(resOpnd, *otherOp, shiftNum, kShiftLeft, primType);
4385 bool reachSignBit = (is64Bits && (shiftVal == k64BitSize)) || (!is64Bits && (shiftVal == k32BitSize));
4386 if (imm->GetValue() < 0 && !reachSignBit) {
4387 SelectNeg(resOpnd, resOpnd, primType);
4388 }
4389
4390 return;
4391 } else if (immValue > 2) { // immValue should larger than 2
4392 uint32 zeroNum = __builtin_ffsll(immValue) - 1;
4393 int64 headVal = static_cast<uint64>(immValue) >> zeroNum;
4394 /*
4395 * if (headVal - 1) & (headVal - 2) == 0, that is (immVal >> zeroNum) - 1 == 1 << n
4396 * otherOp * immVal = (otherOp * (immVal >> zeroNum) * (1 << zeroNum)
4397 * = (otherOp * ((immVal >> zeroNum) - 1) + otherOp) * (1 << zeroNum)
4398 */
4399 if (((static_cast<uint64>(headVal) - 1) & (static_cast<uint64>(headVal) - 2)) == 0) { // 2 see comment above
4400 if (otherOp->GetKind() != Operand::kOpdRegister) {
4401 otherOp = &SelectCopy(*otherOp, primType, primType);
4402 }
4403 ImmOperand &shiftNum1 = CreateImmOperand(__builtin_ffsll(headVal - 1) - 1, dsize, false);
4404 RegOperand &tmpOpnd = CreateRegisterOperandOfType(primType);
4405 SelectShift(tmpOpnd, *otherOp, shiftNum1, kShiftLeft, primType);
4406 SelectAdd(resOpnd, *otherOp, tmpOpnd, primType);
4407 ImmOperand &shiftNum2 = CreateImmOperand(zeroNum, dsize, false);
4408 SelectShift(resOpnd, resOpnd, shiftNum2, kShiftLeft, primType);
4409 if (imm->GetValue() < 0) {
4410 SelectNeg(resOpnd, resOpnd, primType);
4411 }
4412
4413 return;
4414 }
4415 }
4416 }
4417
4418 if ((opnd0Type != Operand::kOpdRegister) && (opnd1Type != Operand::kOpdRegister)) {
4419 SelectMpy(resOpnd, SelectCopy(opnd0, primType, primType), opnd1, primType);
4420 } else if ((opnd0Type == Operand::kOpdRegister) && (opnd1Type != Operand::kOpdRegister)) {
4421 SelectMpy(resOpnd, opnd0, SelectCopy(opnd1, primType, primType), primType);
4422 } else if ((opnd0Type != Operand::kOpdRegister) && (opnd1Type == Operand::kOpdRegister)) {
4423 SelectMpy(resOpnd, opnd1, opnd0, primType);
4424 } else {
4425 DEBUG_ASSERT(IsPrimitiveFloat(primType) || IsPrimitiveInteger(primType), "NYI Mpy");
4426 MOperator mOp =
4427 IsPrimitiveFloat(primType) ? (is64Bits ? MOP_xvmuld : MOP_xvmuls) : (is64Bits ? MOP_xmulrrr : MOP_wmulrrr);
4428 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, opnd1));
4429 }
4430 }
4431
SelectDiv(Operand & resOpnd,Operand & origOpnd0,Operand & opnd1,PrimType primType)4432 void AArch64CGFunc::SelectDiv(Operand &resOpnd, Operand &origOpnd0, Operand &opnd1, PrimType primType)
4433 {
4434 Operand &opnd0 = LoadIntoRegister(origOpnd0, primType);
4435 Operand::OperandType opnd0Type = opnd0.GetKind();
4436 Operand::OperandType opnd1Type = opnd1.GetKind();
4437 uint32 dsize = GetPrimTypeBitSize(primType);
4438 bool is64Bits = (dsize == k64BitSize);
4439
4440 if (Globals::GetInstance()->GetOptimLevel() > CGOptions::kLevel0) {
4441 if (((opnd1Type == Operand::kOpdImmediate) || (opnd1Type == Operand::kOpdOffset)) &&
4442 IsSignedInteger(primType)) {
4443 ImmOperand *imm = static_cast<ImmOperand *>(&opnd1);
4444 int64 immValue = llabs(imm->GetValue());
4445 if ((immValue != 0) && (static_cast<uint64>(immValue) & (static_cast<uint64>(immValue) - 1)) == 0) {
4446 if (immValue == 1) {
4447 if (imm->GetValue() > 0) {
4448 uint32 mOp = is64Bits ? MOP_xmovrr : MOP_wmovrr;
4449 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0));
4450 } else {
4451 SelectNeg(resOpnd, opnd0, primType);
4452 }
4453
4454 return;
4455 }
4456 int32 shiftNumber = __builtin_ffsll(immValue) - 1;
4457 ImmOperand &shiftNum = CreateImmOperand(shiftNumber, dsize, false);
4458 Operand &tmpOpnd = CreateRegisterOperandOfType(primType);
4459 SelectShift(tmpOpnd, opnd0, CreateImmOperand(dsize - 1, dsize, false), kShiftAright, primType);
4460 uint32 mopBadd = is64Bits ? MOP_xaddrrrs : MOP_waddrrrs;
4461 int32 bitLen = is64Bits ? kBitLenOfShift64Bits : kBitLenOfShift32Bits;
4462 BitShiftOperand &shiftOpnd = CreateBitShiftOperand(BitShiftOperand::kLSR, dsize - shiftNumber, bitLen);
4463 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBadd, tmpOpnd, opnd0, tmpOpnd, shiftOpnd));
4464 SelectShift(resOpnd, tmpOpnd, shiftNum, kShiftAright, primType);
4465 if (imm->GetValue() < 0) {
4466 SelectNeg(resOpnd, resOpnd, primType);
4467 }
4468
4469 return;
4470 }
4471 } else if (((opnd1Type == Operand::kOpdImmediate) || (opnd1Type == Operand::kOpdOffset)) &&
4472 IsUnsignedInteger(primType)) {
4473 ImmOperand *imm = static_cast<ImmOperand *>(&opnd1);
4474 if (imm->GetValue() != 0) {
4475 if ((imm->GetValue() > 0) &&
4476 ((static_cast<uint64>(imm->GetValue()) & (static_cast<uint64>(imm->GetValue()) - 1)) == 0)) {
4477 ImmOperand &shiftNum = CreateImmOperand(__builtin_ffsll(imm->GetValue()) - 1, dsize, false);
4478 SelectShift(resOpnd, opnd0, shiftNum, kShiftLright, primType);
4479
4480 return;
4481 } else if (imm->GetValue() < 0) {
4482 SelectAArch64Cmp(opnd0, *imm, true, dsize);
4483 SelectAArch64CSet(resOpnd, GetCondOperand(CC_CS), is64Bits);
4484
4485 return;
4486 }
4487 }
4488 }
4489 }
4490
4491 if (opnd0Type != Operand::kOpdRegister) {
4492 SelectDiv(resOpnd, SelectCopy(opnd0, primType, primType), opnd1, primType);
4493 } else if (opnd1Type != Operand::kOpdRegister) {
4494 SelectDiv(resOpnd, opnd0, SelectCopy(opnd1, primType, primType), primType);
4495 } else {
4496 DEBUG_ASSERT(IsPrimitiveFloat(primType) || IsPrimitiveInteger(primType), "NYI Div");
4497 MOperator mOp = IsPrimitiveFloat(primType)
4498 ? (is64Bits ? MOP_ddivrrr : MOP_sdivrrr)
4499 : (IsSignedInteger(primType) ? (is64Bits ? MOP_xsdivrrr : MOP_wsdivrrr)
4500 : (is64Bits ? MOP_xudivrrr : MOP_wudivrrr));
4501 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, opnd1));
4502 }
4503 }
4504
SelectDiv(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4505 Operand *AArch64CGFunc::SelectDiv(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4506 {
4507 PrimType dtype = node.GetPrimType();
4508 bool isSigned = IsSignedInteger(dtype);
4509 uint32 dsize = GetPrimTypeBitSize(dtype);
4510 bool is64Bits = (dsize == k64BitSize);
4511 bool isFloat = IsPrimitiveFloat(dtype);
4512 CHECK_FATAL(!IsPrimitiveVector(dtype), "NYI DIV vector operands");
4513 /* promoted type */
4514 PrimType primType =
4515 isFloat ? dtype : ((is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32)));
4516 RegOperand &resOpnd = GetOrCreateResOperand(parent, primType);
4517 SelectDiv(resOpnd, opnd0, opnd1, primType);
4518 return &resOpnd;
4519 }
4520
SelectRem(Operand & resOpnd,Operand & lhsOpnd,Operand & rhsOpnd,PrimType primType,bool isSigned,bool is64Bits)4521 void AArch64CGFunc::SelectRem(Operand &resOpnd, Operand &lhsOpnd, Operand &rhsOpnd, PrimType primType, bool isSigned,
4522 bool is64Bits)
4523 {
4524 Operand &opnd0 = LoadIntoRegister(lhsOpnd, primType);
4525 Operand &opnd1 = LoadIntoRegister(rhsOpnd, primType);
4526
4527 DEBUG_ASSERT(IsPrimitiveInteger(primType), "Wrong type for REM");
4528 /*
4529 * printf("%d \n", 29 % 7 );
4530 * -> 1
4531 * printf("%u %d \n", (unsigned)-7, (unsigned)(-7) % 7 );
4532 * -> 4294967289 4
4533 * printf("%d \n", (-7) % 7 );
4534 * -> 0
4535 * printf("%d \n", 237 % -7 );
4536 * 6->
4537 * printf("implicit i->u conversion %d \n", ((unsigned)237) % -7 );
4538 * implicit conversion 237
4539
4540 * http://stackoverflow.com/questions/35351470/obtaining-remainder-using-single-aarch64-instruction
4541 * input: x0=dividend, x1=divisor
4542 * udiv|sdiv x2, x0, x1
4543 * msub x3, x2, x1, x0 -- multply-sub : x3 <- x0 - x2*x1
4544 * result: x2=quotient, x3=remainder
4545 *
4546 * allocate temporary register
4547 */
4548 RegOperand &temp = CreateRegisterOperandOfType(primType);
4549 /*
4550 * mov w1, #2
4551 * sdiv wTemp, w0, w1
4552 * msub wRespond, wTemp, w1, w0
4553 * ========>
4554 * asr wTemp, w0, #31
4555 * lsr wTemp, wTemp, #31 (#30 for 4, #29 for 8, ...)
4556 * add wRespond, w0, wTemp
4557 * and wRespond, wRespond, #1 (#3 for 4, #7 for 8, ...)
4558 * sub wRespond, wRespond, w2
4559 *
4560 * if divde by 2
4561 * ========>
4562 * lsr wTemp, w0, #31
4563 * add wRespond, w0, wTemp
4564 * and wRespond, wRespond, #1
4565 * sub wRespond, wRespond, w2
4566 *
4567 * for unsigned rem op, just use and
4568 */
4569 if ((Globals::GetInstance()->GetOptimLevel() >= CGOptions::kLevel2)) {
4570 ImmOperand *imm = nullptr;
4571 Insn *movImmInsn = GetCurBB()->GetLastInsn();
4572 if (movImmInsn &&
4573 ((movImmInsn->GetMachineOpcode() == MOP_wmovri32) || (movImmInsn->GetMachineOpcode() == MOP_xmovri64)) &&
4574 movImmInsn->GetOperand(0).Equals(opnd1)) {
4575 /*
4576 * mov w1, #2
4577 * rem res, w0, w1
4578 */
4579 imm = static_cast<ImmOperand *>(&movImmInsn->GetOperand(kInsnSecondOpnd));
4580 } else if (opnd1.IsImmediate()) {
4581 /*
4582 * rem res, w0, #2
4583 */
4584 imm = static_cast<ImmOperand *>(&opnd1);
4585 }
4586 /* positive or negative do not have effect on the result */
4587 int64 dividor = 0;
4588 if (imm && (imm->GetValue() != LONG_MIN)) {
4589 dividor = abs(imm->GetValue());
4590 }
4591 const int64 Log2OfDividor = GetLog2(static_cast<uint64>(dividor));
4592 if ((dividor != 0) && (Log2OfDividor > 0)) {
4593 if (is64Bits) {
4594 CHECK_FATAL(Log2OfDividor < k64BitSize, "imm out of bound");
4595 if (isSigned) {
4596 ImmOperand &rightShiftValue = CreateImmOperand(k64BitSize - Log2OfDividor, k64BitSize, isSigned);
4597 if (Log2OfDividor != 1) {
4598 /* 63->shift ALL , 32 ->32bit register */
4599 ImmOperand &rightShiftAll = CreateImmOperand(63, k64BitSize, isSigned);
4600 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xasrrri6, temp, opnd0, rightShiftAll));
4601
4602 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xlsrrri6, temp, temp, rightShiftValue));
4603 } else {
4604 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xlsrrri6, temp, opnd0, rightShiftValue));
4605 }
4606 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrrr, resOpnd, opnd0, temp));
4607 ImmOperand &remBits = CreateImmOperand(dividor - 1, k64BitSize, isSigned);
4608 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xandrri13, resOpnd, resOpnd, remBits));
4609 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xsubrrr, resOpnd, resOpnd, temp));
4610 return;
4611 } else if (imm && imm->GetValue() > 0) {
4612 ImmOperand &remBits = CreateImmOperand(dividor - 1, k64BitSize, isSigned);
4613 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xandrri13, resOpnd, opnd0, remBits));
4614 return;
4615 }
4616 } else {
4617 CHECK_FATAL(Log2OfDividor < k32BitSize, "imm out of bound");
4618 if (isSigned) {
4619 ImmOperand &rightShiftValue = CreateImmOperand(k32BitSize - Log2OfDividor, k32BitSize, isSigned);
4620 if (Log2OfDividor != 1) {
4621 /* 31->shift ALL , 32 ->32bit register */
4622 ImmOperand &rightShiftAll = CreateImmOperand(31, k32BitSize, isSigned);
4623 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wasrrri5, temp, opnd0, rightShiftAll));
4624
4625 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wlsrrri5, temp, temp, rightShiftValue));
4626 } else {
4627 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wlsrrri5, temp, opnd0, rightShiftValue));
4628 }
4629
4630 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_waddrrr, resOpnd, opnd0, temp));
4631 ImmOperand &remBits = CreateImmOperand(dividor - 1, k32BitSize, isSigned);
4632 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wandrri12, resOpnd, resOpnd, remBits));
4633
4634 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wsubrrr, resOpnd, resOpnd, temp));
4635 return;
4636 } else if (imm && imm->GetValue() > 0) {
4637 ImmOperand &remBits = CreateImmOperand(dividor - 1, k32BitSize, isSigned);
4638 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wandrri12, resOpnd, opnd0, remBits));
4639 return;
4640 }
4641 }
4642 }
4643 }
4644
4645 uint32 mopDiv = is64Bits ? (isSigned ? MOP_xsdivrrr : MOP_xudivrrr) : (isSigned ? MOP_wsdivrrr : MOP_wudivrrr);
4646 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopDiv, temp, opnd0, opnd1));
4647
4648 uint32 mopSub = is64Bits ? MOP_xmsubrrrr : MOP_wmsubrrrr;
4649 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopSub, resOpnd, temp, opnd1, opnd0));
4650 }
4651
SelectRem(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4652 Operand *AArch64CGFunc::SelectRem(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4653 {
4654 PrimType dtype = node.GetPrimType();
4655 DEBUG_ASSERT(IsPrimitiveInteger(dtype), "wrong type for rem");
4656 bool isSigned = IsSignedInteger(dtype);
4657 uint32 dsize = GetPrimTypeBitSize(dtype);
4658 bool is64Bits = (dsize == k64BitSize);
4659 CHECK_FATAL(!IsPrimitiveVector(dtype), "NYI DIV vector operands");
4660
4661 /* promoted type */
4662 PrimType primType = ((is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32)));
4663 RegOperand &resOpnd = GetOrCreateResOperand(parent, primType);
4664 SelectRem(resOpnd, opnd0, opnd1, primType, isSigned, is64Bits);
4665 return &resOpnd;
4666 }
4667
SelectLand(BinaryNode & node,Operand & lhsOpnd,Operand & rhsOpnd,const BaseNode & parent)4668 Operand *AArch64CGFunc::SelectLand(BinaryNode &node, Operand &lhsOpnd, Operand &rhsOpnd, const BaseNode &parent)
4669 {
4670 PrimType primType = node.GetPrimType();
4671 DEBUG_ASSERT(IsPrimitiveInteger(primType), "Land should be integer type");
4672 bool is64Bits = (GetPrimTypeBitSize(primType) == k64BitSize);
4673 RegOperand &resOpnd = GetOrCreateResOperand(parent, is64Bits ? PTY_u64 : PTY_u32);
4674 /*
4675 * OP0 band Op1
4676 * cmp OP0, 0 # compare X0 with 0, sets Z bit
4677 * ccmp OP1, 0, 4 //==0100b, ne # if(OP0!=0) cmp Op1 and 0, else NZCV <- 0100 makes OP0==0
4678 * cset RES, ne # if Z==1(i.e., OP0==0||OP1==0) RES<-0, RES<-1
4679 */
4680 Operand &opnd0 = LoadIntoRegister(lhsOpnd, primType);
4681 SelectAArch64Cmp(opnd0, CreateImmOperand(0, primType, false), true, GetPrimTypeBitSize(primType));
4682 Operand &opnd1 = LoadIntoRegister(rhsOpnd, primType);
4683 constexpr int64 immValue4 = 4; //integer as comment above
4684 SelectAArch64CCmp(opnd1, CreateImmOperand(0, primType, false), CreateImmOperand(immValue4, PTY_u8, false),
4685 GetCondOperand(CC_NE), is64Bits);
4686 SelectAArch64CSet(resOpnd, GetCondOperand(CC_NE), is64Bits);
4687 return &resOpnd;
4688 }
4689
SelectLor(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent,bool parentIsBr)4690 Operand *AArch64CGFunc::SelectLor(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent,
4691 bool parentIsBr)
4692 {
4693 PrimType primType = node.GetPrimType();
4694 DEBUG_ASSERT(IsPrimitiveInteger(primType), "Lior should be integer type");
4695 bool is64Bits = (GetPrimTypeBitSize(primType) == k64BitSize);
4696 RegOperand &resOpnd = GetOrCreateResOperand(parent, is64Bits ? PTY_u64 : PTY_u32);
4697 /*
4698 * OP0 band Op1
4699 * cmp OP0, 0 # compare X0 with 0, sets Z bit
4700 * ccmp OP1, 0, 0 //==0100b, eq # if(OP0==0,eq) cmp Op1 and 0, else NZCV <- 0000 makes OP0!=0
4701 * cset RES, ne # if Z==1(i.e., OP0==0&&OP1==0) RES<-0, RES<-1
4702 */
4703 if (parentIsBr && !is64Bits && opnd0.IsRegister() && (static_cast<RegOperand *>(&opnd0)->GetValidBitsNum() == 1) &&
4704 opnd1.IsRegister() && (static_cast<RegOperand *>(&opnd1)->GetValidBitsNum() == 1)) {
4705 uint32 mOp = MOP_wiorrrr;
4706 static_cast<RegOperand &>(resOpnd).SetValidBitsNum(1);
4707 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, opnd1));
4708 } else {
4709 SelectBior(resOpnd, opnd0, opnd1, primType);
4710 SelectAArch64Cmp(resOpnd, CreateImmOperand(0, primType, false), true, GetPrimTypeBitSize(primType));
4711 SelectAArch64CSet(resOpnd, GetCondOperand(CC_NE), is64Bits);
4712 }
4713 return &resOpnd;
4714 }
4715
SelectCmpOp(Operand & resOpnd,Operand & lhsOpnd,Operand & rhsOpnd,Opcode opcode,PrimType primType,const BaseNode & parent)4716 void AArch64CGFunc::SelectCmpOp(Operand &resOpnd, Operand &lhsOpnd, Operand &rhsOpnd, Opcode opcode, PrimType primType,
4717 const BaseNode &parent)
4718 {
4719 uint32 dsize = resOpnd.GetSize();
4720 bool isFloat = IsPrimitiveFloat(primType);
4721 Operand &opnd0 = LoadIntoRegister(lhsOpnd, primType);
4722
4723 /*
4724 * most of FP constants are passed as MemOperand
4725 * except 0.0 which is passed as kOpdFPImmediate
4726 */
4727 Operand::OperandType opnd1Type = rhsOpnd.GetKind();
4728 Operand *opnd1 = &rhsOpnd;
4729 if ((opnd1Type != Operand::kOpdImmediate) && (opnd1Type != Operand::kOpdFPImmediate) &&
4730 (opnd1Type != Operand::kOpdOffset)) {
4731 opnd1 = &LoadIntoRegister(rhsOpnd, primType);
4732 }
4733
4734 bool unsignedIntegerComparison = !isFloat && !IsSignedInteger(primType);
4735 /*
4736 * OP_cmp, OP_cmpl, OP_cmpg
4737 * <cmp> OP0, OP1 ; fcmp for OP_cmpl/OP_cmpg, cmp/fcmpe for OP_cmp
4738 * CSINV RES, WZR, WZR, GE
4739 * CSINC RES, RES, WZR, LE
4740 * if OP_cmpl, CSINV RES, RES, WZR, VC (no overflow)
4741 * if OP_cmpg, CSINC RES, RES, WZR, VC (no overflow)
4742 */
4743 RegOperand &xzr = GetZeroOpnd(dsize);
4744 if ((opcode == OP_cmpl) || (opcode == OP_cmpg)) {
4745 DEBUG_ASSERT(isFloat, "incorrect operand types");
4746 SelectTargetFPCmpQuiet(opnd0, *opnd1, GetPrimTypeBitSize(primType));
4747 SelectAArch64CSINV(resOpnd, xzr, xzr, GetCondOperand(CC_GE), (dsize == k64BitSize));
4748 SelectAArch64CSINC(resOpnd, resOpnd, xzr, GetCondOperand(CC_LE), (dsize == k64BitSize));
4749 if (opcode == OP_cmpl) {
4750 SelectAArch64CSINV(resOpnd, resOpnd, xzr, GetCondOperand(CC_VC), (dsize == k64BitSize));
4751 } else {
4752 SelectAArch64CSINC(resOpnd, resOpnd, xzr, GetCondOperand(CC_VC), (dsize == k64BitSize));
4753 }
4754 return;
4755 }
4756
4757 if (opcode == OP_cmp) {
4758 SelectAArch64Cmp(opnd0, *opnd1, !isFloat, GetPrimTypeBitSize(primType));
4759 if (unsignedIntegerComparison) {
4760 SelectAArch64CSINV(resOpnd, xzr, xzr, GetCondOperand(CC_HS), (dsize == k64BitSize));
4761 SelectAArch64CSINC(resOpnd, resOpnd, xzr, GetCondOperand(CC_LS), (dsize == k64BitSize));
4762 } else {
4763 SelectAArch64CSINV(resOpnd, xzr, xzr, GetCondOperand(CC_GE), (dsize == k64BitSize));
4764 SelectAArch64CSINC(resOpnd, resOpnd, xzr, GetCondOperand(CC_LE), (dsize == k64BitSize));
4765 }
4766 return;
4767 }
4768
4769 // lt u8 i32 ( xxx, 0 ) => get sign bit
4770 if ((opcode == OP_lt) && opnd0.IsRegister() && opnd1->IsImmediate() &&
4771 (static_cast<ImmOperand *>(opnd1)->GetValue() == 0) && parent.GetOpCode() != OP_select && !isFloat) {
4772 bool is64Bits = (opnd0.GetSize() == k64BitSize);
4773 if (!unsignedIntegerComparison) {
4774 int32 bitLen = is64Bits ? kBitLenOfShift64Bits : kBitLenOfShift32Bits;
4775 ImmOperand &shiftNum = CreateImmOperand(is64Bits ? kHighestBitOf64Bits : kHighestBitOf32Bits,
4776 static_cast<uint32>(bitLen), false);
4777 MOperator mOpCode = is64Bits ? MOP_xlsrrri6 : MOP_wlsrrri5;
4778 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, opnd0, shiftNum));
4779 return;
4780 }
4781 ImmOperand &constNum = CreateImmOperand(0, is64Bits ? k64BitSize : k32BitSize, false);
4782 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(is64Bits ? MOP_xmovri64 : MOP_wmovri32, resOpnd, constNum));
4783 return;
4784 }
4785 SelectAArch64Cmp(opnd0, *opnd1, !isFloat, GetPrimTypeBitSize(primType));
4786
4787 ConditionCode cc = CC_EQ;
4788 // need to handle unordered situation here.
4789 switch (opcode) {
4790 case OP_eq:
4791 cc = CC_EQ;
4792 break;
4793 case OP_ne:
4794 cc = isFloat ? CC_MI : CC_NE;
4795 break;
4796 case OP_le:
4797 cc = isFloat ? CC_LS : unsignedIntegerComparison ? CC_LS : CC_LE;
4798 break;
4799 case OP_ge:
4800 cc = unsignedIntegerComparison ? CC_HS : CC_GE;
4801 break;
4802 case OP_gt:
4803 cc = unsignedIntegerComparison ? CC_HI : CC_GT;
4804 break;
4805 case OP_lt:
4806 cc = isFloat ? CC_MI : unsignedIntegerComparison ? CC_LO : CC_LT;
4807 break;
4808 default:
4809 CHECK_FATAL(false, "illegal logical operator");
4810 }
4811 SelectAArch64CSet(resOpnd, GetCondOperand(cc), (dsize == k64BitSize));
4812 if (isFloat && opcode == OP_ne) {
4813 SelectAArch64CSINC(resOpnd, resOpnd, xzr, GetCondOperand(CC_LE), (dsize == k64BitSize));
4814 }
4815 }
4816
SelectCmpOp(CompareNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4817 Operand *AArch64CGFunc::SelectCmpOp(CompareNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4818 {
4819 RegOperand *resOpnd = nullptr;
4820 if (!IsPrimitiveVector(node.GetPrimType())) {
4821 resOpnd = &GetOrCreateResOperand(parent, node.GetPrimType());
4822 SelectCmpOp(*resOpnd, opnd0, opnd1, node.GetOpCode(), node.GetOpndType(), parent);
4823 } else {
4824 resOpnd = SelectVectorCompare(&opnd0, node.Opnd(0)->GetPrimType(), &opnd1, node.Opnd(1)->GetPrimType(),
4825 node.GetOpCode());
4826 }
4827 return resOpnd;
4828 }
4829
SelectTargetFPCmpQuiet(Operand & o0,Operand & o1,uint32 dsize)4830 void AArch64CGFunc::SelectTargetFPCmpQuiet(Operand &o0, Operand &o1, uint32 dsize)
4831 {
4832 MOperator mOpCode = 0;
4833 if (o1.GetKind() == Operand::kOpdFPImmediate) {
4834 CHECK_FATAL(static_cast<ImmOperand &>(o0).GetValue() == 0, "NIY");
4835 mOpCode = (dsize == k64BitSize) ? MOP_dcmpqri : (dsize == k32BitSize) ? MOP_scmpqri : MOP_hcmpqri;
4836 } else if (o1.GetKind() == Operand::kOpdRegister) {
4837 mOpCode = (dsize == k64BitSize) ? MOP_dcmpqrr : (dsize == k32BitSize) ? MOP_scmpqrr : MOP_hcmpqrr;
4838 } else {
4839 CHECK_FATAL(false, "unsupported operand type");
4840 }
4841 Operand &rflag = GetOrCreateRflag();
4842 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, rflag, o0, o1));
4843 }
4844
SelectAArch64Cmp(Operand & o0,Operand & o1,bool isIntType,uint32 dsize)4845 void AArch64CGFunc::SelectAArch64Cmp(Operand &o0, Operand &o1, bool isIntType, uint32 dsize)
4846 {
4847 MOperator mOpCode = 0;
4848 Operand *newO1 = &o1;
4849 if (isIntType) {
4850 if ((o1.GetKind() == Operand::kOpdImmediate) || (o1.GetKind() == Operand::kOpdOffset)) {
4851 ImmOperand *immOpnd = static_cast<ImmOperand *>(&o1);
4852 /*
4853 * imm : 0 ~ 4095, shift: none, LSL #0, or LSL #12
4854 * aarch64 assembly takes up to 24-bits, if the lower 12 bits is all 0
4855 */
4856 if (immOpnd->IsInBitSize(kMaxImmVal12Bits, 0) || immOpnd->IsInBitSize(kMaxImmVal12Bits, kMaxImmVal12Bits)) {
4857 mOpCode = (dsize == k64BitSize) ? MOP_xcmpri : MOP_wcmpri;
4858 } else {
4859 /* load into register */
4860 PrimType ptype = (dsize == k64BitSize) ? PTY_i64 : PTY_i32;
4861 newO1 = &SelectCopy(o1, ptype, ptype);
4862 mOpCode = (dsize == k64BitSize) ? MOP_xcmprr : MOP_wcmprr;
4863 }
4864 } else if (o1.GetKind() == Operand::kOpdRegister) {
4865 mOpCode = (dsize == k64BitSize) ? MOP_xcmprr : MOP_wcmprr;
4866 } else {
4867 CHECK_FATAL(false, "unsupported operand type");
4868 }
4869 } else { /* float */
4870 if (o1.GetKind() == Operand::kOpdFPImmediate) {
4871 CHECK_FATAL(static_cast<ImmOperand &>(o1).GetValue() == 0, "NIY");
4872 mOpCode = (dsize == k64BitSize) ? MOP_dcmperi : ((dsize == k32BitSize) ? MOP_scmperi : MOP_hcmperi);
4873 } else if (o1.GetKind() == Operand::kOpdRegister) {
4874 mOpCode = (dsize == k64BitSize) ? MOP_dcmperr : ((dsize == k32BitSize) ? MOP_scmperr : MOP_hcmperr);
4875 } else {
4876 CHECK_FATAL(false, "unsupported operand type");
4877 }
4878 }
4879 DEBUG_ASSERT(mOpCode != 0, "mOpCode undefined");
4880 Operand &rflag = GetOrCreateRflag();
4881 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, rflag, o0, *newO1));
4882 }
4883
SelectAArch64CCmp(Operand & o,Operand & i,Operand & nzcv,CondOperand & cond,bool is64Bits)4884 void AArch64CGFunc::SelectAArch64CCmp(Operand &o, Operand &i, Operand &nzcv, CondOperand &cond, bool is64Bits)
4885 {
4886 uint32 mOpCode = is64Bits ? MOP_xccmpriic : MOP_wccmpriic;
4887 Operand &rflag = GetOrCreateRflag();
4888 std::vector<Operand *> opndVec;
4889 opndVec.push_back(&rflag);
4890 opndVec.push_back(&o);
4891 opndVec.push_back(&i);
4892 opndVec.push_back(&nzcv);
4893 opndVec.push_back(&cond);
4894 opndVec.push_back(&rflag);
4895 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, opndVec));
4896 }
4897
SelectAArch64CSet(Operand & r,CondOperand & cond,bool is64Bits)4898 void AArch64CGFunc::SelectAArch64CSet(Operand &r, CondOperand &cond, bool is64Bits)
4899 {
4900 MOperator mOpCode = is64Bits ? MOP_xcsetrc : MOP_wcsetrc;
4901 Operand &rflag = GetOrCreateRflag();
4902 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, r, cond, rflag));
4903 }
4904
SelectAArch64CSINV(Operand & res,Operand & o0,Operand & o1,CondOperand & cond,bool is64Bits)4905 void AArch64CGFunc::SelectAArch64CSINV(Operand &res, Operand &o0, Operand &o1, CondOperand &cond, bool is64Bits)
4906 {
4907 MOperator mOpCode = is64Bits ? MOP_xcsinvrrrc : MOP_wcsinvrrrc;
4908 Operand &rflag = GetOrCreateRflag();
4909 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, res, o0, o1, cond, rflag));
4910 }
4911
SelectAArch64CSINC(Operand & res,Operand & o0,Operand & o1,CondOperand & cond,bool is64Bits)4912 void AArch64CGFunc::SelectAArch64CSINC(Operand &res, Operand &o0, Operand &o1, CondOperand &cond, bool is64Bits)
4913 {
4914 MOperator mOpCode = is64Bits ? MOP_xcsincrrrc : MOP_wcsincrrrc;
4915 Operand &rflag = GetOrCreateRflag();
4916 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, res, o0, o1, cond, rflag));
4917 }
4918
SelectBand(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4919 Operand *AArch64CGFunc::SelectBand(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
4920 {
4921 return SelectRelationOperator(kAND, node, opnd0, opnd1, parent);
4922 }
4923
SelectBand(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)4924 void AArch64CGFunc::SelectBand(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
4925 {
4926 SelectRelationOperator(kAND, resOpnd, opnd0, opnd1, primType);
4927 }
4928
SelectRelationOperator(RelationOperator operatorCode,const BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)4929 Operand *AArch64CGFunc::SelectRelationOperator(RelationOperator operatorCode, const BinaryNode &node, Operand &opnd0,
4930 Operand &opnd1, const BaseNode &parent)
4931 {
4932 PrimType dtype = node.GetPrimType();
4933 bool isSigned = IsSignedInteger(dtype);
4934 uint32 dsize = GetPrimTypeBitSize(dtype);
4935 bool is64Bits = (dsize == k64BitSize);
4936 RegOperand *resOpnd = nullptr;
4937 if (!IsPrimitiveVector(dtype)) {
4938 PrimType primType =
4939 is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32); /* promoted type */
4940 resOpnd = &GetOrCreateResOperand(parent, primType);
4941 SelectRelationOperator(operatorCode, *resOpnd, opnd0, opnd1, primType);
4942 } else {
4943 /* vector operations */
4944 resOpnd = SelectVectorBitwiseOp(dtype, &opnd0, node.Opnd(0)->GetPrimType(), &opnd1, node.Opnd(1)->GetPrimType(),
4945 (operatorCode == kAND) ? OP_band : (operatorCode == kIOR ? OP_bior : OP_bxor));
4946 }
4947 return resOpnd;
4948 }
4949
SelectRelationMop(RelationOperator operatorCode,RelationOperatorOpndPattern opndPattern,bool is64Bits,bool isBitmaskImmediate,bool isBitNumLessThan16) const4950 MOperator AArch64CGFunc::SelectRelationMop(RelationOperator operatorCode, RelationOperatorOpndPattern opndPattern,
4951 bool is64Bits, bool isBitmaskImmediate, bool isBitNumLessThan16) const
4952 {
4953 MOperator mOp = MOP_undef;
4954 if (opndPattern == kRegReg) {
4955 switch (operatorCode) {
4956 case kAND:
4957 mOp = is64Bits ? MOP_xandrrr : MOP_wandrrr;
4958 break;
4959 case kIOR:
4960 mOp = is64Bits ? MOP_xiorrrr : MOP_wiorrrr;
4961 break;
4962 case kEOR:
4963 mOp = is64Bits ? MOP_xeorrrr : MOP_weorrrr;
4964 break;
4965 default:
4966 break;
4967 }
4968 return mOp;
4969 }
4970 /* opndPattern == KRegImm */
4971 if (isBitmaskImmediate) {
4972 switch (operatorCode) {
4973 case kAND:
4974 mOp = is64Bits ? MOP_xandrri13 : MOP_wandrri12;
4975 break;
4976 case kIOR:
4977 mOp = is64Bits ? MOP_xiorrri13 : MOP_wiorrri12;
4978 break;
4979 case kEOR:
4980 mOp = is64Bits ? MOP_xeorrri13 : MOP_weorrri12;
4981 break;
4982 default:
4983 break;
4984 }
4985 return mOp;
4986 }
4987 /* normal imm value */
4988 if (isBitNumLessThan16) {
4989 switch (operatorCode) {
4990 case kAND:
4991 mOp = is64Bits ? MOP_xandrrrs : MOP_wandrrrs;
4992 break;
4993 case kIOR:
4994 mOp = is64Bits ? MOP_xiorrrrs : MOP_wiorrrrs;
4995 break;
4996 case kEOR:
4997 mOp = is64Bits ? MOP_xeorrrrs : MOP_weorrrrs;
4998 break;
4999 default:
5000 break;
5001 }
5002 return mOp;
5003 }
5004 return mOp;
5005 }
5006
SelectRelationOperator(RelationOperator operatorCode,Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)5007 void AArch64CGFunc::SelectRelationOperator(RelationOperator operatorCode, Operand &resOpnd, Operand &opnd0,
5008 Operand &opnd1, PrimType primType)
5009 {
5010 Operand::OperandType opnd0Type = opnd0.GetKind();
5011 Operand::OperandType opnd1Type = opnd1.GetKind();
5012 uint32 dsize = GetPrimTypeBitSize(primType);
5013 bool is64Bits = (dsize == k64BitSize);
5014 /* op #imm. #imm */
5015 if ((opnd0Type != Operand::kOpdRegister) && (opnd1Type != Operand::kOpdRegister)) {
5016 SelectRelationOperator(operatorCode, resOpnd, SelectCopy(opnd0, primType, primType), opnd1, primType);
5017 return;
5018 }
5019 /* op #imm, reg -> op reg, #imm */
5020 if ((opnd0Type != Operand::kOpdRegister) && (opnd1Type == Operand::kOpdRegister)) {
5021 SelectRelationOperator(operatorCode, resOpnd, opnd1, opnd0, primType);
5022 return;
5023 }
5024 /* op reg, reg */
5025 if ((opnd0Type == Operand::kOpdRegister) && (opnd1Type == Operand::kOpdRegister)) {
5026 DEBUG_ASSERT(IsPrimitiveInteger(primType), "NYI band");
5027 MOperator mOp = SelectRelationMop(operatorCode, kRegReg, is64Bits, false, false);
5028 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, opnd1));
5029 return;
5030 }
5031 /* op reg, #imm */
5032 if ((opnd0Type == Operand::kOpdRegister) && (opnd1Type != Operand::kOpdRegister)) {
5033 if (!((opnd1Type == Operand::kOpdImmediate) || (opnd1Type == Operand::kOpdOffset))) {
5034 SelectRelationOperator(operatorCode, resOpnd, opnd0, SelectCopy(opnd1, primType, primType), primType);
5035 return;
5036 }
5037
5038 ImmOperand *immOpnd = static_cast<ImmOperand *>(&opnd1);
5039 if (immOpnd->IsZero()) {
5040 if (operatorCode == kAND) {
5041 uint32 mopMv = is64Bits ? MOP_xmovrr : MOP_wmovrr;
5042 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopMv, resOpnd, GetZeroOpnd(dsize)));
5043 } else if ((operatorCode == kIOR) || (operatorCode == kEOR)) {
5044 SelectCopy(resOpnd, primType, opnd0, primType);
5045 }
5046 } else if ((immOpnd->IsAllOnes()) || (!is64Bits && immOpnd->IsAllOnes32bit())) {
5047 if (operatorCode == kAND) {
5048 SelectCopy(resOpnd, primType, opnd0, primType);
5049 } else if (operatorCode == kIOR) {
5050 uint32 mopMovn = is64Bits ? MOP_xmovnri16 : MOP_wmovnri16;
5051 ImmOperand &src16 = CreateImmOperand(0, k16BitSize, false);
5052 BitShiftOperand *lslOpnd = GetLogicalShiftLeftOperand(0, is64Bits);
5053 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopMovn, resOpnd, src16, *lslOpnd));
5054 } else if (operatorCode == kEOR) {
5055 SelectMvn(resOpnd, opnd0, primType);
5056 }
5057 } else if (immOpnd->IsBitmaskImmediate()) {
5058 MOperator mOp = SelectRelationMop(operatorCode, kRegImm, is64Bits, true, false);
5059 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, opnd1));
5060 } else {
5061 int64 immVal = immOpnd->GetValue();
5062 int32 tail0BitNum = GetTail0BitNum(immVal);
5063 int32 head0BitNum = GetHead0BitNum(immVal);
5064 const int32 bitNum = (k64BitSizeInt - head0BitNum) - tail0BitNum;
5065 RegOperand ®Opnd = CreateRegisterOperandOfType(primType);
5066
5067 if (bitNum <= k16ValidBit) {
5068 int64 newImm = (static_cast<uint64>(immVal) >> static_cast<uint32>(tail0BitNum)) & 0xFFFF;
5069 ImmOperand &immOpnd1 = CreateImmOperand(newImm, k16BitSize, false);
5070 SelectCopyImm(regOpnd, immOpnd1, primType);
5071 MOperator mOp = SelectRelationMop(operatorCode, kRegImm, is64Bits, false, true);
5072 int32 bitLen = is64Bits ? kBitLenOfShift64Bits : kBitLenOfShift32Bits;
5073 BitShiftOperand &shiftOpnd =
5074 CreateBitShiftOperand(BitShiftOperand::kLSL, static_cast<uint32>(tail0BitNum), bitLen);
5075 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, regOpnd, shiftOpnd));
5076 } else {
5077 SelectCopyImm(regOpnd, *immOpnd, primType);
5078 MOperator mOp = SelectRelationMop(operatorCode, kRegReg, is64Bits, false, false);
5079 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0, regOpnd));
5080 }
5081 }
5082 }
5083 }
5084
SelectBior(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5085 Operand *AArch64CGFunc::SelectBior(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5086 {
5087 return SelectRelationOperator(kIOR, node, opnd0, opnd1, parent);
5088 }
5089
SelectBior(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)5090 void AArch64CGFunc::SelectBior(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
5091 {
5092 SelectRelationOperator(kIOR, resOpnd, opnd0, opnd1, primType);
5093 }
5094
SelectMinOrMax(bool isMin,const BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5095 Operand *AArch64CGFunc::SelectMinOrMax(bool isMin, const BinaryNode &node, Operand &opnd0, Operand &opnd1,
5096 const BaseNode &parent)
5097 {
5098 PrimType dtype = node.GetPrimType();
5099 bool isSigned = IsSignedInteger(dtype);
5100 uint32 dsize = GetPrimTypeBitSize(dtype);
5101 bool is64Bits = (dsize == k64BitSize);
5102 bool isFloat = IsPrimitiveFloat(dtype);
5103 /* promoted type */
5104 PrimType primType = isFloat ? dtype : (is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32));
5105 RegOperand &resOpnd = GetOrCreateResOperand(parent, primType);
5106 SelectMinOrMax(isMin, resOpnd, opnd0, opnd1, primType);
5107 return &resOpnd;
5108 }
5109
SelectMinOrMax(bool isMin,Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)5110 void AArch64CGFunc::SelectMinOrMax(bool isMin, Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
5111 {
5112 uint32 dsize = GetPrimTypeBitSize(primType);
5113 bool is64Bits = (dsize == k64BitSize);
5114 if (IsPrimitiveInteger(primType)) {
5115 RegOperand ®Opnd0 = LoadIntoRegister(opnd0, primType);
5116 Operand ®Opnd1 = LoadIntoRegister(opnd1, primType);
5117 SelectAArch64Cmp(regOpnd0, regOpnd1, true, dsize);
5118 Operand &newResOpnd = LoadIntoRegister(resOpnd, primType);
5119 if (isMin) {
5120 CondOperand &cc = IsSignedInteger(primType) ? GetCondOperand(CC_LT) : GetCondOperand(CC_LO);
5121 SelectAArch64Select(newResOpnd, regOpnd0, regOpnd1, cc, true, dsize);
5122 } else {
5123 CondOperand &cc = IsSignedInteger(primType) ? GetCondOperand(CC_GT) : GetCondOperand(CC_HI);
5124 SelectAArch64Select(newResOpnd, regOpnd0, regOpnd1, cc, true, dsize);
5125 }
5126 } else if (IsPrimitiveFloat(primType)) {
5127 RegOperand ®Opnd0 = LoadIntoRegister(opnd0, primType);
5128 RegOperand ®Opnd1 = LoadIntoRegister(opnd1, primType);
5129 SelectFMinFMax(resOpnd, regOpnd0, regOpnd1, is64Bits, isMin);
5130 } else {
5131 CHECK_FATAL(false, "NIY type max or min");
5132 }
5133 }
5134
SelectMin(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5135 Operand *AArch64CGFunc::SelectMin(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5136 {
5137 return SelectMinOrMax(true, node, opnd0, opnd1, parent);
5138 }
5139
SelectMin(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)5140 void AArch64CGFunc::SelectMin(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
5141 {
5142 SelectMinOrMax(true, resOpnd, opnd0, opnd1, primType);
5143 }
5144
SelectMax(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5145 Operand *AArch64CGFunc::SelectMax(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5146 {
5147 return SelectMinOrMax(false, node, opnd0, opnd1, parent);
5148 }
5149
SelectMax(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)5150 void AArch64CGFunc::SelectMax(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
5151 {
5152 SelectMinOrMax(false, resOpnd, opnd0, opnd1, primType);
5153 }
5154
SelectFMinFMax(Operand & resOpnd,Operand & opnd0,Operand & opnd1,bool is64Bits,bool isMin)5155 void AArch64CGFunc::SelectFMinFMax(Operand &resOpnd, Operand &opnd0, Operand &opnd1, bool is64Bits, bool isMin)
5156 {
5157 uint32 mOpCode = isMin ? (is64Bits ? MOP_xfminrrr : MOP_wfminrrr) : (is64Bits ? MOP_xfmaxrrr : MOP_wfmaxrrr);
5158 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, opnd0, opnd1));
5159 }
5160
SelectBxor(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5161 Operand *AArch64CGFunc::SelectBxor(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5162 {
5163 return SelectRelationOperator(kEOR, node, opnd0, opnd1, parent);
5164 }
5165
SelectBxor(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType)5166 void AArch64CGFunc::SelectBxor(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType)
5167 {
5168 SelectRelationOperator(kEOR, resOpnd, opnd0, opnd1, primType);
5169 }
5170
SelectShift(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5171 Operand *AArch64CGFunc::SelectShift(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5172 {
5173 PrimType dtype = node.GetPrimType();
5174 bool isSigned = IsSignedInteger(dtype);
5175 uint32 dsize = GetPrimTypeBitSize(dtype);
5176 bool is64Bits = (dsize == k64BitSize);
5177 bool isFloat = IsPrimitiveFloat(dtype);
5178 RegOperand *resOpnd = nullptr;
5179 Opcode opcode = node.GetOpCode();
5180
5181 bool isOneElemVector = false;
5182 BaseNode *expr = node.Opnd(0);
5183 if (expr->GetOpCode() == OP_dread) {
5184 MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(static_cast<DreadNode *>(expr)->GetStIdx());
5185 isOneElemVector = symbol->GetAttr(ATTR_oneelem_simd);
5186 }
5187
5188 Operand *opd0 = &opnd0;
5189 PrimType otyp0 = expr->GetPrimType();
5190 if (IsPrimitiveVector(dtype) && opnd0.IsConstImmediate()) {
5191 opd0 = SelectVectorFromScalar(dtype, opd0, node.Opnd(0)->GetPrimType());
5192 otyp0 = dtype;
5193 }
5194
5195 if (IsPrimitiveVector(dtype) && opnd1.IsConstImmediate()) {
5196 int64 sConst = static_cast<ImmOperand &>(opnd1).GetValue();
5197 resOpnd = SelectVectorShiftImm(dtype, opd0, &opnd1, static_cast<int32>(sConst), opcode);
5198 } else if ((IsPrimitiveVector(dtype) || isOneElemVector) && !opnd1.IsConstImmediate()) {
5199 resOpnd = SelectVectorShift(dtype, opd0, otyp0, &opnd1, node.Opnd(1)->GetPrimType(), opcode);
5200 } else {
5201 PrimType primType =
5202 isFloat ? dtype : (is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32));
5203 resOpnd = &GetOrCreateResOperand(parent, primType);
5204 ShiftDirection direct = (opcode == OP_lshr) ? kShiftLright : ((opcode == OP_ashr) ? kShiftAright : kShiftLeft);
5205 SelectShift(*resOpnd, opnd0, opnd1, direct, primType);
5206 }
5207
5208 if (dtype == PTY_i16) {
5209 MOperator exOp = is64Bits ? MOP_xsxth64 : MOP_xsxth32;
5210 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(exOp, *resOpnd, *resOpnd));
5211 } else if (dtype == PTY_i8) {
5212 MOperator exOp = is64Bits ? MOP_xsxtb64 : MOP_xsxtb32;
5213 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(exOp, *resOpnd, *resOpnd));
5214 }
5215 return resOpnd;
5216 }
5217
SelectRor(BinaryNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5218 Operand *AArch64CGFunc::SelectRor(BinaryNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5219 {
5220 PrimType dtype = node.GetPrimType();
5221 uint32 dsize = GetPrimTypeBitSize(dtype);
5222 PrimType primType = (dsize == k64BitSize) ? PTY_u64 : PTY_u32;
5223 RegOperand *resOpnd = &GetOrCreateResOperand(parent, primType);
5224 Operand *firstOpnd = &LoadIntoRegister(opnd0, primType);
5225 MOperator mopRor = (dsize == k64BitSize) ? MOP_xrorrrr : MOP_wrorrrr;
5226 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopRor, *resOpnd, *firstOpnd, opnd1));
5227 return resOpnd;
5228 }
5229
SelectBxorShift(Operand & resOpnd,Operand * opnd0,Operand * opnd1,Operand & opnd2,PrimType primType)5230 void AArch64CGFunc::SelectBxorShift(Operand &resOpnd, Operand *opnd0, Operand *opnd1, Operand &opnd2, PrimType primType)
5231 {
5232 opnd0 = &LoadIntoRegister(*opnd0, primType);
5233 opnd1 = &LoadIntoRegister(*opnd1, primType);
5234 uint32 dsize = GetPrimTypeBitSize(primType);
5235 bool is64Bits = (dsize == k64BitSize);
5236 MOperator mopBxor = is64Bits ? MOP_xeorrrrs : MOP_weorrrrs;
5237 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBxor, resOpnd, *opnd0, *opnd1, opnd2));
5238 }
5239
SelectShift(Operand & resOpnd,Operand & opnd0,Operand & opnd1,ShiftDirection direct,PrimType primType)5240 void AArch64CGFunc::SelectShift(Operand &resOpnd, Operand &opnd0, Operand &opnd1, ShiftDirection direct,
5241 PrimType primType)
5242 {
5243 Operand::OperandType opnd1Type = opnd1.GetKind();
5244 uint32 dsize = GetPrimTypeBitSize(primType);
5245 bool is64Bits = (dsize == k64BitSize);
5246 Operand *firstOpnd = &LoadIntoRegister(opnd0, primType);
5247
5248 MOperator mopShift;
5249 if ((opnd1Type == Operand::kOpdImmediate) || (opnd1Type == Operand::kOpdOffset)) {
5250 ImmOperand *immOpnd1 = static_cast<ImmOperand *>(&opnd1);
5251 const int64 kVal = immOpnd1->GetValue();
5252 const uint32 kShiftamt = is64Bits ? kHighestBitOf64Bits : kHighestBitOf32Bits;
5253 if (kVal == 0) {
5254 SelectCopy(resOpnd, primType, *firstOpnd, primType);
5255 return;
5256 }
5257 /* e.g. a >> -1 */
5258 if ((kVal < 0) || (kVal > kShiftamt)) {
5259 SelectShift(resOpnd, *firstOpnd, SelectCopy(opnd1, primType, primType), direct, primType);
5260 return;
5261 }
5262 switch (direct) {
5263 case kShiftLeft:
5264 mopShift = is64Bits ? MOP_xlslrri6 : MOP_wlslrri5;
5265 break;
5266 case kShiftAright:
5267 mopShift = is64Bits ? MOP_xasrrri6 : MOP_wasrrri5;
5268 break;
5269 case kShiftLright:
5270 mopShift = is64Bits ? MOP_xlsrrri6 : MOP_wlsrrri5;
5271 break;
5272 }
5273 } else if (opnd1Type != Operand::kOpdRegister) {
5274 SelectShift(resOpnd, *firstOpnd, SelectCopy(opnd1, primType, primType), direct, primType);
5275 return;
5276 } else {
5277 switch (direct) {
5278 case kShiftLeft:
5279 mopShift = is64Bits ? MOP_xlslrrr : MOP_wlslrrr;
5280 break;
5281 case kShiftAright:
5282 mopShift = is64Bits ? MOP_xasrrrr : MOP_wasrrrr;
5283 break;
5284 case kShiftLright:
5285 mopShift = is64Bits ? MOP_xlsrrrr : MOP_wlsrrrr;
5286 break;
5287 }
5288 }
5289
5290 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopShift, resOpnd, *firstOpnd, opnd1));
5291 }
5292
SelectAbsSub(Insn & lastInsn,const UnaryNode & node,Operand & newOpnd0)5293 Operand *AArch64CGFunc::SelectAbsSub(Insn &lastInsn, const UnaryNode &node, Operand &newOpnd0)
5294 {
5295 PrimType dtyp = node.GetPrimType();
5296 bool is64Bits = (GetPrimTypeBitSize(dtyp) == k64BitSize);
5297 /* promoted type */
5298 PrimType primType = is64Bits ? (PTY_i64) : (PTY_i32);
5299 RegOperand &resOpnd = CreateRegisterOperandOfType(primType);
5300 uint32 mopCsneg = is64Bits ? MOP_xcnegrrrc : MOP_wcnegrrrc;
5301 /* ABS requires the operand be interpreted as a signed integer */
5302 CondOperand &condOpnd = GetCondOperand(CC_MI);
5303 MOperator newMop = lastInsn.GetMachineOpcode() + 1;
5304 Operand &rflag = GetOrCreateRflag();
5305 std::vector<Operand *> opndVec;
5306 opndVec.push_back(&rflag);
5307 for (uint32 i = 0; i < lastInsn.GetOperandSize(); i++) {
5308 opndVec.push_back(&lastInsn.GetOperand(i));
5309 }
5310 Insn *subsInsn = &GetInsnBuilder()->BuildInsn(newMop, opndVec);
5311 GetCurBB()->ReplaceInsn(lastInsn, *subsInsn);
5312 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopCsneg, resOpnd, newOpnd0, condOpnd, rflag));
5313 return &resOpnd;
5314 }
5315
SelectAbs(UnaryNode & node,Operand & opnd0)5316 Operand *AArch64CGFunc::SelectAbs(UnaryNode &node, Operand &opnd0)
5317 {
5318 PrimType dtyp = node.GetPrimType();
5319 if (IsPrimitiveVector(dtyp)) {
5320 return SelectVectorAbs(dtyp, &opnd0);
5321 } else if (IsPrimitiveFloat(dtyp)) {
5322 CHECK_FATAL(GetPrimTypeBitSize(dtyp) >= k32BitSize, "We don't support hanf-word FP operands yet");
5323 bool is64Bits = (GetPrimTypeBitSize(dtyp) == k64BitSize);
5324 Operand &newOpnd0 = LoadIntoRegister(opnd0, dtyp);
5325 RegOperand &resOpnd = CreateRegisterOperandOfType(dtyp);
5326 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(is64Bits ? MOP_dabsrr : MOP_sabsrr, resOpnd, newOpnd0));
5327 return &resOpnd;
5328 } else {
5329 bool is64Bits = (GetPrimTypeBitSize(dtyp) == k64BitSize);
5330 /* promoted type */
5331 PrimType primType = is64Bits ? (PTY_i64) : (PTY_i32);
5332 Operand &newOpnd0 = LoadIntoRegister(opnd0, primType);
5333 Insn *lastInsn = GetCurBB()->GetLastInsn();
5334 if (lastInsn != nullptr && lastInsn->GetMachineOpcode() >= MOP_xsubrrr &&
5335 lastInsn->GetMachineOpcode() <= MOP_wsubrri12) {
5336 return SelectAbsSub(*lastInsn, node, newOpnd0);
5337 }
5338 RegOperand &resOpnd = CreateRegisterOperandOfType(primType);
5339 SelectAArch64Cmp(newOpnd0, CreateImmOperand(0, is64Bits ? PTY_u64 : PTY_u32, false), true,
5340 GetPrimTypeBitSize(dtyp));
5341 uint32 mopCsneg = is64Bits ? MOP_xcsnegrrrc : MOP_wcsnegrrrc;
5342 /* ABS requires the operand be interpreted as a signed integer */
5343 CondOperand &condOpnd = GetCondOperand(CC_GE);
5344 Operand &rflag = GetOrCreateRflag();
5345 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopCsneg, resOpnd, newOpnd0, newOpnd0, condOpnd, rflag));
5346 return &resOpnd;
5347 }
5348 }
5349
SelectBnot(UnaryNode & node,Operand & opnd0,const BaseNode & parent)5350 Operand *AArch64CGFunc::SelectBnot(UnaryNode &node, Operand &opnd0, const BaseNode &parent)
5351 {
5352 PrimType dtype = node.GetPrimType();
5353 DEBUG_ASSERT(IsPrimitiveInteger(dtype) || IsPrimitiveVectorInteger(dtype), "bnot expect integer or NYI");
5354 uint32 bitSize = GetPrimTypeBitSize(dtype);
5355 bool is64Bits = (bitSize == k64BitSize);
5356 bool isSigned = IsSignedInteger(dtype);
5357 RegOperand *resOpnd = nullptr;
5358 if (!IsPrimitiveVector(dtype)) {
5359 /* promoted type */
5360 PrimType primType = is64Bits ? (isSigned ? PTY_i64 : PTY_u64) : (isSigned ? PTY_i32 : PTY_u32);
5361 resOpnd = &GetOrCreateResOperand(parent, primType);
5362
5363 Operand &newOpnd0 = LoadIntoRegister(opnd0, primType);
5364
5365 uint32 mopBnot = is64Bits ? MOP_xnotrr : MOP_wnotrr;
5366 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBnot, *resOpnd, newOpnd0));
5367 /* generate and resOpnd, resOpnd, 0x1/0xFF/0xFFFF for PTY_u1/PTY_u8/PTY_u16 */
5368 int64 immValue = 0;
5369 if (bitSize == k1BitSize) {
5370 immValue = 1;
5371 } else if (bitSize == k8BitSize) {
5372 immValue = 0xFF;
5373 } else if (bitSize == k16BitSize) {
5374 immValue = 0xFFFF;
5375 }
5376 if (immValue != 0) {
5377 ImmOperand &imm = CreateImmOperand(PTY_u32, immValue);
5378 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wandrri12, *resOpnd, *resOpnd, imm));
5379 }
5380 } else {
5381 /* vector operand */
5382 resOpnd = SelectVectorNot(dtype, &opnd0);
5383 }
5384 return resOpnd;
5385 }
5386
SelectBswap(IntrinsicopNode & node,Operand & opnd0,const BaseNode & parent)5387 Operand *AArch64CGFunc::SelectBswap(IntrinsicopNode &node, Operand &opnd0, const BaseNode &parent)
5388 {
5389 PrimType dtype = node.GetPrimType();
5390 auto bitWidth = (GetPrimTypeBitSize(dtype));
5391 RegOperand *resOpnd = nullptr;
5392 resOpnd = &GetOrCreateResOperand(parent, dtype);
5393 Operand &newOpnd0 = LoadIntoRegister(opnd0, dtype);
5394 uint32 mopBswap = bitWidth == 64 ? MOP_xrevrr : (bitWidth == 32 ? MOP_wrevrr : MOP_wrevrr16);
5395 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBswap, *resOpnd, newOpnd0));
5396 return resOpnd;
5397 }
5398
SelectRegularBitFieldLoad(ExtractbitsNode & node,const BaseNode & parent)5399 Operand *AArch64CGFunc::SelectRegularBitFieldLoad(ExtractbitsNode &node, const BaseNode &parent)
5400 {
5401 PrimType dtype = node.GetPrimType();
5402 bool isSigned = IsSignedInteger(dtype);
5403 uint8 bitOffset = node.GetBitsOffset();
5404 uint8 bitSize = node.GetBitsSize();
5405 bool is64Bits = (GetPrimTypeBitSize(dtype) == k64BitSize);
5406 CHECK_FATAL(!is64Bits, "dest opnd should not be 64bit");
5407 PrimType destType = GetIntegerPrimTypeBySizeAndSign(bitSize, isSigned);
5408 Operand *result =
5409 SelectIread(parent, *static_cast<IreadNode *>(node.Opnd(0)), static_cast<int>(bitOffset / k8BitSize), destType);
5410 return result;
5411 }
5412
SelectExtractbits(ExtractbitsNode & node,Operand & srcOpnd,const BaseNode & parent)5413 Operand *AArch64CGFunc::SelectExtractbits(ExtractbitsNode &node, Operand &srcOpnd, const BaseNode &parent)
5414 {
5415 uint8 bitOffset = node.GetBitsOffset();
5416 uint8 bitSize = node.GetBitsSize();
5417 RegOperand *srcVecRegOperand = static_cast<RegOperand *>(&srcOpnd);
5418 if (srcVecRegOperand && srcVecRegOperand->IsRegister() && (srcVecRegOperand->GetSize() == k128BitSize)) {
5419 if ((bitSize == k8BitSize || bitSize == k16BitSize || bitSize == k32BitSize || bitSize == k64BitSize) &&
5420 (bitOffset % bitSize) == k0BitSize) {
5421 uint32 lane = bitOffset / bitSize;
5422 PrimType srcVecPtype;
5423 if (bitSize == k64BitSize) {
5424 srcVecPtype = PTY_v2u64;
5425 } else if (bitSize == k32BitSize) {
5426 srcVecPtype = PTY_v4u32;
5427 } else if (bitSize == k16BitSize) {
5428 srcVecPtype = PTY_v8u16;
5429 } else {
5430 srcVecPtype = PTY_v16u8;
5431 }
5432 RegOperand *resRegOperand =
5433 SelectVectorGetElement(node.GetPrimType(), &srcOpnd, srcVecPtype, static_cast<int32>(lane));
5434 return resRegOperand;
5435 } else {
5436 CHECK_FATAL(false, "NYI");
5437 }
5438 }
5439 PrimType dtype = node.GetPrimType();
5440 RegOperand &resOpnd = GetOrCreateResOperand(parent, dtype);
5441 bool isSigned =
5442 (node.GetOpCode() == OP_sext) ? true : (node.GetOpCode() == OP_zext) ? false : IsSignedInteger(dtype);
5443 bool is64Bits = (GetPrimTypeBitSize(dtype) == k64BitSize);
5444 uint32 immWidth = is64Bits ? kMaxImmVal13Bits : kMaxImmVal12Bits;
5445 Operand &opnd0 = LoadIntoRegister(srcOpnd, dtype);
5446 if (bitOffset == 0) {
5447 if (!isSigned && (bitSize < immWidth)) {
5448 SelectBand(resOpnd, opnd0,
5449 CreateImmOperand(static_cast<int64>((static_cast<uint64>(1) << bitSize) - 1), immWidth, false),
5450 dtype);
5451 return &resOpnd;
5452 } else {
5453 MOperator mOp = MOP_undef;
5454 if (bitSize == k8BitSize) {
5455 mOp = is64Bits ? (isSigned ? MOP_xsxtb64 : MOP_undef)
5456 : (isSigned ? MOP_xsxtb32 : (opnd0.GetSize() == k32BitSize ? MOP_xuxtb32 : MOP_undef));
5457 } else if (bitSize == k16BitSize) {
5458 mOp = is64Bits ? (isSigned ? MOP_xsxth64 : MOP_undef)
5459 : (isSigned ? MOP_xsxth32 : (opnd0.GetSize() == k32BitSize ? MOP_xuxth32 : MOP_undef));
5460 } else if (bitSize == k32BitSize) {
5461 mOp = is64Bits ? (isSigned ? MOP_xsxtw64 : MOP_xuxtw64) : MOP_wmovrr;
5462 }
5463 if (mOp != MOP_undef) {
5464 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0));
5465 return &resOpnd;
5466 }
5467 }
5468 }
5469 uint32 mopBfx =
5470 is64Bits ? (isSigned ? MOP_xsbfxrri6i6 : MOP_xubfxrri6i6) : (isSigned ? MOP_wsbfxrri5i5 : MOP_wubfxrri5i5);
5471 ImmOperand &immOpnd1 = CreateImmOperand(bitOffset, k8BitSize, false);
5472 ImmOperand &immOpnd2 = CreateImmOperand(bitSize, k8BitSize, false);
5473 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBfx, resOpnd, opnd0, immOpnd1, immOpnd2));
5474 return &resOpnd;
5475 }
5476
5477 /*
5478 * operand fits in MOVK if
5479 * is64Bits && boffst == 0, 16, 32, 48 && bSize == 16, so boffset / 16 == 0, 1, 2, 3; (boffset / 16 ) & (~3) == 0
5480 * or is32Bits && boffset == 0, 16 && bSize == 16, so boffset / 16 == 0, 1; (boffset / 16) & (~1) == 0
5481 * imm range of aarch64-movk [0 - 65536] imm16
5482 */
IsMoveWideKeepable(int64 offsetVal,uint32 bitOffset,uint32 bitSize,bool is64Bits)5483 inline bool IsMoveWideKeepable(int64 offsetVal, uint32 bitOffset, uint32 bitSize, bool is64Bits)
5484 {
5485 DEBUG_ASSERT(is64Bits || (bitOffset < k32BitSize), "");
5486 bool isOutOfRange = offsetVal < 0;
5487 if (!isOutOfRange) {
5488 isOutOfRange = (static_cast<unsigned long int>(offsetVal) >> k16BitSize) > 0;
5489 }
5490 return (!isOutOfRange) && bitSize == k16BitSize &&
5491 ((bitOffset >> k16BitShift) & ~static_cast<uint32>(is64Bits ? 0x3 : 0x1)) == 0;
5492 }
5493
5494 /* we use the fact that A ^ B ^ A == B, A ^ 0 = A */
SelectDepositBits(DepositbitsNode & node,Operand & opnd0,Operand & opnd1,const BaseNode & parent)5495 Operand *AArch64CGFunc::SelectDepositBits(DepositbitsNode &node, Operand &opnd0, Operand &opnd1, const BaseNode &parent)
5496 {
5497 uint32 bitOffset = node.GetBitsOffset();
5498 uint32 bitSize = node.GetBitsSize();
5499 PrimType regType = node.GetPrimType();
5500 bool is64Bits = GetPrimTypeBitSize(regType) == k64BitSize;
5501 /*
5502 * if operand 1 is immediate and fits in MOVK, use it
5503 * MOVK Wd, #imm{, LSL #shift} ; 32-bit general registers
5504 * MOVK Xd, #imm{, LSL #shift} ; 64-bit general registers
5505 */
5506 if (opnd1.IsIntImmediate() &&
5507 IsMoveWideKeepable(static_cast<ImmOperand &>(opnd1).GetValue(), bitOffset, bitSize, is64Bits)) {
5508 RegOperand &resOpnd = GetOrCreateResOperand(parent, regType);
5509 SelectCopy(resOpnd, regType, opnd0, regType);
5510 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn((is64Bits ? MOP_xmovkri16 : MOP_wmovkri16), resOpnd, opnd1,
5511 *GetLogicalShiftLeftOperand(bitOffset, is64Bits)));
5512 return &resOpnd;
5513 } else {
5514 Operand &movOpnd = LoadIntoRegister(opnd1, regType);
5515 uint32 mopBfi = is64Bits ? MOP_xbfirri6i6 : MOP_wbfirri5i5;
5516 ImmOperand &immOpnd1 = CreateImmOperand(bitOffset, k8BitSize, false);
5517 ImmOperand &immOpnd2 = CreateImmOperand(bitSize, k8BitSize, false);
5518 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopBfi, opnd0, movOpnd, immOpnd1, immOpnd2));
5519 return &opnd0;
5520 }
5521 }
5522
SelectLnot(UnaryNode & node,Operand & srcOpnd,const BaseNode & parent)5523 Operand *AArch64CGFunc::SelectLnot(UnaryNode &node, Operand &srcOpnd, const BaseNode &parent)
5524 {
5525 PrimType dtype = node.GetPrimType();
5526 RegOperand &resOpnd = GetOrCreateResOperand(parent, dtype);
5527 bool is64Bits = (GetPrimTypeBitSize(dtype) == k64BitSize);
5528 Operand &opnd0 = LoadIntoRegister(srcOpnd, dtype);
5529 SelectAArch64Cmp(opnd0, CreateImmOperand(0, is64Bits ? PTY_u64 : PTY_u32, false), true, GetPrimTypeBitSize(dtype));
5530 SelectAArch64CSet(resOpnd, GetCondOperand(CC_EQ), is64Bits);
5531 return &resOpnd;
5532 }
5533
SelectNeg(UnaryNode & node,Operand & opnd0,const BaseNode & parent)5534 Operand *AArch64CGFunc::SelectNeg(UnaryNode &node, Operand &opnd0, const BaseNode &parent)
5535 {
5536 PrimType dtype = node.GetPrimType();
5537 bool is64Bits = (GetPrimTypeBitSize(dtype) == k64BitSize);
5538 RegOperand *resOpnd = nullptr;
5539 if (!IsPrimitiveVector(dtype)) {
5540 PrimType primType;
5541 if (IsPrimitiveFloat(dtype)) {
5542 primType = dtype;
5543 } else {
5544 primType = is64Bits ? (PTY_i64) : (PTY_i32); /* promoted type */
5545 }
5546 resOpnd = &GetOrCreateResOperand(parent, primType);
5547 SelectNeg(*resOpnd, opnd0, primType);
5548 } else {
5549 /* vector operand */
5550 resOpnd = SelectVectorNeg(dtype, &opnd0);
5551 }
5552 return resOpnd;
5553 }
5554
SelectNeg(Operand & dest,Operand & srcOpnd,PrimType primType)5555 void AArch64CGFunc::SelectNeg(Operand &dest, Operand &srcOpnd, PrimType primType)
5556 {
5557 Operand &opnd0 = LoadIntoRegister(srcOpnd, primType);
5558 bool is64Bits = (GetPrimTypeBitSize(primType) == k64BitSize);
5559 MOperator mOp;
5560 if (IsPrimitiveFloat(primType)) {
5561 mOp = is64Bits ? MOP_xfnegrr : MOP_wfnegrr;
5562 } else {
5563 mOp = is64Bits ? MOP_xinegrr : MOP_winegrr;
5564 }
5565 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, dest, opnd0));
5566 }
5567
SelectMvn(Operand & dest,Operand & src,PrimType primType)5568 void AArch64CGFunc::SelectMvn(Operand &dest, Operand &src, PrimType primType)
5569 {
5570 Operand &opnd0 = LoadIntoRegister(src, primType);
5571 bool is64Bits = (GetPrimTypeBitSize(primType) == k64BitSize);
5572 MOperator mOp;
5573 DEBUG_ASSERT(!IsPrimitiveFloat(primType), "Instruction 'mvn' do not have float version.");
5574 mOp = is64Bits ? MOP_xnotrr : MOP_wnotrr;
5575 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, dest, opnd0));
5576 }
5577
SelectRecip(UnaryNode & node,Operand & src,const BaseNode & parent)5578 Operand *AArch64CGFunc::SelectRecip(UnaryNode &node, Operand &src, const BaseNode &parent)
5579 {
5580 /*
5581 * fconsts s15, #112
5582 * fdivs s0, s15, s0
5583 */
5584 PrimType dtype = node.GetPrimType();
5585 if (!IsPrimitiveFloat(dtype)) {
5586 DEBUG_ASSERT(false, "should be float type");
5587 return nullptr;
5588 }
5589 Operand &opnd0 = LoadIntoRegister(src, dtype);
5590 RegOperand &resOpnd = GetOrCreateResOperand(parent, dtype);
5591 Operand *one = nullptr;
5592 if (GetPrimTypeBitSize(dtype) == k64BitSize) {
5593 MIRDoubleConst *c = memPool->New<MIRDoubleConst>(1.0, *GlobalTables::GetTypeTable().GetTypeTable().at(PTY_f64));
5594 one = SelectDoubleConst(*c, node);
5595 } else if (GetPrimTypeBitSize(dtype) == k32BitSize) {
5596 MIRFloatConst *c = memPool->New<MIRFloatConst>(1.0f, *GlobalTables::GetTypeTable().GetTypeTable().at(PTY_f32));
5597 one = SelectFloatConst(*c, node);
5598 } else {
5599 CHECK_FATAL(false, "we don't support half-precision fp operations yet");
5600 }
5601 SelectDiv(resOpnd, *one, opnd0, dtype);
5602 return &resOpnd;
5603 }
5604
SelectSqrt(UnaryNode & node,Operand & src,const BaseNode & parent)5605 Operand *AArch64CGFunc::SelectSqrt(UnaryNode &node, Operand &src, const BaseNode &parent)
5606 {
5607 /*
5608 * gcc generates code like below for better accurate
5609 * fsqrts s15, s0
5610 * fcmps s15, s15
5611 * fmstat
5612 * beq .L4
5613 * push {r3, lr}
5614 * bl sqrtf
5615 * pop {r3, pc}
5616 * .L4:
5617 * fcpys s0, s15
5618 * bx lr
5619 */
5620 PrimType dtype = node.GetPrimType();
5621 if (!IsPrimitiveFloat(dtype)) {
5622 DEBUG_ASSERT(false, "should be float type");
5623 return nullptr;
5624 }
5625 bool is64Bits = (GetPrimTypeBitSize(dtype) == k64BitSize);
5626 Operand &opnd0 = LoadIntoRegister(src, dtype);
5627 RegOperand &resOpnd = GetOrCreateResOperand(parent, dtype);
5628 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(is64Bits ? MOP_vsqrtd : MOP_vsqrts, resOpnd, opnd0));
5629 return &resOpnd;
5630 }
5631
SelectCvtFloat2Int(Operand & resOpnd,Operand & srcOpnd,PrimType itype,PrimType ftype)5632 void AArch64CGFunc::SelectCvtFloat2Int(Operand &resOpnd, Operand &srcOpnd, PrimType itype, PrimType ftype)
5633 {
5634 bool is64BitsFloat = (ftype == PTY_f64);
5635 MOperator mOp = 0;
5636
5637 DEBUG_ASSERT(((ftype == PTY_f64) || (ftype == PTY_f32)), "wrong from type");
5638 Operand &opnd0 = LoadIntoRegister(srcOpnd, ftype);
5639 switch (itype) {
5640 case PTY_i32:
5641 mOp = !is64BitsFloat ? MOP_vcvtrf : MOP_vcvtrd;
5642 break;
5643 case PTY_u32:
5644 case PTY_a32:
5645 mOp = !is64BitsFloat ? MOP_vcvturf : MOP_vcvturd;
5646 break;
5647 case PTY_i64:
5648 mOp = !is64BitsFloat ? MOP_xvcvtrf : MOP_xvcvtrd;
5649 break;
5650 case PTY_u64:
5651 case PTY_a64:
5652 mOp = !is64BitsFloat ? MOP_xvcvturf : MOP_xvcvturd;
5653 break;
5654 default:
5655 CHECK_FATAL(false, "unexpected type");
5656 }
5657 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0));
5658 }
5659
SelectCvtInt2Float(Operand & resOpnd,Operand & origOpnd0,PrimType toType,PrimType fromType)5660 void AArch64CGFunc::SelectCvtInt2Float(Operand &resOpnd, Operand &origOpnd0, PrimType toType, PrimType fromType)
5661 {
5662 DEBUG_ASSERT((toType == PTY_f32) || (toType == PTY_f64), "unexpected type");
5663 bool is64BitsFloat = (toType == PTY_f64);
5664 MOperator mOp = 0;
5665 uint32 fsize = GetPrimTypeBitSize(fromType);
5666
5667 PrimType itype = (GetPrimTypeBitSize(fromType) == k64BitSize) ? (IsSignedInteger(fromType) ? PTY_i64 : PTY_u64)
5668 : (IsSignedInteger(fromType) ? PTY_i32 : PTY_u32);
5669
5670 Operand *opnd0 = &LoadIntoRegister(origOpnd0, itype);
5671
5672 /* need extension before cvt */
5673 DEBUG_ASSERT(opnd0->IsRegister(), "opnd should be a register operand");
5674 Operand *srcOpnd = opnd0;
5675 if (IsSignedInteger(fromType) && (fsize < k32BitSize)) {
5676 srcOpnd = &CreateRegisterOperandOfType(itype);
5677 mOp = (fsize == k8BitSize) ? MOP_xsxtb32 : MOP_xsxth32;
5678 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, *srcOpnd, *opnd0));
5679 }
5680
5681 switch (itype) {
5682 case PTY_i32:
5683 mOp = !is64BitsFloat ? MOP_vcvtfr : MOP_vcvtdr;
5684 break;
5685 case PTY_u32:
5686 mOp = !is64BitsFloat ? MOP_vcvtufr : MOP_vcvtudr;
5687 break;
5688 case PTY_i64:
5689 mOp = !is64BitsFloat ? MOP_xvcvtfr : MOP_xvcvtdr;
5690 break;
5691 case PTY_u64:
5692 mOp = !is64BitsFloat ? MOP_xvcvtufr : MOP_xvcvtudr;
5693 break;
5694 default:
5695 CHECK_FATAL(false, "unexpected type");
5696 }
5697 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, *srcOpnd));
5698 }
5699
SelectIntrinsicOpWithOneParam(IntrinsicopNode & intrnNode,std::string name)5700 Operand *AArch64CGFunc::SelectIntrinsicOpWithOneParam(IntrinsicopNode &intrnNode, std::string name)
5701 {
5702 BaseNode *argexpr = intrnNode.Opnd(0);
5703 PrimType ptype = argexpr->GetPrimType();
5704 Operand *opnd = HandleExpr(intrnNode, *argexpr);
5705 if (intrnNode.GetIntrinsic() == INTRN_C_ffs) {
5706 DEBUG_ASSERT(intrnNode.GetPrimType() == PTY_i32, "Unexpect Size");
5707 return SelectAArch64ffs(*opnd, ptype);
5708 }
5709 if (opnd->IsMemoryAccessOperand()) {
5710 RegOperand &ldDest = CreateRegisterOperandOfType(ptype);
5711 Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype), ptype), ldDest, *opnd);
5712 GetCurBB()->AppendInsn(insn);
5713 opnd = &ldDest;
5714 }
5715 std::vector<Operand *> opndVec;
5716 RegOperand *dst = &CreateRegisterOperandOfType(ptype);
5717 opndVec.push_back(dst); /* result */
5718 opndVec.push_back(opnd); /* param 0 */
5719 SelectLibCall(name, opndVec, ptype, ptype);
5720
5721 return dst;
5722 }
5723
SelectIntrinsicOpWithNParams(IntrinsicopNode & intrnNode,PrimType retType,const std::string & name)5724 Operand *AArch64CGFunc::SelectIntrinsicOpWithNParams(IntrinsicopNode &intrnNode, PrimType retType,
5725 const std::string &name)
5726 {
5727 MapleVector<BaseNode *> argNodes = intrnNode.GetNopnd();
5728 std::vector<Operand *> opndVec;
5729 std::vector<PrimType> opndTypes;
5730 RegOperand *retOpnd = &CreateRegisterOperandOfType(retType);
5731 opndVec.push_back(retOpnd);
5732 opndTypes.push_back(retType);
5733
5734 for (BaseNode *argexpr : argNodes) {
5735 PrimType ptype = argexpr->GetPrimType();
5736 Operand *opnd = HandleExpr(intrnNode, *argexpr);
5737 if (opnd->IsMemoryAccessOperand()) {
5738 RegOperand &ldDest = CreateRegisterOperandOfType(ptype);
5739 Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype), ptype), ldDest, *opnd);
5740 GetCurBB()->AppendInsn(insn);
5741 opnd = &ldDest;
5742 }
5743 opndVec.push_back(opnd);
5744 opndTypes.push_back(ptype);
5745 }
5746 SelectLibCallNArg(name, opndVec, opndTypes, retType, false);
5747
5748 return retOpnd;
5749 }
5750
5751 /* According to gcc.target/aarch64/ffs.c */
SelectAArch64ffs(Operand & argOpnd,PrimType argType)5752 Operand *AArch64CGFunc::SelectAArch64ffs(Operand &argOpnd, PrimType argType)
5753 {
5754 RegOperand &destOpnd = LoadIntoRegister(argOpnd, argType);
5755 uint32 argSize = GetPrimTypeBitSize(argType);
5756 DEBUG_ASSERT((argSize == k64BitSize || argSize == k32BitSize), "Unexpect arg type");
5757 /* cmp */
5758 ImmOperand &zeroOpnd = CreateImmOperand(0, argSize, false);
5759 Operand &rflag = GetOrCreateRflag();
5760 GetCurBB()->AppendInsn(
5761 GetInsnBuilder()->BuildInsn(argSize == k64BitSize ? MOP_xcmpri : MOP_wcmpri, rflag, destOpnd, zeroOpnd));
5762 /* rbit */
5763 RegOperand *tempResReg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, GetPrimTypeSize(argType)));
5764 GetCurBB()->AppendInsn(
5765 GetInsnBuilder()->BuildInsn(argSize == k64BitSize ? MOP_xrbit : MOP_wrbit, *tempResReg, destOpnd));
5766 /* clz */
5767 GetCurBB()->AppendInsn(
5768 GetInsnBuilder()->BuildInsn(argSize == k64BitSize ? MOP_xclz : MOP_wclz, *tempResReg, *tempResReg));
5769 /* csincc */
5770 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(argSize == k64BitSize ? MOP_xcsincrrrc : MOP_wcsincrrrc,
5771 *tempResReg, GetZeroOpnd(k32BitSize), *tempResReg,
5772 GetCondOperand(CC_EQ), rflag));
5773 return tempResReg;
5774 }
5775
SelectRoundLibCall(RoundType roundType,const TypeCvtNode & node,Operand & opnd0)5776 Operand *AArch64CGFunc::SelectRoundLibCall(RoundType roundType, const TypeCvtNode &node, Operand &opnd0)
5777 {
5778 PrimType ftype = node.FromType();
5779 PrimType rtype = node.GetPrimType();
5780 bool is64Bits = (ftype == PTY_f64);
5781 std::vector<Operand *> opndVec;
5782 RegOperand *resOpnd;
5783 if (is64Bits) {
5784 resOpnd = &GetOrCreatePhysicalRegisterOperand(D0, k64BitSize, kRegTyFloat);
5785 } else {
5786 resOpnd = &GetOrCreatePhysicalRegisterOperand(S0, k32BitSize, kRegTyFloat);
5787 }
5788 opndVec.push_back(resOpnd);
5789 RegOperand ®Opnd0 = LoadIntoRegister(opnd0, ftype);
5790 opndVec.push_back(®Opnd0);
5791 std::string libName;
5792 if (roundType == kCeil) {
5793 libName.assign(is64Bits ? "ceil" : "ceilf");
5794 } else if (roundType == kFloor) {
5795 libName.assign(is64Bits ? "floor" : "floorf");
5796 } else {
5797 libName.assign(is64Bits ? "round" : "roundf");
5798 }
5799 SelectLibCall(libName, opndVec, ftype, rtype);
5800
5801 return resOpnd;
5802 }
5803
SelectRoundOperator(RoundType roundType,const TypeCvtNode & node,Operand & opnd0,const BaseNode & parent)5804 Operand *AArch64CGFunc::SelectRoundOperator(RoundType roundType, const TypeCvtNode &node, Operand &opnd0,
5805 const BaseNode &parent)
5806 {
5807 PrimType itype = node.GetPrimType();
5808 if ((mirModule.GetSrcLang() == kSrcLangC) && ((itype == PTY_f64) || (itype == PTY_f32))) {
5809 SelectRoundLibCall(roundType, node, opnd0);
5810 }
5811 PrimType ftype = node.FromType();
5812 DEBUG_ASSERT(((ftype == PTY_f64) || (ftype == PTY_f32)), "wrong float type");
5813 bool is64Bits = (ftype == PTY_f64);
5814 RegOperand &resOpnd = GetOrCreateResOperand(parent, itype);
5815 RegOperand ®Opnd0 = LoadIntoRegister(opnd0, ftype);
5816 MOperator mop = MOP_undef;
5817 if (roundType == kCeil) {
5818 mop = is64Bits ? MOP_xvcvtps : MOP_vcvtps;
5819 } else if (roundType == kFloor) {
5820 mop = is64Bits ? MOP_xvcvtms : MOP_vcvtms;
5821 } else {
5822 mop = is64Bits ? MOP_xvcvtas : MOP_vcvtas;
5823 }
5824 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mop, resOpnd, regOpnd0));
5825 return &resOpnd;
5826 }
5827
SelectCeil(TypeCvtNode & node,Operand & opnd0,const BaseNode & parent)5828 Operand *AArch64CGFunc::SelectCeil(TypeCvtNode &node, Operand &opnd0, const BaseNode &parent)
5829 {
5830 return SelectRoundOperator(kCeil, node, opnd0, parent);
5831 }
5832
5833 /* float to int floor */
SelectFloor(TypeCvtNode & node,Operand & opnd0,const BaseNode & parent)5834 Operand *AArch64CGFunc::SelectFloor(TypeCvtNode &node, Operand &opnd0, const BaseNode &parent)
5835 {
5836 return SelectRoundOperator(kFloor, node, opnd0, parent);
5837 }
5838
SelectRound(TypeCvtNode & node,Operand & opnd0,const BaseNode & parent)5839 Operand *AArch64CGFunc::SelectRound(TypeCvtNode &node, Operand &opnd0, const BaseNode &parent)
5840 {
5841 return SelectRoundOperator(kRound, node, opnd0, parent);
5842 }
5843
LIsPrimitivePointer(PrimType ptype)5844 static bool LIsPrimitivePointer(PrimType ptype)
5845 {
5846 return ((ptype >= PTY_ptr) && (ptype <= PTY_a64));
5847 }
5848
SelectRetype(TypeCvtNode & node,Operand & opnd0)5849 Operand *AArch64CGFunc::SelectRetype(TypeCvtNode &node, Operand &opnd0)
5850 {
5851 PrimType fromType = node.Opnd(0)->GetPrimType();
5852 PrimType toType = node.GetPrimType();
5853 DEBUG_ASSERT(GetPrimTypeSize(fromType) == GetPrimTypeSize(toType), "retype bit widith doesn' match");
5854 if (LIsPrimitivePointer(fromType) && LIsPrimitivePointer(toType)) {
5855 return &LoadIntoRegister(opnd0, toType);
5856 }
5857 if (IsPrimitiveVector(fromType) || IsPrimitiveVector(toType)) {
5858 return &LoadIntoRegister(opnd0, toType);
5859 }
5860 Operand::OperandType opnd0Type = opnd0.GetKind();
5861 RegOperand *resOpnd = &CreateRegisterOperandOfType(toType);
5862 if (IsPrimitiveInteger(fromType) || IsPrimitiveFloat(fromType)) {
5863 bool isFromInt = IsPrimitiveInteger(fromType);
5864 bool is64Bits = GetPrimTypeBitSize(fromType) == k64BitSize;
5865 PrimType itype =
5866 isFromInt ? ((GetPrimTypeBitSize(fromType) == k64BitSize) ? (IsSignedInteger(fromType) ? PTY_i64 : PTY_u64)
5867 : (IsSignedInteger(fromType) ? PTY_i32 : PTY_u32))
5868 : (is64Bits ? PTY_f64 : PTY_f32);
5869
5870 /*
5871 * if source operand is in memory,
5872 * simply read it as a value of 'toType 'into the dest operand
5873 * and return
5874 */
5875 if (opnd0Type == Operand::kOpdMem) {
5876 resOpnd = &SelectCopy(opnd0, toType, toType);
5877 return resOpnd;
5878 }
5879 /* according to aarch64 encoding format, convert int to float expression */
5880 bool isImm = false;
5881 ImmOperand *imm = static_cast<ImmOperand *>(&opnd0);
5882 uint64 val = static_cast<uint64>(imm->GetValue());
5883 uint64 canRepreset = is64Bits ? (val & 0xffffffffffff) : (val & 0x7ffff);
5884 uint32 val1 = is64Bits ? (val >> 61) & 0x3 : (val >> 29) & 0x3;
5885 uint32 val2 = is64Bits ? (val >> 54) & 0xff : (val >> 25) & 0x1f;
5886 bool isSame = is64Bits ? ((val2 == 0) || (val2 == 0xff)) : ((val2 == 0) || (val2 == 0x1f));
5887 canRepreset = (canRepreset == 0) && ((val1 & 0x1) ^ ((val1 & 0x2) >> 1)) && isSame;
5888 Operand *newOpnd0 = &opnd0;
5889 if (IsPrimitiveInteger(fromType) && IsPrimitiveFloat(toType) && canRepreset) {
5890 uint64 temp1 = is64Bits ? (val >> 63) << 7 : (val >> 31) << 7;
5891 uint64 temp2 = is64Bits ? val >> 48 : val >> 19;
5892 int64 imm8 = (temp2 & 0x7f) | temp1;
5893 newOpnd0 = &CreateImmOperand(imm8, k8BitSize, false, kNotVary, true);
5894 isImm = true;
5895 } else {
5896 newOpnd0 = &LoadIntoRegister(opnd0, itype);
5897 }
5898 if ((IsPrimitiveFloat(fromType) && IsPrimitiveInteger(toType)) ||
5899 (IsPrimitiveFloat(toType) && IsPrimitiveInteger(fromType))) {
5900 MOperator mopFmov = (isImm ? (is64Bits ? MOP_xdfmovri : MOP_wsfmovri) : isFromInt)
5901 ? (is64Bits ? MOP_xvmovdr : MOP_xvmovsr)
5902 : (is64Bits ? MOP_xvmovrd : MOP_xvmovrs);
5903 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mopFmov, *resOpnd, *newOpnd0));
5904 return resOpnd;
5905 } else {
5906 return newOpnd0;
5907 }
5908 } else {
5909 CHECK_FATAL(false, "NYI retype");
5910 }
5911 return nullptr;
5912 }
5913
SelectCvtFloat2Float(Operand & resOpnd,Operand & srcOpnd,PrimType fromType,PrimType toType)5914 void AArch64CGFunc::SelectCvtFloat2Float(Operand &resOpnd, Operand &srcOpnd, PrimType fromType, PrimType toType)
5915 {
5916 Operand &opnd0 = LoadIntoRegister(srcOpnd, fromType);
5917 MOperator mOp = 0;
5918 switch (toType) {
5919 case PTY_f32: {
5920 CHECK_FATAL(fromType == PTY_f64, "unexpected cvt from type");
5921 mOp = MOP_xvcvtfd;
5922 break;
5923 }
5924 case PTY_f64: {
5925 CHECK_FATAL(fromType == PTY_f32, "unexpected cvt from type");
5926 mOp = MOP_xvcvtdf;
5927 break;
5928 }
5929 default:
5930 CHECK_FATAL(false, "unexpected cvt to type");
5931 }
5932
5933 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, resOpnd, opnd0));
5934 }
5935
5936 /*
5937 * This should be regarded only as a reference.
5938 *
5939 * C11 specification.
5940 * 6.3.1.3 Signed and unsigned integers
5941 * 1 When a value with integer type is converted to another integer
5942 * type other than _Bool, if the value can be represented by the
5943 * new type, it is unchanged.
5944 * 2 Otherwise, if the new type is unsigned, the value is converted
5945 * by repeatedly adding or subtracting one more than the maximum
5946 * value that can be represented in the new type until the value
5947 * is in the range of the new type.60)
5948 * 3 Otherwise, the new type is signed and the value cannot be
5949 * represented in it; either the result is implementation-defined
5950 * or an implementation-defined signal is raised.
5951 */
SelectCvtInt2Int(const BaseNode * parent,Operand * & resOpnd,Operand * opnd0,PrimType fromType,PrimType toType)5952 void AArch64CGFunc::SelectCvtInt2Int(const BaseNode *parent, Operand *&resOpnd, Operand *opnd0, PrimType fromType,
5953 PrimType toType)
5954 {
5955 uint32 fsize = GetPrimTypeBitSize(fromType);
5956 if (fromType == PTY_i128 || fromType == PTY_u128) {
5957 fsize = k64BitSize;
5958 }
5959 uint32 tsize = GetPrimTypeBitSize(toType);
5960 if (toType == PTY_i128 || toType == PTY_u128) {
5961 tsize = k64BitSize;
5962 }
5963 bool isExpand = tsize > fsize;
5964 bool is64Bit = (tsize == k64BitSize);
5965 if ((parent != nullptr) && opnd0->IsIntImmediate() &&
5966 ((parent->GetOpCode() == OP_band) || (parent->GetOpCode() == OP_bior) || (parent->GetOpCode() == OP_bxor) ||
5967 (parent->GetOpCode() == OP_ashr) || (parent->GetOpCode() == OP_lshr) || (parent->GetOpCode() == OP_shl))) {
5968 ImmOperand *simm = static_cast<ImmOperand *>(opnd0);
5969 DEBUG_ASSERT(simm != nullptr, "simm is nullptr in AArch64CGFunc::SelectCvtInt2Int");
5970 bool isSign = false;
5971 int64 origValue = simm->GetValue();
5972 int64 newValue = origValue;
5973 int64 signValue = 0;
5974 if (!isExpand) {
5975 /* 64--->32 */
5976 if (fsize > tsize) {
5977 if (IsSignedInteger(toType)) {
5978 if (origValue < 0) {
5979 signValue = static_cast<int64>(0xFFFFFFFFFFFFFFFFLL & (1ULL << static_cast<uint32>(tsize)));
5980 }
5981 newValue = static_cast<int64>(
5982 (static_cast<uint64>(origValue) & ((1ULL << static_cast<uint32>(tsize)) - 1u)) |
5983 static_cast<uint64>(signValue));
5984 } else {
5985 newValue = static_cast<uint64>(origValue) & ((1ULL << static_cast<uint32>(tsize)) - 1u);
5986 }
5987 }
5988 }
5989 if (IsSignedInteger(toType)) {
5990 isSign = true;
5991 }
5992 resOpnd = &static_cast<Operand &>(CreateImmOperand(newValue, GetPrimTypeSize(toType) * kBitsPerByte, isSign));
5993 return;
5994 }
5995 if (isExpand) { /* Expansion */
5996 /* if cvt expr's parent is add,and,xor and some other,we can use the imm version */
5997 PrimType primType = ((fsize == k64BitSize) ? (IsSignedInteger(fromType) ? PTY_i64 : PTY_u64)
5998 : (IsSignedInteger(fromType) ? PTY_i32 : PTY_u32));
5999 opnd0 = &LoadIntoRegister(*opnd0, primType);
6000
6001 if (IsSignedInteger(fromType)) {
6002 DEBUG_ASSERT((is64Bit || (fsize == k8BitSize || fsize == k16BitSize)), "incorrect from size");
6003
6004 MOperator mOp =
6005 (is64Bit ? ((fsize == k8BitSize) ? MOP_xsxtb64 : ((fsize == k16BitSize) ? MOP_xsxth64 : MOP_xsxtw64))
6006 : ((fsize == k8BitSize) ? MOP_xsxtb32 : MOP_xsxth32));
6007 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, *resOpnd, *opnd0));
6008 } else {
6009 /* Unsigned */
6010 if (is64Bit) {
6011 if (fsize == k8BitSize) {
6012 ImmOperand &immOpnd = CreateImmOperand(0xff, k64BitSize, false);
6013 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xandrri13, *resOpnd, *opnd0, immOpnd));
6014 } else if (fsize == k16BitSize) {
6015 ImmOperand &immOpnd = CreateImmOperand(0xffff, k64BitSize, false);
6016 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xandrri13, *resOpnd, *opnd0, immOpnd));
6017 } else {
6018 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xuxtw64, *resOpnd, *opnd0));
6019 }
6020 } else {
6021 DEBUG_ASSERT(((fsize == k8BitSize) || (fsize == k16BitSize)), "incorrect from size");
6022 if (fsize == k8BitSize) {
6023 static_cast<RegOperand *>(opnd0)->SetValidBitsNum(k8BitSize);
6024 static_cast<RegOperand *>(resOpnd)->SetValidBitsNum(k8BitSize);
6025 }
6026 if (fromType == PTY_u1) {
6027 static_cast<RegOperand *>(opnd0)->SetValidBitsNum(1);
6028 static_cast<RegOperand *>(resOpnd)->SetValidBitsNum(1);
6029 }
6030 GetCurBB()->AppendInsn(
6031 GetInsnBuilder()->BuildInsn((fsize == k8BitSize) ? MOP_xuxtb32 : MOP_xuxth32, *resOpnd, *opnd0));
6032 }
6033 }
6034 } else { /* Same size or truncate */
6035 #ifdef CNV_OPTIMIZE
6036 /*
6037 * No code needed for aarch64 with same reg.
6038 * Just update regno.
6039 */
6040 RegOperand *reg = static_cast<RegOperand *>(resOpnd);
6041 reg->regNo = static_cast<RegOperand *>(opnd0)->regNo;
6042 #else
6043 /*
6044 * This is not really needed if opnd0 is result from a load.
6045 * Hopefully the FE will get rid of the redundant conversions for loads.
6046 */
6047 PrimType primType = ((fsize == k64BitSize) ? (IsSignedInteger(fromType) ? PTY_i64 : PTY_u64)
6048 : (IsSignedInteger(fromType) ? PTY_i32 : PTY_u32));
6049 opnd0 = &LoadIntoRegister(*opnd0, primType);
6050
6051 if (fsize > tsize) {
6052 if (tsize == k8BitSize) {
6053 MOperator mOp = IsSignedInteger(toType) ? MOP_xsxtb32 : MOP_xuxtb32;
6054 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, *resOpnd, *opnd0));
6055 } else if (tsize == k16BitSize) {
6056 MOperator mOp = IsSignedInteger(toType) ? MOP_xsxth32 : MOP_xuxth32;
6057 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, *resOpnd, *opnd0));
6058 } else {
6059 MOperator mOp = IsSignedInteger(toType) ? MOP_xsbfxrri6i6 : MOP_xubfxrri6i6;
6060 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, *resOpnd, *opnd0,
6061 CreateImmOperand(0, k8BitSize, false),
6062 CreateImmOperand(tsize, k8BitSize, false)));
6063 }
6064 } else {
6065 /* same size, so resOpnd can be set */
6066 if ((mirModule.IsJavaModule()) || (IsSignedInteger(fromType) == IsSignedInteger(toType)) ||
6067 (GetPrimTypeSize(toType) >= k4BitSize)) {
6068 resOpnd = opnd0;
6069 } else if (IsUnsignedInteger(toType)) {
6070 MOperator mop;
6071 switch (toType) {
6072 case PTY_u8:
6073 mop = MOP_xuxtb32;
6074 break;
6075 case PTY_u16:
6076 mop = MOP_xuxth32;
6077 break;
6078 default:
6079 CHECK_FATAL(0, "Unhandled unsigned convert");
6080 }
6081 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mop, *resOpnd, *opnd0));
6082 } else {
6083 /* signed target */
6084 uint32 size = GetPrimTypeSize(toType);
6085 MOperator mop;
6086 switch (toType) {
6087 case PTY_i8:
6088 mop = (size > k4BitSize) ? MOP_xsxtb64 : MOP_xsxtb32;
6089 break;
6090 case PTY_i16:
6091 mop = (size > k4BitSize) ? MOP_xsxth64 : MOP_xsxth32;
6092 break;
6093 default:
6094 CHECK_FATAL(0, "Unhandled unsigned convert");
6095 }
6096 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mop, *resOpnd, *opnd0));
6097 }
6098 }
6099 #endif
6100 }
6101 }
6102
SelectCvt(const BaseNode & parent,TypeCvtNode & node,Operand & opnd0)6103 Operand *AArch64CGFunc::SelectCvt(const BaseNode &parent, TypeCvtNode &node, Operand &opnd0)
6104 {
6105 PrimType fromType = node.FromType();
6106 PrimType toType = node.GetPrimType();
6107 if (fromType == toType) {
6108 return &opnd0; /* noop */
6109 }
6110 Operand *resOpnd = &GetOrCreateResOperand(parent, toType);
6111 if (IsPrimitiveFloat(toType) && IsPrimitiveInteger(fromType)) {
6112 SelectCvtInt2Float(*resOpnd, opnd0, toType, fromType);
6113 } else if (IsPrimitiveFloat(fromType) && IsPrimitiveInteger(toType)) {
6114 SelectCvtFloat2Int(*resOpnd, opnd0, toType, fromType);
6115 } else if (IsPrimitiveInteger(fromType) && IsPrimitiveInteger(toType)) {
6116 SelectCvtInt2Int(&parent, resOpnd, &opnd0, fromType, toType);
6117 } else if (IsPrimitiveVector(toType) || IsPrimitiveVector(fromType)) {
6118 CHECK_FATAL(IsPrimitiveVector(toType) && IsPrimitiveVector(fromType), "Invalid vector cvt operands");
6119 SelectVectorCvt(resOpnd, toType, &opnd0, fromType);
6120 } else { /* both are float type */
6121 SelectCvtFloat2Float(*resOpnd, opnd0, fromType, toType);
6122 }
6123 return resOpnd;
6124 }
6125
SelectTrunc(TypeCvtNode & node,Operand & opnd0,const BaseNode & parent)6126 Operand *AArch64CGFunc::SelectTrunc(TypeCvtNode &node, Operand &opnd0, const BaseNode &parent)
6127 {
6128 PrimType ftype = node.FromType();
6129 bool is64Bits = (GetPrimTypeBitSize(node.GetPrimType()) == k64BitSize);
6130 PrimType itype = (is64Bits) ? (IsSignedInteger(node.GetPrimType()) ? PTY_i64 : PTY_u64)
6131 : (IsSignedInteger(node.GetPrimType()) ? PTY_i32 : PTY_u32); /* promoted type */
6132 RegOperand &resOpnd = GetOrCreateResOperand(parent, itype);
6133 SelectCvtFloat2Int(resOpnd, opnd0, itype, ftype);
6134 return &resOpnd;
6135 }
6136
SelectSelect(Operand & resOpnd,Operand & condOpnd,Operand & trueOpnd,Operand & falseOpnd,PrimType dtype,PrimType ctype,bool hasCompare,ConditionCode cc)6137 void AArch64CGFunc::SelectSelect(Operand &resOpnd, Operand &condOpnd, Operand &trueOpnd, Operand &falseOpnd,
6138 PrimType dtype, PrimType ctype, bool hasCompare, ConditionCode cc)
6139 {
6140 DEBUG_ASSERT(&resOpnd != &condOpnd, "resOpnd cannot be the same as condOpnd");
6141 bool isIntType = IsPrimitiveInteger(dtype);
6142 DEBUG_ASSERT((IsPrimitiveInteger(dtype) || IsPrimitiveFloat(dtype)), "unknown type for select");
6143 // making condOpnd and cmpInsn closer will provide more opportunity for opt
6144 Operand &newTrueOpnd = LoadIntoRegister(trueOpnd, dtype);
6145 Operand &newFalseOpnd = LoadIntoRegister(falseOpnd, dtype);
6146 Operand &newCondOpnd = LoadIntoRegister(condOpnd, ctype);
6147 if (hasCompare) {
6148 SelectAArch64Cmp(newCondOpnd, CreateImmOperand(0, ctype, false), true, GetPrimTypeBitSize(ctype));
6149 cc = CC_NE;
6150 }
6151 Operand &newResOpnd = LoadIntoRegister(resOpnd, dtype);
6152 SelectAArch64Select(newResOpnd, newTrueOpnd, newFalseOpnd, GetCondOperand(cc), isIntType,
6153 GetPrimTypeBitSize(dtype));
6154 }
6155
SelectSelect(TernaryNode & expr,Operand & cond,Operand & trueOpnd,Operand & falseOpnd,const BaseNode & parent,bool hasCompare)6156 Operand *AArch64CGFunc::SelectSelect(TernaryNode &expr, Operand &cond, Operand &trueOpnd, Operand &falseOpnd,
6157 const BaseNode &parent, bool hasCompare)
6158 {
6159 PrimType dtype = expr.GetPrimType();
6160 PrimType ctype = expr.Opnd(0)->GetPrimType();
6161
6162 ConditionCode cc = CC_NE;
6163 Opcode opcode = expr.Opnd(0)->GetOpCode();
6164 PrimType cmpType = static_cast<CompareNode *>(expr.Opnd(0))->GetOpndType();
6165 bool isFloat = false;
6166 bool unsignedIntegerComparison = false;
6167 if (!IsPrimitiveVector(cmpType)) {
6168 isFloat = IsPrimitiveFloat(cmpType);
6169 unsignedIntegerComparison = !isFloat && !IsSignedInteger(cmpType);
6170 } else {
6171 isFloat = IsPrimitiveVectorFloat(cmpType);
6172 unsignedIntegerComparison = !isFloat && IsPrimitiveUnSignedVector(cmpType);
6173 }
6174 switch (opcode) {
6175 case OP_eq:
6176 cc = CC_EQ;
6177 break;
6178 case OP_ne:
6179 cc = CC_NE;
6180 break;
6181 case OP_le:
6182 cc = unsignedIntegerComparison ? CC_LS : CC_LE;
6183 break;
6184 case OP_ge:
6185 cc = unsignedIntegerComparison ? CC_HS : CC_GE;
6186 break;
6187 case OP_gt:
6188 cc = unsignedIntegerComparison ? CC_HI : CC_GT;
6189 break;
6190 case OP_lt:
6191 cc = unsignedIntegerComparison ? CC_LO : CC_LT;
6192 break;
6193 default:
6194 hasCompare = true;
6195 break;
6196 }
6197 if (!IsPrimitiveVector(dtype)) {
6198 RegOperand &resOpnd = GetOrCreateResOperand(parent, dtype);
6199 SelectSelect(resOpnd, cond, trueOpnd, falseOpnd, dtype, ctype, hasCompare, cc);
6200 return &resOpnd;
6201 } else {
6202 return SelectVectorSelect(cond, dtype, trueOpnd, falseOpnd);
6203 }
6204 }
6205
6206 /*
6207 * syntax: select <prim-type> (<opnd0>, <opnd1>, <opnd2>)
6208 * <opnd0> must be of integer type.
6209 * <opnd1> and <opnd2> must be of the type given by <prim-type>.
6210 * If <opnd0> is not 0, return <opnd1>. Otherwise, return <opnd2>.
6211 */
SelectAArch64Select(Operand & dest,Operand & o0,Operand & o1,CondOperand & cond,bool isIntType,uint32 dsize)6212 void AArch64CGFunc::SelectAArch64Select(Operand &dest, Operand &o0, Operand &o1, CondOperand &cond, bool isIntType,
6213 uint32 dsize)
6214 {
6215 uint32 mOpCode =
6216 isIntType ? ((dsize == k64BitSize) ? MOP_xcselrrrc : MOP_wcselrrrc)
6217 : ((dsize == k64BitSize) ? MOP_dcselrrrc : ((dsize == k32BitSize) ? MOP_scselrrrc : MOP_hcselrrrc));
6218 Operand &rflag = GetOrCreateRflag();
6219 if (o1.IsImmediate()) {
6220 uint32 movOp = (dsize == k64BitSize ? MOP_xmovri64 : MOP_wmovri32);
6221 RegOperand &movDest =
6222 CreateVirtualRegisterOperand(NewVReg(kRegTyInt, (dsize == k64BitSize) ? k8ByteSize : k4ByteSize));
6223 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(movOp, movDest, o1));
6224 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, dest, o0, movDest, cond, rflag));
6225 return;
6226 }
6227 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOpCode, dest, o0, o1, cond, rflag));
6228 }
6229
SelectRangeGoto(RangeGotoNode & rangeGotoNode,Operand & srcOpnd)6230 void AArch64CGFunc::SelectRangeGoto(RangeGotoNode &rangeGotoNode, Operand &srcOpnd)
6231 {
6232 const SmallCaseVector &switchTable = rangeGotoNode.GetRangeGotoTable();
6233 MIRType *etype = GlobalTables::GetTypeTable().GetTypeFromTyIdx(static_cast<TyIdx>(PTY_a64));
6234 /*
6235 * we store 8-byte displacement ( jump_label - offset_table_address )
6236 * in the table. Refer to AArch64Emit::Emit() in aarch64emit.cpp
6237 */
6238 std::vector<uint32> sizeArray;
6239 sizeArray.emplace_back(switchTable.size());
6240 MIRArrayType *arrayType = memPool->New<MIRArrayType>(etype->GetTypeIndex(), sizeArray);
6241 MIRAggConst *arrayConst = memPool->New<MIRAggConst>(mirModule, *arrayType);
6242 for (const auto &itPair : switchTable) {
6243 LabelIdx labelIdx = itPair.second;
6244 GetCurBB()->PushBackRangeGotoLabel(labelIdx);
6245 MIRConst *mirConst = memPool->New<MIRLblConst>(labelIdx, GetFunction().GetPuidx(), *etype);
6246 arrayConst->AddItem(mirConst, 0);
6247 }
6248
6249 MIRSymbol *lblSt = GetFunction().GetSymTab()->CreateSymbol(kScopeLocal);
6250 lblSt->SetStorageClass(kScFstatic);
6251 lblSt->SetSKind(kStConst);
6252 lblSt->SetTyIdx(arrayType->GetTypeIndex());
6253 lblSt->SetKonst(arrayConst);
6254 std::string lblStr(".LB_");
6255 MIRSymbol *funcSt = GlobalTables::GetGsymTable().GetSymbolFromStidx(GetFunction().GetStIdx().Idx());
6256 uint32 labelIdxTmp = GetLabelIdx();
6257 lblStr += funcSt->GetName();
6258 lblStr += std::to_string(labelIdxTmp++);
6259 SetLabelIdx(labelIdxTmp);
6260 lblSt->SetNameStrIdx(lblStr);
6261 AddEmitSt(GetCurBB()->GetId(), *lblSt);
6262
6263 PrimType itype = rangeGotoNode.Opnd(0)->GetPrimType();
6264 Operand &opnd0 = LoadIntoRegister(srcOpnd, itype);
6265
6266 regno_t vRegNO = NewVReg(kRegTyInt, 8u);
6267 RegOperand *addOpnd = &CreateVirtualRegisterOperand(vRegNO);
6268
6269 int32 minIdx = switchTable[0].first;
6270 SelectAdd(*addOpnd, opnd0,
6271 CreateImmOperand(-static_cast<int64>(minIdx) - static_cast<int64>(rangeGotoNode.GetTagOffset()),
6272 GetPrimTypeBitSize(itype), true),
6273 itype);
6274
6275 /* contains the index */
6276 if (addOpnd->GetSize() != GetPrimTypeBitSize(PTY_u64)) {
6277 addOpnd = static_cast<RegOperand *>(&SelectCopy(*addOpnd, PTY_u64, PTY_u64));
6278 }
6279
6280 RegOperand &baseOpnd = CreateRegisterOperandOfType(PTY_u64);
6281 StImmOperand &stOpnd = CreateStImmOperand(*lblSt, 0, 0);
6282
6283 /* load the address of the switch table */
6284 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrp, baseOpnd, stOpnd));
6285 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrpl12, baseOpnd, baseOpnd, stOpnd));
6286
6287 /* load the displacement into a register by accessing memory at base + index*8 */
6288 Operand *disp = CreateMemOperand(MemOperand::kAddrModeBOrX, k64BitSize, baseOpnd, *addOpnd, k8BitShift);
6289 RegOperand &tgt = CreateRegisterOperandOfType(PTY_a64);
6290 SelectAdd(tgt, baseOpnd, *disp, PTY_u64);
6291 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xbr, tgt));
6292 }
6293
SelectLazyLoad(Operand & opnd0,PrimType primType)6294 Operand *AArch64CGFunc::SelectLazyLoad(Operand &opnd0, PrimType primType)
6295 {
6296 DEBUG_ASSERT(opnd0.IsRegister(), "wrong type.");
6297 RegOperand &resOpnd = CreateRegisterOperandOfType(primType);
6298 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_lazy_ldr, resOpnd, opnd0));
6299 return &resOpnd;
6300 }
6301
SelectLazyLoadStatic(MIRSymbol & st,int64 offset,PrimType primType)6302 Operand *AArch64CGFunc::SelectLazyLoadStatic(MIRSymbol &st, int64 offset, PrimType primType)
6303 {
6304 StImmOperand &srcOpnd = CreateStImmOperand(st, offset, 0);
6305 RegOperand &resOpnd = CreateRegisterOperandOfType(primType);
6306 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_lazy_ldr_static, resOpnd, srcOpnd));
6307 return &resOpnd;
6308 }
6309
SelectLoadArrayClassCache(MIRSymbol & st,int64 offset,PrimType primType)6310 Operand *AArch64CGFunc::SelectLoadArrayClassCache(MIRSymbol &st, int64 offset, PrimType primType)
6311 {
6312 StImmOperand &srcOpnd = CreateStImmOperand(st, offset, 0);
6313 RegOperand &resOpnd = CreateRegisterOperandOfType(primType);
6314 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_arrayclass_cache_ldr, resOpnd, srcOpnd));
6315 return &resOpnd;
6316 }
6317
SelectAlloca(UnaryNode & node,Operand & opnd0)6318 Operand *AArch64CGFunc::SelectAlloca(UnaryNode &node, Operand &opnd0)
6319 {
6320 if (!CGOptions::IsArm64ilp32()) {
6321 DEBUG_ASSERT((node.GetPrimType() == PTY_a64), "wrong type");
6322 }
6323 if (GetCG()->IsLmbc()) {
6324 SetHasVLAOrAlloca(true);
6325 }
6326 PrimType stype = node.Opnd(0)->GetPrimType();
6327 Operand *resOpnd = &opnd0;
6328 if (GetPrimTypeBitSize(stype) < GetPrimTypeBitSize(PTY_u64)) {
6329 resOpnd = &CreateRegisterOperandOfType(PTY_u64);
6330 SelectCvtInt2Int(nullptr, resOpnd, &opnd0, stype, PTY_u64);
6331 }
6332
6333 RegOperand &aliOp = CreateRegisterOperandOfType(PTY_u64);
6334
6335 SelectAdd(aliOp, *resOpnd, CreateImmOperand(kAarch64StackPtrAlignment - 1, k64BitSize, true), PTY_u64);
6336 Operand &shifOpnd = CreateImmOperand(__builtin_ctz(kAarch64StackPtrAlignment), k64BitSize, true);
6337 SelectShift(aliOp, aliOp, shifOpnd, kShiftLright, PTY_u64);
6338 SelectShift(aliOp, aliOp, shifOpnd, kShiftLeft, PTY_u64);
6339 Operand &spOpnd = GetOrCreatePhysicalRegisterOperand(RSP, k64BitSize, kRegTyInt);
6340 SelectSub(spOpnd, spOpnd, aliOp, PTY_u64);
6341 int64 allocaOffset = GetMemlayout()->SizeOfArgsToStackPass();
6342 if (GetCG()->IsLmbc()) {
6343 allocaOffset -= kDivide2 * k8ByteSize;
6344 }
6345 if (allocaOffset > 0) {
6346 RegOperand &resallo = CreateRegisterOperandOfType(PTY_u64);
6347 SelectAdd(resallo, spOpnd, CreateImmOperand(allocaOffset, k64BitSize, true), PTY_u64);
6348 return &resallo;
6349 } else {
6350 return &SelectCopy(spOpnd, PTY_u64, PTY_u64);
6351 }
6352 }
6353
SelectMalloc(UnaryNode & node,Operand & opnd0)6354 Operand *AArch64CGFunc::SelectMalloc(UnaryNode &node, Operand &opnd0)
6355 {
6356 PrimType retType = node.GetPrimType();
6357 DEBUG_ASSERT((retType == PTY_a64), "wrong type");
6358
6359 std::vector<Operand *> opndVec;
6360 RegOperand &resOpnd = CreateRegisterOperandOfType(retType);
6361 opndVec.emplace_back(&resOpnd);
6362 opndVec.emplace_back(&opnd0);
6363 /* Use calloc to make sure allocated memory is zero-initialized */
6364 const std::string &funcName = "calloc";
6365 PrimType srcPty = PTY_u64;
6366 if (opnd0.GetSize() <= k32BitSize) {
6367 srcPty = PTY_u32;
6368 }
6369 Operand &opnd1 = CreateImmOperand(1, srcPty, false);
6370 opndVec.emplace_back(&opnd1);
6371 SelectLibCall(funcName, opndVec, srcPty, retType);
6372 return &resOpnd;
6373 }
6374
SelectGCMalloc(GCMallocNode & node)6375 Operand *AArch64CGFunc::SelectGCMalloc(GCMallocNode &node)
6376 {
6377 PrimType retType = node.GetPrimType();
6378 DEBUG_ASSERT((retType == PTY_a64), "wrong type");
6379
6380 /* Get the size and alignment of the type. */
6381 TyIdx tyIdx = node.GetTyIdx();
6382 uint64 size = GetBecommon().GetTypeSize(tyIdx);
6383 uint8 align = RTSupport::GetRTSupportInstance().GetObjectAlignment();
6384
6385 /* Generate the call to MCC_NewObj */
6386 Operand &opndSize = CreateImmOperand(static_cast<int64>(size), k64BitSize, false);
6387 Operand &opndAlign = CreateImmOperand(align, k64BitSize, false);
6388
6389 RegOperand &resOpnd = CreateRegisterOperandOfType(retType);
6390
6391 std::vector<Operand *> opndVec {&resOpnd, &opndSize, &opndAlign};
6392
6393 const std::string &funcName = "MCC_NewObj";
6394 SelectLibCall(funcName, opndVec, PTY_u64, retType);
6395
6396 return &resOpnd;
6397 }
6398
SelectJarrayMalloc(JarrayMallocNode & node,Operand & opnd0)6399 Operand *AArch64CGFunc::SelectJarrayMalloc(JarrayMallocNode &node, Operand &opnd0)
6400 {
6401 PrimType retType = node.GetPrimType();
6402 DEBUG_ASSERT((retType == PTY_a64), "wrong type");
6403
6404 /* Extract jarray type */
6405 TyIdx tyIdx = node.GetTyIdx();
6406 MIRType *type = GlobalTables::GetTypeTable().GetTypeFromTyIdx(tyIdx);
6407 DEBUG_ASSERT(type != nullptr, "nullptr check");
6408 CHECK_FATAL(type->GetKind() == kTypeJArray, "expect MIRJarrayType");
6409 auto jaryType = static_cast<MIRJarrayType *>(type);
6410 uint64 fixedSize = RTSupport::GetRTSupportInstance().GetArrayContentOffset();
6411 uint8 align = RTSupport::GetRTSupportInstance().GetObjectAlignment();
6412
6413 MIRType *elemType = GlobalTables::GetTypeTable().GetTypeFromTyIdx(jaryType->GetElemTyIdx());
6414 PrimType elemPrimType = elemType->GetPrimType();
6415 uint64 elemSize = GetPrimTypeSize(elemPrimType);
6416
6417 /* Generate the cal to MCC_NewObj_flexible */
6418 Operand &opndFixedSize = CreateImmOperand(PTY_u64, static_cast<int64>(fixedSize));
6419 Operand &opndElemSize = CreateImmOperand(PTY_u64, static_cast<int64>(elemSize));
6420
6421 Operand *opndNElems = &opnd0;
6422
6423 Operand *opndNElems64 = &static_cast<Operand &>(CreateRegisterOperandOfType(PTY_u64));
6424 SelectCvtInt2Int(nullptr, opndNElems64, opndNElems, PTY_u32, PTY_u64);
6425
6426 Operand &opndAlign = CreateImmOperand(PTY_u64, align);
6427
6428 RegOperand &resOpnd = CreateRegisterOperandOfType(retType);
6429
6430 std::vector<Operand *> opndVec {&resOpnd, &opndFixedSize, &opndElemSize, opndNElems64, &opndAlign};
6431
6432 const std::string &funcName = "MCC_NewObj_flexible";
6433 SelectLibCall(funcName, opndVec, PTY_u64, retType);
6434
6435 /* Generate the store of the object length field */
6436 MemOperand &opndArrayLengthField =
6437 CreateMemOpnd(resOpnd, static_cast<int64>(RTSupport::GetRTSupportInstance().GetArrayLengthOffset()), k4BitSize);
6438 RegOperand *regOpndNElems = &SelectCopy(*opndNElems, PTY_u32, PTY_u32);
6439 DEBUG_ASSERT(regOpndNElems != nullptr, "null ptr check!");
6440 SelectCopy(opndArrayLengthField, PTY_u32, *regOpndNElems, PTY_u32);
6441
6442 return &resOpnd;
6443 }
6444
IsRegRematCand(const RegOperand & reg) const6445 bool AArch64CGFunc::IsRegRematCand(const RegOperand ®) const
6446 {
6447 MIRPreg *preg = GetPseudoRegFromVirtualRegNO(reg.GetRegisterNumber(), CGOptions::DoCGSSA());
6448 if (preg != nullptr && preg->GetOp() != OP_undef) {
6449 if (preg->GetOp() == OP_constval && cg->GetRematLevel() >= kRematConst) {
6450 return true;
6451 } else if (preg->GetOp() == OP_addrof && cg->GetRematLevel() >= kRematAddr) {
6452 return true;
6453 } else if (preg->GetOp() == OP_iread && cg->GetRematLevel() >= kRematDreadGlobal) {
6454 return true;
6455 } else {
6456 return false;
6457 }
6458 } else {
6459 return false;
6460 }
6461 }
6462
ClearRegRematInfo(const RegOperand & reg) const6463 void AArch64CGFunc::ClearRegRematInfo(const RegOperand ®) const
6464 {
6465 MIRPreg *preg = GetPseudoRegFromVirtualRegNO(reg.GetRegisterNumber(), CGOptions::DoCGSSA());
6466 if (preg != nullptr && preg->GetOp() != OP_undef) {
6467 preg->SetOp(OP_undef);
6468 }
6469 }
6470
IsRegSameRematInfo(const RegOperand & regDest,const RegOperand & regSrc) const6471 bool AArch64CGFunc::IsRegSameRematInfo(const RegOperand ®Dest, const RegOperand ®Src) const
6472 {
6473 MIRPreg *pregDest = GetPseudoRegFromVirtualRegNO(regDest.GetRegisterNumber(), CGOptions::DoCGSSA());
6474 MIRPreg *pregSrc = GetPseudoRegFromVirtualRegNO(regSrc.GetRegisterNumber(), CGOptions::DoCGSSA());
6475 if (pregDest != nullptr && pregDest == pregSrc) {
6476 if (pregDest->GetOp() == OP_constval && cg->GetRematLevel() >= kRematConst) {
6477 return true;
6478 } else if (pregDest->GetOp() == OP_addrof && cg->GetRematLevel() >= kRematAddr) {
6479 return true;
6480 } else if (pregDest->GetOp() == OP_iread && cg->GetRematLevel() >= kRematDreadGlobal) {
6481 return true;
6482 } else {
6483 return false;
6484 }
6485 } else {
6486 return false;
6487 }
6488 }
6489
ReplaceOpndInInsn(RegOperand & regDest,RegOperand & regSrc,Insn & insn,regno_t destNO)6490 void AArch64CGFunc::ReplaceOpndInInsn(RegOperand ®Dest, RegOperand ®Src, Insn &insn, regno_t destNO)
6491 {
6492 auto opndNum = static_cast<int32>(insn.GetOperandSize());
6493 for (int i = opndNum - 1; i >= 0; --i) {
6494 Operand &opnd = insn.GetOperand(static_cast<uint32>(i));
6495 if (opnd.IsList()) {
6496 std::list<RegOperand *> tempRegStore;
6497 auto &opndList = static_cast<ListOperand &>(opnd).GetOperands();
6498 bool needReplace = false;
6499 for (auto it = opndList.cbegin(), end = opndList.cend(); it != end; ++it) {
6500 auto *regOpnd = *it;
6501 if (regOpnd->GetRegisterNumber() == destNO) {
6502 needReplace = true;
6503 tempRegStore.push_back(®Src);
6504 } else {
6505 tempRegStore.push_back(regOpnd);
6506 }
6507 }
6508 if (needReplace) {
6509 opndList.clear();
6510 for (auto &newOpnd : std::as_const(tempRegStore)) {
6511 static_cast<ListOperand &>(opnd).PushOpnd(*newOpnd);
6512 }
6513 }
6514 } else if (opnd.IsMemoryAccessOperand()) {
6515 auto &memOpnd = static_cast<MemOperand &>(opnd);
6516 RegOperand *baseRegOpnd = memOpnd.GetBaseRegister();
6517 RegOperand *indexRegOpnd = memOpnd.GetIndexRegister();
6518 MemOperand *newMem = static_cast<MemOperand *>(memOpnd.Clone(*GetMemoryPool()));
6519 if ((baseRegOpnd != nullptr && baseRegOpnd->GetRegisterNumber() == destNO) ||
6520 (indexRegOpnd != nullptr && indexRegOpnd->GetRegisterNumber() == destNO)) {
6521 if (baseRegOpnd != nullptr && baseRegOpnd->GetRegisterNumber() == destNO) {
6522 newMem->SetBaseRegister(regSrc);
6523 }
6524 if (indexRegOpnd != nullptr && indexRegOpnd->GetRegisterNumber() == destNO) {
6525 auto *newRegSrc = static_cast<RegOperand*>(regSrc.Clone(*GetMemoryPool()));
6526 newRegSrc->SetSize(indexRegOpnd->GetSize()); // retain the original size
6527 newMem->SetIndexRegister(*newRegSrc);
6528 }
6529 insn.SetMemOpnd(newMem);
6530 }
6531 } else if (opnd.IsRegister()) {
6532 auto ®Opnd = static_cast<RegOperand &>(opnd);
6533 if (regOpnd.GetRegisterNumber() == destNO) {
6534 DEBUG_ASSERT(regOpnd.GetRegisterNumber() != kRFLAG, "both condi and reg");
6535 if (regOpnd.GetBaseRefOpnd() != nullptr) {
6536 regSrc.SetBaseRefOpnd(*regOpnd.GetBaseRefOpnd());
6537 ReplaceRegReference(regOpnd.GetRegisterNumber(), regSrc.GetRegisterNumber());
6538 }
6539 insn.SetOperand(static_cast<uint32>(i), regSrc);
6540 }
6541 if (regOpnd.GetBaseRefOpnd() != nullptr && regOpnd.GetBaseRefOpnd()->GetRegisterNumber() == destNO) {
6542 regOpnd.SetBaseRefOpnd(regSrc);
6543 ReplaceRegReference(regOpnd.GetBaseRefOpnd()->GetRegisterNumber(), regSrc.GetRegisterNumber());
6544 }
6545 }
6546 }
6547 if (insn.GetStackMap() != nullptr) {
6548 for (auto [deoptVreg, opnd] : insn.GetStackMap()->GetDeoptInfo().GetDeoptBundleInfo()) {
6549 if (!opnd->IsRegister()) {
6550 continue;
6551 }
6552 auto ®Opnd = static_cast<RegOperand&>(*opnd);
6553 if (regOpnd.GetRegisterNumber() == destNO) {
6554 insn.GetStackMap()->GetDeoptInfo().ReplaceDeoptBundleInfo(deoptVreg, regSrc);
6555 ReplaceRegReference(regOpnd.GetRegisterNumber(), regSrc.GetRegisterNumber());
6556 }
6557 }
6558 }
6559 }
6560
CleanupDeadMov(bool dumpInfo)6561 void AArch64CGFunc::CleanupDeadMov(bool dumpInfo)
6562 {
6563 /* clean dead mov. */
6564 FOR_ALL_BB(bb, this) {
6565 FOR_BB_INSNS_SAFE(insn, bb, ninsn) {
6566 if (!insn->IsMachineInstruction()) {
6567 continue;
6568 }
6569 if (insn->GetMachineOpcode() == MOP_xmovrr || insn->GetMachineOpcode() == MOP_wmovrr ||
6570 insn->GetMachineOpcode() == MOP_xvmovs || insn->GetMachineOpcode() == MOP_xvmovd) {
6571 RegOperand ®Dest = static_cast<RegOperand &>(insn->GetOperand(kInsnFirstOpnd));
6572 RegOperand ®Src = static_cast<RegOperand &>(insn->GetOperand(kInsnSecondOpnd));
6573 if (!regSrc.IsVirtualRegister() || !regDest.IsVirtualRegister()) {
6574 continue;
6575 }
6576
6577 if (regSrc.GetRegisterNumber() == regDest.GetRegisterNumber()) {
6578 bb->RemoveInsn(*insn);
6579 } else if (insn->IsPhiMovInsn() && dumpInfo) {
6580 LogInfo::MapleLogger() << "fail to remove mov: " << regDest.GetRegisterNumber() << " <- "
6581 << regSrc.GetRegisterNumber() << std::endl;
6582 }
6583 }
6584 }
6585 }
6586 }
6587
GetRealCallerSaveRegs(const Insn & insn,std::set<regno_t> & realSaveRegs)6588 void AArch64CGFunc::GetRealCallerSaveRegs(const Insn &insn, std::set<regno_t> &realSaveRegs)
6589 {
6590 auto *targetOpnd = insn.GetCallTargetOperand();
6591 CHECK_FATAL(targetOpnd != nullptr, "target is null in AArch64Insn::IsCallToFunctionThatNeverReturns");
6592 if (CGOptions::DoIPARA() && targetOpnd->IsFuncNameOpnd()) {
6593 FuncNameOperand *target = static_cast<FuncNameOperand *>(targetOpnd);
6594 const MIRSymbol *funcSt = target->GetFunctionSymbol();
6595 DEBUG_ASSERT(funcSt->GetSKind() == kStFunc, "funcst must be a function name symbol");
6596 MIRFunction *func = funcSt->GetFunction();
6597 if (func != nullptr && func->IsReferedRegsValid()) {
6598 for (auto preg : func->GetReferedRegs()) {
6599 if (AArch64Abi::IsCallerSaveReg(static_cast<AArch64reg>(preg))) {
6600 realSaveRegs.insert(preg);
6601 }
6602 }
6603 return;
6604 }
6605 }
6606 for (uint32 i = R0; i <= kMaxRegNum; ++i) {
6607 if (AArch64Abi::IsCallerSaveReg(static_cast<AArch64reg>(i))) {
6608 realSaveRegs.insert(i);
6609 }
6610 }
6611 }
6612
GetZeroOpnd(uint32 bitLen)6613 RegOperand &AArch64CGFunc::GetZeroOpnd(uint32 bitLen)
6614 {
6615 /*
6616 * It is possible to have a bitLen < 32, eg stb.
6617 * Set it to 32 if it is less than 32.
6618 */
6619 if (bitLen < k32BitSize) {
6620 bitLen = k32BitSize;
6621 }
6622 DEBUG_ASSERT((bitLen == k32BitSize || bitLen == k64BitSize), "illegal bit length = %d", bitLen);
6623 return (bitLen == k32BitSize) ? GetOrCreatePhysicalRegisterOperand(RZR, k32BitSize, kRegTyInt)
6624 : GetOrCreatePhysicalRegisterOperand(RZR, k64BitSize, kRegTyInt);
6625 }
6626
IsFrameReg(const RegOperand & opnd) const6627 bool AArch64CGFunc::IsFrameReg(const RegOperand &opnd) const
6628 {
6629 if (opnd.GetRegisterNumber() == RFP) {
6630 return true;
6631 } else {
6632 return false;
6633 }
6634 }
6635
IsSaveReg(const RegOperand & reg,MIRType & mirType,BECommon & cgBeCommon)6636 bool AArch64CGFunc::IsSaveReg(const RegOperand ®, MIRType &mirType, BECommon &cgBeCommon)
6637 {
6638 CCImpl &retLocator = *GetOrCreateLocator(GetCurCallConvKind());
6639 CCLocInfo retMechanism;
6640 retLocator.InitReturnInfo(mirType, retMechanism);
6641 if (retMechanism.GetRegCount() > 0) {
6642 return reg.GetRegisterNumber() == retMechanism.GetReg0() || reg.GetRegisterNumber() == retMechanism.GetReg1() ||
6643 reg.GetRegisterNumber() == retMechanism.GetReg2() || reg.GetRegisterNumber() == retMechanism.GetReg3();
6644 }
6645 return false;
6646 }
6647
IsSPOrFP(const RegOperand & opnd) const6648 bool AArch64CGFunc::IsSPOrFP(const RegOperand &opnd) const
6649 {
6650 const RegOperand ®Opnd = static_cast<const RegOperand &>(opnd);
6651 regno_t regNO = opnd.GetRegisterNumber();
6652 return (regOpnd.IsPhysicalRegister() &&
6653 (regNO == RSP || regNO == RFP || (regNO == R29 && CGOptions::UseFramePointer())));
6654 }
6655
IsReturnReg(const RegOperand & opnd) const6656 bool AArch64CGFunc::IsReturnReg(const RegOperand &opnd) const
6657 {
6658 regno_t regNO = opnd.GetRegisterNumber();
6659 return (regNO == R0) || (regNO == V0);
6660 }
6661
6662 /*
6663 * This function returns true to indicate that the clean up code needs to be generated,
6664 * otherwise it does not need. In GCOnly mode, it always returns false.
6665 */
NeedCleanup()6666 bool AArch64CGFunc::NeedCleanup()
6667 {
6668 if (CGOptions::IsGCOnly()) {
6669 return false;
6670 }
6671 AArch64MemLayout *layout = static_cast<AArch64MemLayout *>(GetMemlayout());
6672 if (layout->GetSizeOfRefLocals() > 0) {
6673 return true;
6674 }
6675 for (uint32 i = 0; i < GetFunction().GetFormalCount(); i++) {
6676 TypeAttrs ta = GetFunction().GetNthParamAttr(i);
6677 if (ta.GetAttr(ATTR_localrefvar)) {
6678 return true;
6679 }
6680 }
6681
6682 return false;
6683 }
6684
6685 /*
6686 * bb must be the cleanup bb.
6687 * this function must be invoked before register allocation.
6688 * extended epilogue is specific for fast exception handling and is made up of
6689 * clean up code and epilogue.
6690 * clean up code is generated here while epilogue is generated in GeneratePrologEpilog()
6691 */
GenerateCleanupCodeForExtEpilog(BB & bb)6692 void AArch64CGFunc::GenerateCleanupCodeForExtEpilog(BB &bb)
6693 {
6694 DEBUG_ASSERT(GetLastBB()->GetPrev()->GetFirstStmt() == GetCleanupLabel(), "must be");
6695
6696 if (NeedCleanup()) {
6697 /* this is necessary for code insertion. */
6698 SetCurBB(bb);
6699
6700 RegOperand ®Opnd0 =
6701 GetOrCreatePhysicalRegisterOperand(R0, GetPointerSize() * kBitsPerByte, GetRegTyFromPrimTy(PTY_a64));
6702 RegOperand ®Opnd1 =
6703 GetOrCreatePhysicalRegisterOperand(R1, GetPointerSize() * kBitsPerByte, GetRegTyFromPrimTy(PTY_a64));
6704 /* allocate 16 bytes to store reg0 and reg1 (each reg has 8 bytes) */
6705 MemOperand &frameAlloc = CreateCallFrameOperand(-16, GetPointerSize() * kBitsPerByte);
6706 Insn &allocInsn = GetInsnBuilder()->BuildInsn(MOP_xstp, regOpnd0, regOpnd1, frameAlloc);
6707 allocInsn.SetDoNotRemove(true);
6708 AppendInstructionTo(allocInsn, *this);
6709
6710 /* invoke MCC_CleanupLocalStackRef(). */
6711 HandleRCCall(false);
6712 /* deallocate 16 bytes which used to store reg0 and reg1 */
6713 MemOperand &frameDealloc = CreateCallFrameOperand(16, GetPointerSize() * kBitsPerByte);
6714 GenRetCleanup(cleanEANode, true);
6715 Insn &deallocInsn = GetInsnBuilder()->BuildInsn(MOP_xldp, regOpnd0, regOpnd1, frameDealloc);
6716 deallocInsn.SetDoNotRemove(true);
6717 AppendInstructionTo(deallocInsn, *this);
6718 /* Update cleanupbb since bb may have been splitted */
6719 SetCleanupBB(*GetCurBB());
6720 }
6721 }
6722
6723 /*
6724 * bb must be the cleanup bb.
6725 * this function must be invoked before register allocation.
6726 */
GenerateCleanupCode(BB & bb)6727 void AArch64CGFunc::GenerateCleanupCode(BB &bb)
6728 {
6729 DEBUG_ASSERT(GetLastBB()->GetPrev()->GetFirstStmt() == GetCleanupLabel(), "must be");
6730 if (!NeedCleanup()) {
6731 return;
6732 }
6733
6734 /* this is necessary for code insertion. */
6735 SetCurBB(bb);
6736
6737 /* R0 is lived-in for clean-up code, save R0 before invocation */
6738 RegOperand &livein = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
6739
6740 if (!GetCG()->GenLocalRC()) {
6741 /* by pass local RC operations. */
6742 } else if (Globals::GetInstance()->GetOptimLevel() > CGOptions::kLevel0) {
6743 regno_t vreg = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
6744 RegOperand &backupRegOp = CreateVirtualRegisterOperand(vreg);
6745 backupRegOp.SetRegNotBBLocal();
6746 SelectCopy(backupRegOp, PTY_a64, livein, PTY_a64);
6747
6748 /* invoke MCC_CleanupLocalStackRef(). */
6749 HandleRCCall(false);
6750 SelectCopy(livein, PTY_a64, backupRegOp, PTY_a64);
6751 } else {
6752 /*
6753 * Register Allocation for O0 can not handle this case, so use a callee saved register directly.
6754 * If yieldpoint is enabled, we use R20 instead R19.
6755 */
6756 AArch64reg backupRegNO = GetCG()->GenYieldPoint() ? R20 : R19;
6757 RegOperand &backupRegOp =
6758 GetOrCreatePhysicalRegisterOperand(backupRegNO, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
6759 SelectCopy(backupRegOp, PTY_a64, livein, PTY_a64);
6760 /* invoke MCC_CleanupLocalStackRef(). */
6761 HandleRCCall(false);
6762 SelectCopy(livein, PTY_a64, backupRegOp, PTY_a64);
6763 }
6764
6765 /* invoke _Unwind_Resume */
6766 std::string funcName("_Unwind_Resume");
6767 MIRSymbol *sym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
6768 sym->SetNameStrIdx(funcName);
6769 sym->SetStorageClass(kScText);
6770 sym->SetSKind(kStFunc);
6771 ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
6772 srcOpnds->PushOpnd(livein);
6773 AppendCall(*sym, *srcOpnds);
6774 /*
6775 * this instruction is unreachable, but we need it as the return address of previous
6776 * "bl _Unwind_Resume" for stack unwinding.
6777 */
6778 Insn &nop = GetInsnBuilder()->BuildInsn(MOP_xblr, livein, *srcOpnds);
6779 GetCurBB()->AppendInsn(nop);
6780 GetCurBB()->SetHasCall();
6781
6782 /* Update cleanupbb since bb may have been splitted */
6783 SetCleanupBB(*GetCurBB());
6784 }
6785
FloatParamRegRequired(MIRStructType * structType,uint32 & fpSize)6786 uint32 AArch64CGFunc::FloatParamRegRequired(MIRStructType *structType, uint32 &fpSize)
6787 {
6788 CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
6789 AArch64CallConvImpl parmlocator(GetBecommon());
6790 return parmlocator.FloatParamRegRequired(*structType, fpSize);
6791 }
6792
6793 /*
6794 * Map param registers to formals. For small structs passed in param registers,
6795 * create a move to vreg since lmbc IR does not create a regassign for them.
6796 */
AssignLmbcFormalParams()6797 void AArch64CGFunc::AssignLmbcFormalParams()
6798 {
6799 PrimType primType;
6800 uint32 offset;
6801 regno_t intReg = R0;
6802 regno_t fpReg = V0;
6803 for (auto param : GetLmbcParamVec()) {
6804 primType = param->GetPrimType();
6805 offset = param->GetOffset();
6806 if (param->IsReturn()) {
6807 param->SetRegNO(R8);
6808 } else if (IsPrimitiveInteger(primType)) {
6809 if (intReg > R7) {
6810 param->SetRegNO(0);
6811 } else {
6812 param->SetRegNO(intReg);
6813 if (!param->HasRegassign()) {
6814 uint32 bytelen = GetPrimTypeSize(primType);
6815 uint32 bitlen = bytelen * kBitsPerByte;
6816 MemOperand *mOpnd = GenLmbcFpMemOperand(static_cast<int32>(offset), bytelen);
6817 RegOperand &src = GetOrCreatePhysicalRegisterOperand(AArch64reg(intReg), bitlen, kRegTyInt);
6818 MOperator mOp = PickStInsn(bitlen, primType);
6819 Insn &store = GetInsnBuilder()->BuildInsn(mOp, src, *mOpnd);
6820 GetCurBB()->AppendInsn(store);
6821 }
6822 intReg++;
6823 }
6824 } else if (IsPrimitiveFloat(primType)) {
6825 if (fpReg > V7) {
6826 param->SetRegNO(0);
6827 } else {
6828 param->SetRegNO(fpReg);
6829 if (!param->HasRegassign()) {
6830 uint32 bytelen = GetPrimTypeSize(primType);
6831 uint32 bitlen = bytelen * kBitsPerByte;
6832 MemOperand *mOpnd = GenLmbcFpMemOperand(static_cast<int32>(offset), bytelen);
6833 RegOperand &src = GetOrCreatePhysicalRegisterOperand(AArch64reg(fpReg), bitlen, kRegTyFloat);
6834 MOperator mOp = PickStInsn(bitlen, primType);
6835 Insn &store = GetInsnBuilder()->BuildInsn(mOp, src, *mOpnd);
6836 GetCurBB()->AppendInsn(store);
6837 }
6838 fpReg++;
6839 }
6840 } else if (primType == PTY_agg) {
6841 if (param->IsPureFloat()) {
6842 uint32 numFpRegs = param->GetNumRegs();
6843 if ((fpReg + numFpRegs - kOneRegister) > V7) {
6844 param->SetRegNO(0);
6845 } else {
6846 param->SetRegNO(fpReg);
6847 param->SetNumRegs(numFpRegs);
6848 fpReg += numFpRegs;
6849 }
6850 } else if (param->GetSize() > k16ByteSize) {
6851 if (intReg > R7) {
6852 param->SetRegNO(0);
6853 } else {
6854 param->SetRegNO(intReg);
6855 param->SetIsOnStack();
6856 param->SetOnStackOffset(((intReg - R0 + fpReg) - V0) * k8ByteSize);
6857 uint32 bytelen = GetPrimTypeSize(PTY_a64);
6858 uint32 bitlen = bytelen * kBitsPerByte;
6859 MemOperand *mOpnd = GenLmbcFpMemOperand(static_cast<int32>(param->GetOnStackOffset()), bytelen);
6860 RegOperand &src = GetOrCreatePhysicalRegisterOperand(AArch64reg(intReg), bitlen, kRegTyInt);
6861 MOperator mOp = PickStInsn(bitlen, PTY_a64);
6862 Insn &store = GetInsnBuilder()->BuildInsn(mOp, src, *mOpnd);
6863 GetCurBB()->AppendInsn(store);
6864 intReg++;
6865 }
6866 } else if (param->GetSize() <= k8ByteSize) {
6867 if (intReg > R7) {
6868 param->SetRegNO(0);
6869 } else {
6870 param->SetRegNO(intReg);
6871 param->SetNumRegs(kOneRegister);
6872 intReg++;
6873 }
6874 } else {
6875 /* size > 8 && size <= 16 */
6876 if ((intReg + kOneRegister) > R7) {
6877 param->SetRegNO(0);
6878 } else {
6879 param->SetRegNO(intReg);
6880 param->SetNumRegs(kTwoRegister);
6881 intReg += kTwoRegister;
6882 }
6883 }
6884 if (param->GetRegNO() != 0) {
6885 for (uint32 i = 0; i < param->GetNumRegs(); ++i) {
6886 PrimType pType = PTY_i64;
6887 RegType rType = kRegTyInt;
6888 uint32 rSize = k8ByteSize;
6889 if (param->IsPureFloat()) {
6890 rType = kRegTyFloat;
6891 if (param->GetFpSize() <= k4ByteSize) {
6892 pType = PTY_f32;
6893 rSize = k4ByteSize;
6894 } else {
6895 pType = PTY_f64;
6896 }
6897 }
6898 regno_t vreg = NewVReg(rType, rSize);
6899 RegOperand &dest = GetOrCreateVirtualRegisterOperand(vreg);
6900 RegOperand &src = GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(param->GetRegNO() + i),
6901 rSize * kBitsPerByte, rType);
6902 SelectCopy(dest, pType, src, pType);
6903 if (param->GetVregNO() == 0) {
6904 param->SetVregNO(vreg);
6905 }
6906 Operand *memOpd = &CreateMemOpnd(RFP, offset + (i * rSize), rSize);
6907 GetCurBB()->AppendInsn(
6908 GetInsnBuilder()->BuildInsn(PickStInsn(rSize * kBitsPerByte, pType), dest, *memOpd));
6909 }
6910 }
6911 } else {
6912 CHECK_FATAL(false, "lmbc formal primtype not handled");
6913 }
6914 }
6915 }
6916
LmbcGenSaveSpForAlloca()6917 void AArch64CGFunc::LmbcGenSaveSpForAlloca()
6918 {
6919 if (GetMirModule().GetFlavor() != MIRFlavor::kFlavorLmbc || !HasVLAOrAlloca()) {
6920 return;
6921 }
6922 Operand &spOpnd = GetOrCreatePhysicalRegisterOperand(RSP, k64BitSize, kRegTyInt);
6923 RegOperand &spSaveOpnd = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, GetPointerSize()));
6924 Insn &save = GetInsnBuilder()->BuildInsn(MOP_xmovrr, spSaveOpnd, spOpnd);
6925 GetFirstBB()->AppendInsn(save);
6926 for (auto *retBB : GetExitBBsVec()) {
6927 Insn &restore = GetInsnBuilder()->BuildInsn(MOP_xmovrr, spOpnd, spSaveOpnd);
6928 retBB->AppendInsn(restore);
6929 restore.SetFrameDef(true);
6930 }
6931 }
6932
6933 /* if offset < 0, allocation; otherwise, deallocation */
CreateCallFrameOperand(int32 offset,uint32 size)6934 MemOperand &AArch64CGFunc::CreateCallFrameOperand(int32 offset, uint32 size)
6935 {
6936 MemOperand *memOpnd = CreateStackMemOpnd(RSP, offset, size);
6937 memOpnd->SetIndexOpt((offset < 0) ? MemOperand::kPreIndex : MemOperand::kPostIndex);
6938 return *memOpnd;
6939 }
6940
GetLogicalShiftLeftOperand(uint32 shiftAmount,bool is64bits) const6941 BitShiftOperand *AArch64CGFunc::GetLogicalShiftLeftOperand(uint32 shiftAmount, bool is64bits) const
6942 {
6943 /* num(0, 16, 32, 48) >> 4 is num1(0, 1, 2, 3), num1 & (~3) == 0 */
6944 DEBUG_ASSERT((!shiftAmount || ((shiftAmount >> 4) & ~static_cast<uint32>(3)) == 0),
6945 "shift amount should be one of 0, 16, 32, 48");
6946 /* movkLslOperands[4]~movkLslOperands[7] is for 64 bits */
6947 return &movkLslOperands[(shiftAmount >> 4) + (is64bits ? 4 : 0)];
6948 }
6949
6950 AArch64CGFunc::MovkLslOperandArray AArch64CGFunc::movkLslOperands = {
6951 BitShiftOperand(BitShiftOperand::kLSL, 0, 4),
6952 BitShiftOperand(BitShiftOperand::kLSL, 16, 4),
6953 BitShiftOperand(BitShiftOperand::kLSL, static_cast<uint32>(-1), 0), /* invalid entry */
6954 BitShiftOperand(BitShiftOperand::kLSL, static_cast<uint32>(-1), 0), /* invalid entry */
6955 BitShiftOperand(BitShiftOperand::kLSL, 0, 6),
6956 BitShiftOperand(BitShiftOperand::kLSL, 16, 6),
6957 BitShiftOperand(BitShiftOperand::kLSL, 32, 6),
6958 BitShiftOperand(BitShiftOperand::kLSL, 48, 6),
6959 };
6960
CreateStkTopOpnd(uint32 offset,uint32 size)6961 MemOperand &AArch64CGFunc::CreateStkTopOpnd(uint32 offset, uint32 size)
6962 {
6963 AArch64reg reg;
6964 if (GetMirModule().GetFlavor() == MIRFlavor::kFlavorLmbc) {
6965 reg = RSP;
6966 } else {
6967 reg = RFP;
6968 }
6969 MemOperand *memOp = CreateStackMemOpnd(reg, static_cast<int32>(offset), size);
6970 return *memOp;
6971 }
6972
CreateStackMemOpnd(regno_t preg,int32 offset,uint32 size)6973 MemOperand *AArch64CGFunc::CreateStackMemOpnd(regno_t preg, int32 offset, uint32 size)
6974 {
6975 auto *memOp =
6976 memPool->New<MemOperand>(memPool->New<RegOperand>(preg, k64BitSize, kRegTyInt),
6977 &CreateOfstOpnd(static_cast<uint64>(static_cast<int64>(offset)), k32BitSize), size);
6978 if (preg == RFP || preg == RSP) {
6979 memOp->SetStackMem(true);
6980 }
6981 return memOp;
6982 }
6983
CreateMemOperand(MemOperand::AArch64AddressingMode mode,uint32 size,RegOperand & base,RegOperand * index,ImmOperand * offset,const MIRSymbol * symbol) const6984 MemOperand *AArch64CGFunc::CreateMemOperand(MemOperand::AArch64AddressingMode mode, uint32 size, RegOperand &base,
6985 RegOperand *index, ImmOperand *offset, const MIRSymbol *symbol) const
6986 {
6987 auto *memOp = memPool->New<MemOperand>(mode, size, base, index, offset, symbol);
6988 if (base.GetRegisterNumber() == RFP || base.GetRegisterNumber() == RSP) {
6989 memOp->SetStackMem(true);
6990 }
6991 return memOp;
6992 }
6993
CreateMemOperand(MemOperand::AArch64AddressingMode mode,uint32 size,RegOperand & base,RegOperand & index,ImmOperand * offset,const MIRSymbol & symbol,bool noExtend)6994 MemOperand *AArch64CGFunc::CreateMemOperand(MemOperand::AArch64AddressingMode mode, uint32 size, RegOperand &base,
6995 RegOperand &index, ImmOperand *offset, const MIRSymbol &symbol,
6996 bool noExtend)
6997 {
6998 auto *memOp = memPool->New<MemOperand>(mode, size, base, index, offset, symbol, noExtend);
6999 if (base.GetRegisterNumber() == RFP || base.GetRegisterNumber() == RSP) {
7000 memOp->SetStackMem(true);
7001 }
7002 return memOp;
7003 }
7004
CreateMemOperand(MemOperand::AArch64AddressingMode mode,uint32 dSize,RegOperand & base,RegOperand & indexOpnd,uint32 shift,bool isSigned) const7005 MemOperand *AArch64CGFunc::CreateMemOperand(MemOperand::AArch64AddressingMode mode, uint32 dSize, RegOperand &base,
7006 RegOperand &indexOpnd, uint32 shift, bool isSigned) const
7007 {
7008 auto *memOp = memPool->New<MemOperand>(mode, dSize, base, indexOpnd, shift, isSigned);
7009 if (base.GetRegisterNumber() == RFP || base.GetRegisterNumber() == RSP) {
7010 memOp->SetStackMem(true);
7011 }
7012 return memOp;
7013 }
7014
CreateMemOperand(MemOperand::AArch64AddressingMode mode,uint32 dSize,const MIRSymbol & sym)7015 MemOperand *AArch64CGFunc::CreateMemOperand(MemOperand::AArch64AddressingMode mode, uint32 dSize, const MIRSymbol &sym)
7016 {
7017 auto *memOp = memPool->New<MemOperand>(mode, dSize, sym);
7018 return memOp;
7019 }
7020
GenSaveMethodInfoCode(BB & bb)7021 void AArch64CGFunc::GenSaveMethodInfoCode(BB &bb)
7022 {
7023 if (GetCG()->UseFastUnwind()) {
7024 BB *formerCurBB = GetCurBB();
7025 GetDummyBB()->ClearInsns();
7026 SetCurBB(*GetDummyBB());
7027 /*
7028 * FUNCATTR_bridge for function: Ljava_2Flang_2FString_3B_7CcompareTo_7C_28Ljava_2Flang_2FObject_3B_29I, to
7029 * exclude this funciton this function is a bridge function generated for Java Genetic
7030 */
7031 if ((GetFunction().GetAttr(FUNCATTR_native) || GetFunction().GetAttr(FUNCATTR_fast_native)) &&
7032 !GetFunction().GetAttr(FUNCATTR_critical_native) && !GetFunction().GetAttr(FUNCATTR_bridge)) {
7033 RegOperand &fpReg = GetOrCreatePhysicalRegisterOperand(RFP, GetPointerSize() * kBitsPerByte, kRegTyInt);
7034
7035 ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
7036 RegOperand &parmRegOpnd1 = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, kRegTyInt);
7037 srcOpnds->PushOpnd(parmRegOpnd1);
7038 Operand &immOpnd = CreateImmOperand(0, k64BitSize, false);
7039 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadri64, parmRegOpnd1, immOpnd));
7040 RegOperand &parmRegOpnd2 = GetOrCreatePhysicalRegisterOperand(R1, k64BitSize, kRegTyInt);
7041 srcOpnds->PushOpnd(parmRegOpnd2);
7042 SelectCopy(parmRegOpnd2, PTY_a64, fpReg, PTY_a64);
7043
7044 MIRSymbol *sym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
7045 std::string funcName("MCC_SetRiskyUnwindContext");
7046 sym->SetNameStrIdx(funcName);
7047
7048 sym->SetStorageClass(kScText);
7049 sym->SetSKind(kStFunc);
7050 AppendCall(*sym, *srcOpnds);
7051 bb.SetHasCall();
7052 }
7053
7054 bb.InsertAtBeginning(*GetDummyBB());
7055 SetCurBB(*formerCurBB);
7056 }
7057 }
7058
HasStackLoadStore()7059 bool AArch64CGFunc::HasStackLoadStore()
7060 {
7061 FOR_ALL_BB(bb, this) {
7062 FOR_BB_INSNS(insn, bb) {
7063 uint32 opndNum = insn->GetOperandSize();
7064 for (uint32 i = 0; i < opndNum; ++i) {
7065 Operand &opnd = insn->GetOperand(i);
7066 if (opnd.IsMemoryAccessOperand()) {
7067 auto &memOpnd = static_cast<MemOperand &>(opnd);
7068 Operand *base = memOpnd.GetBaseRegister();
7069
7070 if ((base != nullptr) && base->IsRegister()) {
7071 RegOperand *regOpnd = static_cast<RegOperand *>(base);
7072 RegType regType = regOpnd->GetRegisterType();
7073 uint32 regNO = regOpnd->GetRegisterNumber();
7074 if (((regType != kRegTyCc) && ((regNO == RFP) || (regNO == RSP))) || (regType == kRegTyVary)) {
7075 return true;
7076 }
7077 }
7078 }
7079 }
7080 }
7081 }
7082 return false;
7083 }
7084
GenerateYieldpoint(BB & bb)7085 void AArch64CGFunc::GenerateYieldpoint(BB &bb)
7086 {
7087 /* ldr wzr, [RYP] # RYP hold address of the polling page. */
7088 auto &wzr = GetZeroOpnd(k32BitSize);
7089 auto &pollingPage = CreateMemOpnd(RYP, 0, k32BitSize);
7090 auto &yieldPoint = GetInsnBuilder()->BuildInsn(MOP_wldr, wzr, pollingPage);
7091 if (GetCG()->GenerateVerboseCG()) {
7092 yieldPoint.SetComment("yieldpoint");
7093 }
7094 bb.AppendInsn(yieldPoint);
7095 }
7096
ProcessReturnReg(PrimType primType,int32 sReg)7097 Operand &AArch64CGFunc::ProcessReturnReg(PrimType primType, int32 sReg)
7098 {
7099 return GetTargetRetOperand(primType, sReg);
7100 }
7101
GetTargetRetOperand(PrimType primType,int32 sReg)7102 Operand &AArch64CGFunc::GetTargetRetOperand(PrimType primType, int32 sReg)
7103 {
7104 uint32 bitSize = GetPrimTypeBitSize(primType) < k32BitSize ? k32BitSize : GetPrimTypeBitSize(primType);
7105 AArch64reg pReg;
7106 if (sReg < 0) {
7107 return GetOrCreatePhysicalRegisterOperand(IsPrimitiveFloat(primType) || (IsPrimitiveVector(primType)) ? S0 : R0,
7108 bitSize, GetRegTyFromPrimTy(primType));
7109 } else {
7110 switch (sReg) {
7111 case kSregRetval0:
7112 pReg = IsPrimitiveFloat(primType) || (IsPrimitiveVector(primType)) ? S0 : R0;
7113 break;
7114 case kSregRetval1:
7115 pReg = R1;
7116 break;
7117 default:
7118 pReg = RLAST_INT_REG;
7119 DEBUG_ASSERT(0, "GetTargetRetOperand: NYI");
7120 }
7121 return GetOrCreatePhysicalRegisterOperand(pReg, bitSize, GetRegTyFromPrimTy(primType));
7122 }
7123 }
7124
CreateRegisterOperandOfType(PrimType primType)7125 RegOperand &AArch64CGFunc::CreateRegisterOperandOfType(PrimType primType)
7126 {
7127 RegType regType = GetRegTyFromPrimTy(primType);
7128 uint32 byteLength = GetPrimTypeSize(primType);
7129 return CreateRegisterOperandOfType(regType, byteLength);
7130 }
7131
CreateRegisterOperandOfType(RegType regty,uint32 byteLen)7132 RegOperand &AArch64CGFunc::CreateRegisterOperandOfType(RegType regty, uint32 byteLen)
7133 {
7134 /* BUG: if half-precision floating point operations are supported? */
7135 /* AArch64 has 32-bit and 64-bit registers only */
7136 if (byteLen < k4ByteSize) {
7137 byteLen = k4ByteSize;
7138 }
7139 regno_t vRegNO = NewVReg(regty, byteLen);
7140 return CreateVirtualRegisterOperand(vRegNO);
7141 }
7142
CreateRflagOperand()7143 RegOperand &AArch64CGFunc::CreateRflagOperand()
7144 {
7145 /* AArch64 has Status register that is 32-bit wide. */
7146 regno_t vRegNO = NewVRflag();
7147 return CreateVirtualRegisterOperand(vRegNO);
7148 }
7149
MergeReturn()7150 void AArch64CGFunc::MergeReturn()
7151 {
7152 DEBUG_ASSERT(GetCurBB()->GetPrev()->GetFirstStmt() == GetCleanupLabel(), "must be");
7153
7154 uint32 exitBBSize = GetExitBBsVec().size();
7155 if (exitBBSize == 0) {
7156 return;
7157 }
7158 if ((exitBBSize == 1) && GetExitBB(0) == GetCurBB()) {
7159 return;
7160 }
7161 if (exitBBSize == 1) {
7162 BB *onlyExitBB = GetExitBB(0);
7163 BB *onlyExitBBNext = onlyExitBB->GetNext();
7164 StmtNode *stmt = onlyExitBBNext->GetFirstStmt();
7165 /* only deal with the return_BB in the middle */
7166 if (stmt != GetCleanupLabel()) {
7167 LabelIdx labidx = CreateLabel();
7168 BB *retBB = CreateNewBB(labidx, onlyExitBB->IsUnreachable(), BB::kBBReturn, onlyExitBB->GetFrequency());
7169 onlyExitBB->AppendBB(*retBB);
7170 /* modify the original return BB. */
7171 DEBUG_ASSERT(onlyExitBB->GetKind() == BB::kBBReturn, "Error: suppose to merge multi return bb");
7172 onlyExitBB->SetKind(BB::kBBFallthru);
7173
7174 GetExitBBsVec().pop_back();
7175 GetExitBBsVec().emplace_back(retBB);
7176 return;
7177 }
7178 }
7179
7180 LabelIdx labidx = CreateLabel();
7181 LabelOperand &targetOpnd = GetOrCreateLabelOperand(labidx);
7182 uint32 freq = 0;
7183 for (auto *tmpBB : GetExitBBsVec()) {
7184 DEBUG_ASSERT(tmpBB->GetKind() == BB::kBBReturn, "Error: suppose to merge multi return bb");
7185 tmpBB->SetKind(BB::kBBGoto);
7186 tmpBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xuncond, targetOpnd));
7187 freq += tmpBB->GetFrequency();
7188 }
7189 BB *retBB = CreateNewBB(labidx, false, BB::kBBReturn, freq);
7190 GetCleanupBB()->PrependBB(*retBB);
7191
7192 GetExitBBsVec().clear();
7193 GetExitBBsVec().emplace_back(retBB);
7194 }
7195
HandleRetCleanup(NaryStmtNode & retNode)7196 void AArch64CGFunc::HandleRetCleanup(NaryStmtNode &retNode)
7197 {
7198 if (!GetCG()->GenLocalRC()) {
7199 /* handle local rc is disabled. */
7200 return;
7201 }
7202
7203 constexpr uint8 opSize = 11;
7204 Opcode ops[opSize] = {OP_label, OP_goto, OP_brfalse, OP_brtrue, OP_return, OP_call,
7205 OP_icall, OP_rangegoto, OP_catch, OP_try, OP_endtry};
7206 std::set<Opcode> branchOp(ops, ops + opSize);
7207
7208 /* get cleanup intrinsic */
7209 bool found = false;
7210 StmtNode *cleanupNode = retNode.GetPrev();
7211 cleanEANode = nullptr;
7212 while (cleanupNode != nullptr) {
7213 if (branchOp.find(cleanupNode->GetOpCode()) != branchOp.end()) {
7214 if (cleanupNode->GetOpCode() == OP_call) {
7215 CallNode *callNode = static_cast<CallNode *>(cleanupNode);
7216 MIRFunction *fn = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(callNode->GetPUIdx());
7217 MIRSymbol *fsym = GetFunction().GetLocalOrGlobalSymbol(fn->GetStIdx(), false);
7218 if ((fsym->GetName() == "MCC_DecRef_NaiveRCFast") || (fsym->GetName() == "MCC_IncRef_NaiveRCFast") ||
7219 (fsym->GetName() == "MCC_IncDecRef_NaiveRCFast") || (fsym->GetName() == "MCC_LoadRefStatic") ||
7220 (fsym->GetName() == "MCC_LoadRefField") || (fsym->GetName() == "MCC_LoadReferentField") ||
7221 (fsym->GetName() == "MCC_LoadRefField_NaiveRCFast") ||
7222 (fsym->GetName() == "MCC_LoadVolatileField") ||
7223 (fsym->GetName() == "MCC_LoadVolatileStaticField") || (fsym->GetName() == "MCC_LoadWeakField") ||
7224 (fsym->GetName() == "MCC_CheckObjMem")) {
7225 cleanupNode = cleanupNode->GetPrev();
7226 continue;
7227 } else {
7228 break;
7229 }
7230 } else {
7231 break;
7232 }
7233 }
7234
7235 if (cleanupNode->GetOpCode() == OP_intrinsiccall) {
7236 IntrinsiccallNode *tempNode = static_cast<IntrinsiccallNode *>(cleanupNode);
7237 if ((tempNode->GetIntrinsic() == INTRN_MPL_CLEANUP_LOCALREFVARS) ||
7238 (tempNode->GetIntrinsic() == INTRN_MPL_CLEANUP_LOCALREFVARS_SKIP)) {
7239 GenRetCleanup(tempNode);
7240 if (cleanEANode != nullptr) {
7241 GenRetCleanup(cleanEANode, true);
7242 }
7243 found = true;
7244 break;
7245 }
7246 if (tempNode->GetIntrinsic() == INTRN_MPL_CLEANUP_NORETESCOBJS) {
7247 cleanEANode = tempNode;
7248 }
7249 }
7250 cleanupNode = cleanupNode->GetPrev();
7251 }
7252
7253 if (!found) {
7254 MIRSymbol *retRef = nullptr;
7255 if (retNode.NumOpnds() != 0) {
7256 retRef = GetRetRefSymbol(*static_cast<NaryStmtNode &>(retNode).Opnd(0));
7257 }
7258 HandleRCCall(false, retRef);
7259 }
7260 }
7261
GenRetCleanup(const IntrinsiccallNode * cleanupNode,bool forEA)7262 bool AArch64CGFunc::GenRetCleanup(const IntrinsiccallNode *cleanupNode, bool forEA)
7263 {
7264 #undef CC_DEBUG_INFO
7265
7266 #ifdef CC_DEBUG_INFO
7267 LogInfo::MapleLogger() << "==============" << GetFunction().GetName() << "==============" << '\n';
7268 #endif
7269
7270 if (cleanupNode == nullptr) {
7271 return false;
7272 }
7273
7274 int32 minByteOffset = INT_MAX;
7275 int32 maxByteOffset = 0;
7276
7277 int32 skipIndex = -1;
7278 MIRSymbol *skipSym = nullptr;
7279 size_t refSymNum = 0;
7280 if (cleanupNode->GetIntrinsic() == INTRN_MPL_CLEANUP_LOCALREFVARS) {
7281 refSymNum = cleanupNode->GetNopndSize();
7282 if (refSymNum < 1) {
7283 return true;
7284 }
7285 } else if (cleanupNode->GetIntrinsic() == INTRN_MPL_CLEANUP_LOCALREFVARS_SKIP) {
7286 refSymNum = cleanupNode->GetNopndSize();
7287 /* refSymNum == 0, no local refvars; refSymNum == 1 and cleanup skip, so nothing to do */
7288 if (refSymNum <= 1) {
7289 return true;
7290 }
7291 BaseNode *skipExpr = cleanupNode->Opnd(refSymNum - 1);
7292
7293 CHECK_FATAL(skipExpr->GetOpCode() == OP_dread, "should be dread");
7294 DreadNode *refNode = static_cast<DreadNode *>(skipExpr);
7295 skipSym = GetFunction().GetLocalOrGlobalSymbol(refNode->GetStIdx());
7296
7297 refSymNum -= 1;
7298 } else if (cleanupNode->GetIntrinsic() == INTRN_MPL_CLEANUP_NORETESCOBJS) {
7299 refSymNum = cleanupNode->GetNopndSize();
7300 /* the number of operands of intrinsic call INTRN_MPL_CLEANUP_NORETESCOBJS must be more than 1 */
7301 if (refSymNum <= 1) {
7302 return true;
7303 }
7304 BaseNode *skipexpr = cleanupNode->Opnd(0);
7305 CHECK_FATAL(skipexpr->GetOpCode() == OP_dread, "should be dread");
7306 DreadNode *refnode = static_cast<DreadNode *>(skipexpr);
7307 skipSym = GetFunction().GetLocalOrGlobalSymbol(refnode->GetStIdx());
7308 }
7309
7310 /* now compute the offset range */
7311 std::vector<int32> offsets;
7312 AArch64MemLayout *memLayout = static_cast<AArch64MemLayout *>(this->GetMemlayout());
7313 for (size_t i = 0; i < refSymNum; ++i) {
7314 BaseNode *argExpr = cleanupNode->Opnd(i);
7315 CHECK_FATAL(argExpr->GetOpCode() == OP_dread, "should be dread");
7316 DreadNode *refNode = static_cast<DreadNode *>(argExpr);
7317 MIRSymbol *refSymbol = GetFunction().GetLocalOrGlobalSymbol(refNode->GetStIdx());
7318 if (memLayout->GetSymAllocTable().size() <= refSymbol->GetStIndex()) {
7319 ERR(kLncErr, "access memLayout->GetSymAllocTable() failed");
7320 return false;
7321 }
7322 AArch64SymbolAlloc *symLoc =
7323 static_cast<AArch64SymbolAlloc *>(memLayout->GetSymAllocInfo(refSymbol->GetStIndex()));
7324 int32 tempOffset = GetBaseOffset(*symLoc);
7325 offsets.emplace_back(tempOffset);
7326 #ifdef CC_DEBUG_INFO
7327 LogInfo::MapleLogger() << "refsym " << refSymbol->GetName() << " offset " << tempOffset << '\n';
7328 #endif
7329 minByteOffset = (minByteOffset > tempOffset) ? tempOffset : minByteOffset;
7330 maxByteOffset = (maxByteOffset < tempOffset) ? tempOffset : maxByteOffset;
7331 }
7332
7333 /* get the skip offset */
7334 int32 skipOffset = -1;
7335 if (skipSym != nullptr) {
7336 AArch64SymbolAlloc *symLoc =
7337 static_cast<AArch64SymbolAlloc *>(memLayout->GetSymAllocInfo(skipSym->GetStIndex()));
7338 CHECK_FATAL(GetBaseOffset(*symLoc) < std::numeric_limits<int32>::max(), "out of range");
7339 skipOffset = GetBaseOffset(*symLoc);
7340 offsets.emplace_back(skipOffset);
7341
7342 #ifdef CC_DEBUG_INFO
7343 LogInfo::MapleLogger() << "skip " << skipSym->GetName() << " offset " << skipOffset << '\n';
7344 #endif
7345
7346 skipIndex = symLoc->GetOffset() / kOffsetAlign;
7347 }
7348
7349 /* call runtime cleanup */
7350 if (minByteOffset < INT_MAX) {
7351 int32 refLocBase = memLayout->GetRefLocBaseLoc();
7352 uint32 refNum = memLayout->GetSizeOfRefLocals() / kOffsetAlign;
7353 CHECK_FATAL((refLocBase + (refNum - 1) * kIntregBytelen) < std::numeric_limits<int32>::max(), "out of range");
7354 int32 refLocEnd = refLocBase + (refNum - 1) * kIntregBytelen;
7355 int32 realMin = minByteOffset < refLocBase ? refLocBase : minByteOffset;
7356 int32 realMax = maxByteOffset > refLocEnd ? refLocEnd : maxByteOffset;
7357 if (forEA) {
7358 std::sort(offsets.begin(), offsets.end());
7359 int32 prev = offsets[0];
7360 for (size_t i = 1; i < offsets.size(); i++) {
7361 CHECK_FATAL((offsets[i] == prev) || ((offsets[i] - prev) == kIntregBytelen), "must be");
7362 prev = offsets[i];
7363 }
7364 CHECK_FATAL((refLocBase - prev) == kIntregBytelen, "must be");
7365 realMin = minByteOffset;
7366 realMax = maxByteOffset;
7367 }
7368 #ifdef CC_DEBUG_INFO
7369 LogInfo::MapleLogger() << " realMin " << realMin << " realMax " << realMax << '\n';
7370 #endif
7371 if (realMax < realMin) {
7372 /* maybe there is a cleanup intrinsic bug, use CHECK_FATAL instead? */
7373 CHECK_FATAL(false, "must be");
7374 }
7375
7376 /* optimization for little slot cleanup */
7377 if (realMax == realMin && !forEA) {
7378 RegOperand &phyOpnd = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7379 Operand &stackLoc = CreateStkTopOpnd(static_cast<uint32>(realMin), GetPointerSize() * kBitsPerByte);
7380 Insn &ldrInsn = GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_a64), phyOpnd, stackLoc);
7381 GetCurBB()->AppendInsn(ldrInsn);
7382
7383 ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
7384 srcOpnds->PushOpnd(phyOpnd);
7385 MIRSymbol *callSym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
7386 std::string funcName("MCC_DecRef_NaiveRCFast");
7387 callSym->SetNameStrIdx(funcName);
7388 callSym->SetStorageClass(kScText);
7389 callSym->SetSKind(kStFunc);
7390 Insn &callInsn = AppendCall(*callSym, *srcOpnds);
7391 callInsn.SetRefSkipIdx(skipIndex);
7392 GetCurBB()->SetHasCall();
7393 /* because of return stmt is often the last stmt */
7394 GetCurBB()->SetFrequency(frequency);
7395
7396 return true;
7397 }
7398 ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
7399
7400 ImmOperand &beginOpnd = CreateImmOperand(realMin, k64BitSize, true);
7401 regno_t vRegNO0 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
7402 RegOperand &vReg0 = CreateVirtualRegisterOperand(vRegNO0);
7403 RegOperand &fpOpnd = GetOrCreateStackBaseRegOperand();
7404 SelectAdd(vReg0, fpOpnd, beginOpnd, PTY_i64);
7405
7406 RegOperand &parmRegOpnd1 = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7407 srcOpnds->PushOpnd(parmRegOpnd1);
7408 SelectCopy(parmRegOpnd1, PTY_a64, vReg0, PTY_a64);
7409
7410 uint32 realRefNum = (realMax - realMin) / kOffsetAlign + 1;
7411
7412 ImmOperand &countOpnd = CreateImmOperand(realRefNum, k64BitSize, true);
7413
7414 RegOperand &parmRegOpnd2 = GetOrCreatePhysicalRegisterOperand(R1, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7415 srcOpnds->PushOpnd(parmRegOpnd2);
7416 SelectCopyImm(parmRegOpnd2, countOpnd, PTY_i64);
7417
7418 MIRSymbol *funcSym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
7419 if ((skipSym != nullptr) && (skipOffset >= realMin) && (skipOffset <= realMax)) {
7420 /* call cleanupskip */
7421 uint32 stOffset = (skipOffset - realMin) / kOffsetAlign;
7422 ImmOperand &retLoc = CreateImmOperand(stOffset, k64BitSize, true);
7423
7424 RegOperand &parmRegOpnd3 = GetOrCreatePhysicalRegisterOperand(R2, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7425 srcOpnds->PushOpnd(parmRegOpnd3);
7426 SelectCopyImm(parmRegOpnd3, retLoc, PTY_i64);
7427
7428 std::string funcName;
7429 if (forEA) {
7430 funcName = "MCC_CleanupNonRetEscObj";
7431 } else {
7432 funcName = "MCC_CleanupLocalStackRefSkip_NaiveRCFast";
7433 }
7434 funcSym->SetNameStrIdx(funcName);
7435 #ifdef CC_DEBUG_INFO
7436 LogInfo::MapleLogger() << "num " << real_ref_num << " skip loc " << stOffset << '\n';
7437 #endif
7438 } else {
7439 /* call cleanup */
7440 CHECK_FATAL(!forEA, "must be");
7441 std::string funcName("MCC_CleanupLocalStackRef_NaiveRCFast");
7442 funcSym->SetNameStrIdx(funcName);
7443 #ifdef CC_DEBUG_INFO
7444 LogInfo::MapleLogger() << "num " << real_ref_num << '\n';
7445 #endif
7446 }
7447
7448 funcSym->SetStorageClass(kScText);
7449 funcSym->SetSKind(kStFunc);
7450 Insn &callInsn = AppendCall(*funcSym, *srcOpnds);
7451 callInsn.SetRefSkipIdx(skipIndex);
7452 GetCurBB()->SetHasCall();
7453 GetCurBB()->SetFrequency(frequency);
7454 }
7455 return true;
7456 }
7457
CreateVirtualRegisterOperand(regno_t vRegNO,uint32 size,RegType kind,uint32 flg) const7458 RegOperand *AArch64CGFunc::CreateVirtualRegisterOperand(regno_t vRegNO, uint32 size, RegType kind, uint32 flg) const
7459 {
7460 RegOperand *res = memPool->New<RegOperand>(vRegNO, size, kind, flg);
7461 return res;
7462 }
7463
CreateVirtualRegisterOperand(regno_t vRegNO)7464 RegOperand &AArch64CGFunc::CreateVirtualRegisterOperand(regno_t vRegNO)
7465 {
7466 DEBUG_ASSERT((vRegOperandTable.find(vRegNO) == vRegOperandTable.end()), "already exist");
7467 DEBUG_ASSERT(vRegNO < vRegTable.size(), "index out of range");
7468 uint8 bitSize = static_cast<uint8>((static_cast<uint32>(vRegTable[vRegNO].GetSize())) * kBitsPerByte);
7469 RegOperand *res = CreateVirtualRegisterOperand(vRegNO, bitSize, vRegTable.at(vRegNO).GetType());
7470 vRegOperandTable[vRegNO] = res;
7471 return *res;
7472 }
7473
GetOrCreateVirtualRegisterOperand(regno_t vRegNO)7474 RegOperand &AArch64CGFunc::GetOrCreateVirtualRegisterOperand(regno_t vRegNO)
7475 {
7476 auto it = vRegOperandTable.find(vRegNO);
7477 return (it != vRegOperandTable.end()) ? *(it->second) : CreateVirtualRegisterOperand(vRegNO);
7478 }
7479
GetOrCreateVirtualRegisterOperand(RegOperand & regOpnd)7480 RegOperand &AArch64CGFunc::GetOrCreateVirtualRegisterOperand(RegOperand ®Opnd)
7481 {
7482 regno_t regNO = regOpnd.GetRegisterNumber();
7483 auto it = vRegOperandTable.find(regNO);
7484 if (it != vRegOperandTable.end()) {
7485 it->second->SetSize(regOpnd.GetSize());
7486 it->second->SetRegisterNumber(regNO);
7487 it->second->SetRegisterType(regOpnd.GetRegisterType());
7488 it->second->SetValidBitsNum(regOpnd.GetValidBitsNum());
7489 return *it->second;
7490 } else {
7491 auto *newRegOpnd = static_cast<RegOperand *>(regOpnd.Clone(*memPool));
7492 regno_t newRegNO = newRegOpnd->GetRegisterNumber();
7493 if (newRegNO >= maxRegCount) {
7494 maxRegCount = newRegNO + kRegIncrStepLen;
7495 vRegTable.resize(maxRegCount);
7496 }
7497 vRegOperandTable[newRegNO] = newRegOpnd;
7498 VirtualRegNode *vregNode = memPool->New<VirtualRegNode>(newRegOpnd->GetRegisterType(), newRegOpnd->GetSize());
7499 vRegTable[newRegNO] = *vregNode;
7500 vRegCount = maxRegCount;
7501 return *newRegOpnd;
7502 }
7503 }
7504
7505 /*
7506 * Traverse all call insn to determine return type of it
7507 * If the following insn is mov/str/blr and use R0/V0, it means the call insn have reture value
7508 */
DetermineReturnTypeofCall()7509 void AArch64CGFunc::DetermineReturnTypeofCall()
7510 {
7511 FOR_ALL_BB(bb, this) {
7512 if (bb->IsUnreachable() || !bb->HasCall()) {
7513 continue;
7514 }
7515 FOR_BB_INSNS(insn, bb) {
7516 if (!insn->IsTargetInsn()) {
7517 continue;
7518 }
7519 if (!insn->IsCall() || insn->GetMachineOpcode() == MOP_asm) {
7520 continue;
7521 }
7522 Insn *nextInsn = insn->GetNextMachineInsn();
7523 if (nextInsn == nullptr) {
7524 continue;
7525 }
7526 if ((nextInsn->GetMachineOpcode() != MOP_asm) &&
7527 ((nextInsn->IsMove() && nextInsn->GetOperand(kInsnSecondOpnd).IsRegister()) || nextInsn->IsStore() ||
7528 (nextInsn->IsCall() && nextInsn->GetOperand(kInsnFirstOpnd).IsRegister()))) {
7529 auto *srcOpnd = static_cast<RegOperand *>(&nextInsn->GetOperand(kInsnFirstOpnd));
7530 CHECK_FATAL(srcOpnd != nullptr, "nullptr");
7531 if (!srcOpnd->IsPhysicalRegister()) {
7532 continue;
7533 }
7534 if (srcOpnd->GetRegisterNumber() == R0) {
7535 insn->SetRetType(Insn::kRegInt);
7536 continue;
7537 }
7538 if (srcOpnd->GetRegisterNumber() == V0) {
7539 insn->SetRetType(Insn::kRegFloat);
7540 }
7541 }
7542 }
7543 }
7544 }
7545
HandleRCCall(bool begin,const MIRSymbol * retRef)7546 void AArch64CGFunc::HandleRCCall(bool begin, const MIRSymbol *retRef)
7547 {
7548 if (!GetCG()->GenLocalRC() && !begin) {
7549 /* handle local rc is disabled. */
7550 return;
7551 }
7552
7553 AArch64MemLayout *memLayout = static_cast<AArch64MemLayout *>(this->GetMemlayout());
7554 int32 refNum = static_cast<int32>(memLayout->GetSizeOfRefLocals() / kOffsetAlign);
7555 if (!refNum) {
7556 if (begin) {
7557 GenerateYieldpoint(*GetCurBB());
7558 yieldPointInsn = GetCurBB()->GetLastInsn();
7559 }
7560 return;
7561 }
7562
7563 /* no MCC_CleanupLocalStackRefSkip when ret_ref is the only ref symbol */
7564 if ((refNum == 1) && (retRef != nullptr)) {
7565 if (begin) {
7566 GenerateYieldpoint(*GetCurBB());
7567 yieldPointInsn = GetCurBB()->GetLastInsn();
7568 }
7569 return;
7570 }
7571 CHECK_FATAL(refNum < 0xFFFF, "not enough room for size.");
7572 int32 refLocBase = memLayout->GetRefLocBaseLoc();
7573 CHECK_FATAL((refLocBase >= 0) && (refLocBase < 0xFFFF), "not enough room for offset.");
7574 int32 formalRef = 0;
7575 /* avoid store zero to formal localrefvars. */
7576 if (begin) {
7577 for (uint32 i = 0; i < GetFunction().GetFormalCount(); ++i) {
7578 if (GetFunction().GetNthParamAttr(i).GetAttr(ATTR_localrefvar)) {
7579 refNum--;
7580 formalRef++;
7581 }
7582 }
7583 }
7584 /*
7585 * if the number of local refvar is less than 12, use stp or str to init local refvar
7586 * else call function MCC_InitializeLocalStackRef to init.
7587 */
7588 if (begin && (refNum <= kRefNum12) && ((refLocBase + kIntregBytelen * (refNum - 1)) < kStpLdpImm64UpperBound)) {
7589 int32 pairNum = refNum / kDivide2;
7590 int32 singleNum = refNum % kDivide2;
7591 const int32 pairRefBytes = 16; /* the size of each pair of ref is 16 bytes */
7592 int32 ind = 0;
7593 while (ind < pairNum) {
7594 int32 offset = memLayout->GetRefLocBaseLoc() + kIntregBytelen * formalRef + pairRefBytes * ind;
7595 Operand &zeroOp = GetZeroOpnd(k64BitSize);
7596 Operand &stackLoc = CreateStkTopOpnd(static_cast<uint32>(offset), GetPointerSize() * kBitsPerByte);
7597 Insn &setInc = GetInsnBuilder()->BuildInsn(MOP_xstp, zeroOp, zeroOp, stackLoc);
7598 GetCurBB()->AppendInsn(setInc);
7599 ind++;
7600 }
7601 if (singleNum > 0) {
7602 int32 offset = memLayout->GetRefLocBaseLoc() + kIntregBytelen * formalRef + kIntregBytelen * (refNum - 1);
7603 Operand &zeroOp = GetZeroOpnd(k64BitSize);
7604 Operand &stackLoc = CreateStkTopOpnd(static_cast<uint32>(offset), GetPointerSize() * kBitsPerByte);
7605 Insn &setInc = GetInsnBuilder()->BuildInsn(MOP_xstr, zeroOp, stackLoc);
7606 GetCurBB()->AppendInsn(setInc);
7607 }
7608 /* Insert Yield Point just after localrefvar are initialized. */
7609 GenerateYieldpoint(*GetCurBB());
7610 yieldPointInsn = GetCurBB()->GetLastInsn();
7611 return;
7612 }
7613
7614 /* refNum is 1 and refvar is not returned, this refvar need to call MCC_DecRef_NaiveRCFast. */
7615 if ((refNum == 1) && !begin && (retRef == nullptr)) {
7616 RegOperand &phyOpnd = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7617 Operand &stackLoc =
7618 CreateStkTopOpnd(static_cast<uint32>(memLayout->GetRefLocBaseLoc()), GetPointerSize() * kBitsPerByte);
7619 Insn &ldrInsn = GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_a64), phyOpnd, stackLoc);
7620 GetCurBB()->AppendInsn(ldrInsn);
7621
7622 ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
7623 srcOpnds->PushOpnd(phyOpnd);
7624 MIRSymbol *callSym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
7625 std::string funcName("MCC_DecRef_NaiveRCFast");
7626 callSym->SetNameStrIdx(funcName);
7627 callSym->SetStorageClass(kScText);
7628 callSym->SetSKind(kStFunc);
7629
7630 AppendCall(*callSym, *srcOpnds);
7631 GetCurBB()->SetHasCall();
7632 if (frequency != 0) {
7633 GetCurBB()->SetFrequency(frequency);
7634 }
7635 return;
7636 }
7637
7638 /* refNum is 2 and one of refvar is returned, only another one is needed to call MCC_DecRef_NaiveRCFast. */
7639 if ((refNum == 2) && !begin && retRef != nullptr) {
7640 AArch64SymbolAlloc *symLoc =
7641 static_cast<AArch64SymbolAlloc *>(memLayout->GetSymAllocInfo(retRef->GetStIndex()));
7642 int32 stOffset = symLoc->GetOffset() / kOffsetAlign;
7643 RegOperand &phyOpnd = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7644 Operand *stackLoc = nullptr;
7645 if (stOffset == 0) {
7646 /* just have to Dec the next one. */
7647 stackLoc = &CreateStkTopOpnd(static_cast<uint32>(memLayout->GetRefLocBaseLoc()) + kIntregBytelen,
7648 GetPointerSize() * kBitsPerByte);
7649 } else {
7650 /* just have to Dec the current one. */
7651 stackLoc =
7652 &CreateStkTopOpnd(static_cast<uint32>(memLayout->GetRefLocBaseLoc()), GetPointerSize() * kBitsPerByte);
7653 }
7654 Insn &ldrInsn = GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_a64), phyOpnd, *stackLoc);
7655 GetCurBB()->AppendInsn(ldrInsn);
7656
7657 ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
7658 srcOpnds->PushOpnd(phyOpnd);
7659 MIRSymbol *callSym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
7660 std::string funcName("MCC_DecRef_NaiveRCFast");
7661 callSym->SetNameStrIdx(funcName);
7662 callSym->SetStorageClass(kScText);
7663 callSym->SetSKind(kStFunc);
7664 Insn &callInsn = AppendCall(*callSym, *srcOpnds);
7665 callInsn.SetRefSkipIdx(stOffset);
7666 GetCurBB()->SetHasCall();
7667 if (frequency != 0) {
7668 GetCurBB()->SetFrequency(frequency);
7669 }
7670 return;
7671 }
7672
7673 bool needSkip = false;
7674 ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
7675
7676 ImmOperand *beginOpnd =
7677 &CreateImmOperand(memLayout->GetRefLocBaseLoc() + kIntregBytelen * formalRef, k64BitSize, true);
7678 ImmOperand *countOpnd = &CreateImmOperand(refNum, k64BitSize, true);
7679 int32 refSkipIndex = -1;
7680 if (!begin && retRef != nullptr) {
7681 AArch64SymbolAlloc *symLoc =
7682 static_cast<AArch64SymbolAlloc *>(memLayout->GetSymAllocInfo(retRef->GetStIndex()));
7683 int32 stOffset = symLoc->GetOffset() / kOffsetAlign;
7684 refSkipIndex = stOffset;
7685 if (stOffset == 0) {
7686 /* ret_ref at begin. */
7687 beginOpnd = &CreateImmOperand(memLayout->GetRefLocBaseLoc() + kIntregBytelen, k64BitSize, true);
7688 countOpnd = &CreateImmOperand(refNum - 1, k64BitSize, true);
7689 } else if (stOffset == (refNum - 1)) {
7690 /* ret_ref at end. */
7691 countOpnd = &CreateImmOperand(refNum - 1, k64BitSize, true);
7692 } else {
7693 needSkip = true;
7694 }
7695 }
7696
7697 regno_t vRegNO0 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
7698 RegOperand &vReg0 = CreateVirtualRegisterOperand(vRegNO0);
7699 RegOperand &fpOpnd = GetOrCreateStackBaseRegOperand();
7700 SelectAdd(vReg0, fpOpnd, *beginOpnd, PTY_i64);
7701
7702 RegOperand &parmRegOpnd1 = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7703 srcOpnds->PushOpnd(parmRegOpnd1);
7704 SelectCopy(parmRegOpnd1, PTY_a64, vReg0, PTY_a64);
7705
7706 regno_t vRegNO1 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
7707 RegOperand &vReg1 = CreateVirtualRegisterOperand(vRegNO1);
7708 SelectCopyImm(vReg1, *countOpnd, PTY_i64);
7709
7710 RegOperand &parmRegOpnd2 = GetOrCreatePhysicalRegisterOperand(R1, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7711 srcOpnds->PushOpnd(parmRegOpnd2);
7712 SelectCopy(parmRegOpnd2, PTY_a64, vReg1, PTY_a64);
7713
7714 MIRSymbol *sym = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
7715 if (begin) {
7716 std::string funcName("MCC_InitializeLocalStackRef");
7717 sym->SetNameStrIdx(funcName);
7718 CHECK_FATAL(countOpnd->GetValue() > 0, "refCount should be greater than 0.");
7719 refCount = static_cast<uint32>(countOpnd->GetValue());
7720 beginOffset = beginOpnd->GetValue();
7721 } else if (!needSkip) {
7722 std::string funcName("MCC_CleanupLocalStackRef_NaiveRCFast");
7723 sym->SetNameStrIdx(funcName);
7724 } else {
7725 CHECK_NULL_FATAL(retRef);
7726 if (retRef->GetStIndex() >= memLayout->GetSymAllocTable().size()) {
7727 CHECK_FATAL(false, "index out of range in AArch64CGFunc::HandleRCCall");
7728 }
7729 AArch64SymbolAlloc *symLoc =
7730 static_cast<AArch64SymbolAlloc *>(memLayout->GetSymAllocInfo(retRef->GetStIndex()));
7731 int32 stOffset = symLoc->GetOffset() / kOffsetAlign;
7732 ImmOperand &retLoc = CreateImmOperand(stOffset, k64BitSize, true);
7733
7734 regno_t vRegNO2 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
7735 RegOperand &vReg2 = CreateVirtualRegisterOperand(vRegNO2);
7736 SelectCopyImm(vReg2, retLoc, PTY_i64);
7737
7738 RegOperand &parmRegOpnd3 = GetOrCreatePhysicalRegisterOperand(R2, k64BitSize, GetRegTyFromPrimTy(PTY_a64));
7739 srcOpnds->PushOpnd(parmRegOpnd3);
7740 SelectCopy(parmRegOpnd3, PTY_a64, vReg2, PTY_a64);
7741
7742 std::string funcName("MCC_CleanupLocalStackRefSkip_NaiveRCFast");
7743 sym->SetNameStrIdx(funcName);
7744 }
7745 sym->SetStorageClass(kScText);
7746 sym->SetSKind(kStFunc);
7747
7748 Insn &callInsn = AppendCall(*sym, *srcOpnds);
7749 callInsn.SetRefSkipIdx(refSkipIndex);
7750 if (frequency != 0) {
7751 GetCurBB()->SetFrequency(frequency);
7752 }
7753 GetCurBB()->SetHasCall();
7754 if (begin) {
7755 /* Insert Yield Point just after localrefvar are initialized. */
7756 GenerateYieldpoint(*GetCurBB());
7757 yieldPointInsn = GetCurBB()->GetLastInsn();
7758 }
7759 }
7760
SelectParmListDreadSmallAggregate(const MIRSymbol & sym,MIRType & structType,ListOperand & srcOpnds,int32 offset,AArch64CallConvImpl & parmLocator,FieldID fieldID)7761 void AArch64CGFunc::SelectParmListDreadSmallAggregate(const MIRSymbol &sym, MIRType &structType, ListOperand &srcOpnds,
7762 int32 offset, AArch64CallConvImpl &parmLocator, FieldID fieldID)
7763 {
7764 /*
7765 * in two param regs if possible
7766 * If struct is <= 8 bytes, then it fits into one param reg.
7767 * If struct is <= 16 bytes, then it fits into two param regs.
7768 * Otherwise, it goes onto the stack.
7769 * If the number of available param reg is less than what is
7770 * needed to fit the entire struct into them, then the param
7771 * reg is skipped and the struct goes onto the stack.
7772 * Example 1.
7773 * struct size == 8 bytes.
7774 * param regs x0 to x6 are used.
7775 * struct is passed in x7.
7776 * Example 2.
7777 * struct is 16 bytes.
7778 * param regs x0 to x5 are used.
7779 * struct is passed in x6 and x7.
7780 * Example 3.
7781 * struct is 16 bytes.
7782 * param regs x0 to x6 are used. x7 alone is not enough to pass the struct.
7783 * struct is passed on the stack.
7784 * x7 is not used, as the following param will go onto the stack also.
7785 */
7786 int32 symSize = GetBecommon().GetTypeSize(structType.GetTypeIndex().GetIdx());
7787 CCLocInfo ploc;
7788 CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
7789 parmLocator.LocateNextParm(structType, ploc);
7790 if (ploc.reg0 == 0) {
7791 /* No param regs available, pass on stack. */
7792 /* If symSize is <= 8 bytes then use 1 reg, else 2 */
7793 CreateCallStructParamPassByStack(symSize, &sym, nullptr, ploc.memOffset);
7794 } else {
7795 /* pass by param regs. */
7796 RegOperand *parmOpnd0 = SelectParmListDreadAccessField(sym, fieldID, ploc, offset, 0);
7797 srcOpnds.PushOpnd(*parmOpnd0);
7798 if (ploc.reg1) {
7799 RegOperand *parmOpnd1 = SelectParmListDreadAccessField(sym, fieldID, ploc, offset, 1);
7800 srcOpnds.PushOpnd(*parmOpnd1);
7801 }
7802 if (ploc.reg2) {
7803 RegOperand *parmOpnd2 = SelectParmListDreadAccessField(sym, fieldID, ploc, offset, 2);
7804 srcOpnds.PushOpnd(*parmOpnd2);
7805 }
7806 if (ploc.reg3) {
7807 RegOperand *parmOpnd3 = SelectParmListDreadAccessField(sym, fieldID, ploc, offset, 3);
7808 srcOpnds.PushOpnd(*parmOpnd3);
7809 }
7810 }
7811 }
7812
SelectParmListIreadSmallAggregate(const IreadNode & iread,MIRType & structType,ListOperand & srcOpnds,int32 offset,AArch64CallConvImpl & parmLocator)7813 void AArch64CGFunc::SelectParmListIreadSmallAggregate(const IreadNode &iread, MIRType &structType,
7814 ListOperand &srcOpnds, int32 offset,
7815 AArch64CallConvImpl &parmLocator)
7816 {
7817 int32 symSize = GetBecommon().GetTypeSize(structType.GetTypeIndex().GetIdx());
7818 RegOperand *addrOpnd0 = static_cast<RegOperand *>(HandleExpr(iread, *(iread.Opnd(0))));
7819 RegOperand *addrOpnd1 = &LoadIntoRegister(*addrOpnd0, iread.Opnd(0)->GetPrimType());
7820 CCLocInfo ploc;
7821 CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
7822 parmLocator.LocateNextParm(structType, ploc);
7823 if (ploc.reg0 == 0) {
7824 /* No param regs available, pass on stack. */
7825 CreateCallStructParamPassByStack(symSize, nullptr, addrOpnd1, ploc.memOffset);
7826 } else {
7827 /* pass by param regs. */
7828 fpParamState state = kStateUnknown;
7829 uint32 memSize = 0;
7830 switch (ploc.fpSize) {
7831 case k0BitSize:
7832 state = kNotFp;
7833 memSize = k64BitSize;
7834 break;
7835 case k4BitSize:
7836 state = kFp32Bit;
7837 memSize = k32BitSize;
7838 break;
7839 case k8BitSize:
7840 state = kFp64Bit;
7841 memSize = k64BitSize;
7842 break;
7843 default:
7844 break;
7845 }
7846 OfstOperand *offOpnd0 = &GetOrCreateOfstOpnd(static_cast<uint64>(static_cast<int64>(offset)), k32BitSize);
7847 MemOperand *mopnd =
7848 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, memSize, addrOpnd1, nullptr, offOpnd0, nullptr);
7849 CreateCallStructParamPassByReg(ploc.reg0, *mopnd, srcOpnds, state);
7850 if (ploc.reg1) {
7851 OfstOperand *offOpnd1 = &GetOrCreateOfstOpnd(
7852 ((ploc.fpSize ? ploc.fpSize : GetPointerSize()) + static_cast<uint32>(offset)), k32BitSize);
7853 mopnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, memSize, addrOpnd1, nullptr, offOpnd1, nullptr);
7854 CreateCallStructParamPassByReg(ploc.reg1, *mopnd, srcOpnds, state);
7855 }
7856 if (ploc.reg2) {
7857 OfstOperand *offOpnd2 = &GetOrCreateOfstOpnd(
7858 ((ploc.fpSize ? (ploc.fpSize * k4BitShift) : GetPointerSize()) + static_cast<uint32>(offset)),
7859 k32BitSize);
7860 mopnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, memSize, addrOpnd1, nullptr, offOpnd2, nullptr);
7861 CreateCallStructParamPassByReg(ploc.reg2, *mopnd, srcOpnds, state);
7862 }
7863 if (ploc.reg3) {
7864 OfstOperand *offOpnd3 = &GetOrCreateOfstOpnd(
7865 ((ploc.fpSize ? (ploc.fpSize * k8BitShift) : GetPointerSize()) + static_cast<uint32>(offset)),
7866 k32BitSize);
7867 mopnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, memSize, addrOpnd1, nullptr, offOpnd3, nullptr);
7868 CreateCallStructParamPassByReg(ploc.reg3, *mopnd, srcOpnds, state);
7869 }
7870 }
7871 }
7872
SelectParmListDreadLargeAggregate(const MIRSymbol & sym,MIRType & structType,ListOperand & srcOpnds,AArch64CallConvImpl & parmLocator,int32 & structCopyOffset,int32 fromOffset)7873 void AArch64CGFunc::SelectParmListDreadLargeAggregate(const MIRSymbol &sym, MIRType &structType, ListOperand &srcOpnds,
7874 AArch64CallConvImpl &parmLocator, int32 &structCopyOffset,
7875 int32 fromOffset)
7876 {
7877 /*
7878 * Pass larger sized struct on stack.
7879 * Need to copy the entire structure onto the stack.
7880 * The pointer to the starting address of the copied struct is then
7881 * used as the parameter for the struct.
7882 * This pointer is passed as the next parameter.
7883 * Example 1:
7884 * struct is 23 bytes.
7885 * param regs x0 to x5 are used.
7886 * First around up 23 to 24, so 3 of 8-byte slots.
7887 * Copy struct to a created space on the stack.
7888 * Pointer of copied struct is passed in x6.
7889 * Example 2:
7890 * struct is 25 bytes.
7891 * param regs x0 to x7 are used.
7892 * First around up 25 to 32, so 4 of 8-byte slots.
7893 * Copy struct to a created space on the stack.
7894 * Pointer of copied struct is passed on stack as the 9th parameter.
7895 */
7896 uint64 symSize = GetBecommon().GetTypeSize(structType.GetTypeIndex().GetIdx());
7897 CCLocInfo ploc;
7898 CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
7899 parmLocator.LocateNextParm(structType, ploc);
7900 uint32 numMemOp = static_cast<uint32>(RoundUp(symSize, GetPointerSize()) / GetPointerSize()); /* round up */
7901 /* Create the struct copies. */
7902 RegOperand *parmOpnd =
7903 CreateCallStructParamCopyToStack(numMemOp, &sym, nullptr, structCopyOffset, fromOffset, ploc);
7904 if (parmOpnd) {
7905 srcOpnds.PushOpnd(*parmOpnd);
7906 }
7907 structCopyOffset += static_cast<int32>(numMemOp * GetPointerSize());
7908 }
7909
SelectParmListIreadLargeAggregate(const IreadNode & iread,MIRType & structType,ListOperand & srcOpnds,AArch64CallConvImpl & parmLocator,int32 & structCopyOffset,int32 fromOffset)7910 void AArch64CGFunc::SelectParmListIreadLargeAggregate(const IreadNode &iread, MIRType &structType,
7911 ListOperand &srcOpnds, AArch64CallConvImpl &parmLocator,
7912 int32 &structCopyOffset, int32 fromOffset)
7913 {
7914 uint64 symSize = GetBecommon().GetTypeSize(structType.GetTypeIndex().GetIdx());
7915 RegOperand *addrOpnd0 = static_cast<RegOperand *>(HandleExpr(iread, *(iread.Opnd(0))));
7916 RegOperand *addrOpnd1 = &LoadIntoRegister(*addrOpnd0, iread.Opnd(0)->GetPrimType());
7917 CCLocInfo ploc;
7918 CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
7919 parmLocator.LocateNextParm(structType, ploc);
7920 uint32 numMemOp = static_cast<uint32>(RoundUp(symSize, GetPointerSize()) / GetPointerSize()); /* round up */
7921 RegOperand *parmOpnd =
7922 CreateCallStructParamCopyToStack(numMemOp, nullptr, addrOpnd1, structCopyOffset, fromOffset, ploc);
7923 structCopyOffset += static_cast<int32>(numMemOp * GetPointerSize());
7924 if (parmOpnd) {
7925 srcOpnds.PushOpnd(*parmOpnd);
7926 }
7927 }
7928
CreateCallStructParamPassByStack(int32 symSize,const MIRSymbol * sym,RegOperand * addrOpnd,int32 baseOffset)7929 void AArch64CGFunc::CreateCallStructParamPassByStack(int32 symSize, const MIRSymbol *sym, RegOperand *addrOpnd,
7930 int32 baseOffset)
7931 {
7932 MemOperand *ldMopnd = nullptr;
7933 MemOperand *stMopnd = nullptr;
7934 uint32 numRegNeeded = (static_cast<uint32>(symSize) <= k8ByteSize) ? kOneRegister : kTwoRegister;
7935 for (int j = 0; j < static_cast<int>(numRegNeeded); j++) {
7936 if (sym) {
7937 if (CGOptions::IsArm64ilp32()) {
7938 ldMopnd = &GetOrCreateMemOpnd(*sym, (j * static_cast<int>(k8ByteSize)), k64BitSize);
7939 } else {
7940 ldMopnd = &GetOrCreateMemOpnd(*sym, (j * static_cast<int>(GetPointerSize())), k64BitSize);
7941 }
7942 } else {
7943 if (CGOptions::IsArm64ilp32()) {
7944 ldMopnd =
7945 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, addrOpnd, nullptr,
7946 &GetOrCreateOfstOpnd(static_cast<uint32>(j) * k8ByteSize, k32BitSize), nullptr);
7947 } else {
7948 ldMopnd = &GetOrCreateMemOpnd(
7949 MemOperand::kAddrModeBOi, k64BitSize, addrOpnd, nullptr,
7950 &GetOrCreateOfstOpnd(static_cast<uint32>(j) * GetPointerSize(), k32BitSize), nullptr);
7951 }
7952 }
7953 RegOperand *vreg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
7954 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_i64), *vreg, *ldMopnd));
7955 if (CGOptions::IsArm64ilp32()) {
7956 stMopnd =
7957 &CreateMemOpnd(RSP, (static_cast<int64>(baseOffset) + (j * static_cast<int>(k8ByteSize))), k64BitSize);
7958 } else {
7959 stMopnd = &CreateMemOpnd(RSP, (static_cast<int64>(baseOffset) + (j * GetPointerSize())), k64BitSize);
7960 }
7961 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(k64BitSize, PTY_i64), *vreg, *stMopnd));
7962 }
7963 }
7964
SelectParmListDreadAccessField(const MIRSymbol & sym,FieldID fieldID,const CCLocInfo & ploc,int32 offset,uint32 parmNum)7965 RegOperand *AArch64CGFunc::SelectParmListDreadAccessField(const MIRSymbol &sym, FieldID fieldID, const CCLocInfo &ploc,
7966 int32 offset, uint32 parmNum)
7967 {
7968 uint32 memSize;
7969 PrimType primType;
7970 RegOperand *parmOpnd;
7971 uint32 dataSizeBits;
7972 AArch64reg reg;
7973 switch (parmNum) {
7974 case 0:
7975 reg = static_cast<AArch64reg>(ploc.reg0);
7976 break;
7977 case 1:
7978 reg = static_cast<AArch64reg>(ploc.reg1);
7979 break;
7980 case 2: // parmNum 2 use reg2
7981 reg = static_cast<AArch64reg>(ploc.reg2);
7982 break;
7983 case 3: // parmNum 3 use reg3
7984 reg = static_cast<AArch64reg>(ploc.reg3);
7985 break;
7986 default:
7987 CHECK_FATAL(false, "Exceeded maximum allowed fp parameter registers for struct passing");
7988 }
7989 if (ploc.fpSize == 0) {
7990 memSize = k64BitSize;
7991 primType = PTY_i64;
7992 dataSizeBits = GetPrimTypeSize(PTY_i64) * kBitsPerByte;
7993 parmOpnd = &GetOrCreatePhysicalRegisterOperand(reg, k64BitSize, kRegTyInt);
7994 } else if (ploc.fpSize == k4ByteSize) {
7995 memSize = k32BitSize;
7996 primType = PTY_f32;
7997 dataSizeBits = GetPrimTypeSize(PTY_f32) * kBitsPerByte;
7998 parmOpnd = &GetOrCreatePhysicalRegisterOperand(reg, k32BitSize, kRegTyFloat);
7999 } else if (ploc.fpSize == k8ByteSize) {
8000 memSize = k64BitSize;
8001 primType = PTY_f64;
8002 dataSizeBits = GetPrimTypeSize(PTY_i64) * kBitsPerByte;
8003 parmOpnd = &GetOrCreatePhysicalRegisterOperand(reg, k64BitSize, kRegTyFloat);
8004 } else {
8005 CHECK_FATAL(false, "Unknown call parameter state");
8006 }
8007 MemOperand *memOpnd;
8008 if (sym.GetStorageClass() == kScFormal && fieldID > 0) {
8009 MemOperand &baseOpnd = GetOrCreateMemOpnd(sym, 0, memSize);
8010 RegOperand &base = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8011 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_i64), base, baseOpnd));
8012 memOpnd = &CreateMemOpnd(base, (static_cast<int64>(offset) + parmNum * GetPointerSize()), memSize);
8013 } else if (ploc.fpSize) {
8014 memOpnd = &GetOrCreateMemOpnd(sym, (ploc.fpSize * parmNum + static_cast<int64>(offset)), memSize);
8015 } else {
8016 if (CGOptions::IsArm64ilp32()) {
8017 memOpnd = &GetOrCreateMemOpnd(sym, (k8ByteSize * parmNum + static_cast<int64>(offset)), memSize);
8018 } else {
8019 memOpnd = &GetOrCreateMemOpnd(sym, (GetPointerSize() * parmNum + static_cast<int64>(offset)), memSize);
8020 }
8021 }
8022 MOperator selectedMop = PickLdInsn(dataSizeBits, primType);
8023 if ((memOpnd->GetAddrMode() == MemOperand::kAddrModeBOi) &&
8024 !IsOperandImmValid(selectedMop, memOpnd, kInsnSecondOpnd)) {
8025 memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, dataSizeBits);
8026 }
8027 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(selectedMop, *parmOpnd, *memOpnd));
8028
8029 return parmOpnd;
8030 }
8031
CreateCallStructParamPassByReg(regno_t regno,MemOperand & memOpnd,ListOperand & srcOpnds,fpParamState state)8032 void AArch64CGFunc::CreateCallStructParamPassByReg(regno_t regno, MemOperand &memOpnd, ListOperand &srcOpnds,
8033 fpParamState state)
8034 {
8035 RegOperand *parmOpnd;
8036 uint32 dataSizeBits = 0;
8037 PrimType pType = PTY_void;
8038 parmOpnd = nullptr;
8039 AArch64reg reg = static_cast<AArch64reg>(regno);
8040 if (state == kNotFp) {
8041 parmOpnd = &GetOrCreatePhysicalRegisterOperand(reg, k64BitSize, kRegTyInt);
8042 dataSizeBits = GetPrimTypeSize(PTY_i64) * kBitsPerByte;
8043 pType = PTY_i64;
8044 } else if (state == kFp32Bit) {
8045 parmOpnd = &GetOrCreatePhysicalRegisterOperand(reg, k32BitSize, kRegTyFloat);
8046 dataSizeBits = GetPrimTypeSize(PTY_f32) * kBitsPerByte;
8047 pType = PTY_f32;
8048 } else if (state == kFp64Bit) {
8049 parmOpnd = &GetOrCreatePhysicalRegisterOperand(reg, k64BitSize, kRegTyFloat);
8050 dataSizeBits = GetPrimTypeSize(PTY_f64) * kBitsPerByte;
8051 pType = PTY_f64;
8052 } else {
8053 DEBUG_ASSERT(0, "CreateCallStructParamPassByReg: Unknown state");
8054 }
8055
8056 MOperator selectedMop = PickLdInsn(dataSizeBits, pType);
8057 if (!IsOperandImmValid(selectedMop, &memOpnd, kInsnSecondOpnd)) {
8058 memOpnd = SplitOffsetWithAddInstruction(memOpnd, dataSizeBits);
8059 }
8060 DEBUG_ASSERT(parmOpnd != nullptr, "parmOpnd should not be nullptr");
8061 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(selectedMop, *parmOpnd, memOpnd));
8062 srcOpnds.PushOpnd(*parmOpnd);
8063 }
8064
CreateCallStructParamMemcpy(const MIRSymbol * sym,RegOperand * addropnd,uint32 structSize,int32 copyOffset,int32 fromOffset)8065 void AArch64CGFunc::CreateCallStructParamMemcpy(const MIRSymbol *sym, RegOperand *addropnd, uint32 structSize,
8066 int32 copyOffset, int32 fromOffset)
8067 {
8068 std::vector<Operand *> opndVec;
8069
8070 RegOperand *vreg1 = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8BitSize));
8071 opndVec.push_back(vreg1); /* result */
8072
8073 RegOperand *parmOpnd = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8BitSize));
8074 RegOperand *spReg = &GetOrCreatePhysicalRegisterOperand(RSP, k64BitSize, kRegTyInt);
8075 ImmOperand *offsetOpnd0 = &CreateImmOperand(copyOffset, k64BitSize, false);
8076 SelectAdd(*parmOpnd, *spReg, *offsetOpnd0, PTY_a64);
8077 opndVec.push_back(parmOpnd); /* param 0 */
8078
8079 if (sym != nullptr) {
8080 if (sym->GetStorageClass() == kScGlobal || sym->GetStorageClass() == kScExtern) {
8081 StImmOperand &stopnd = CreateStImmOperand(*sym, fromOffset, 0);
8082 RegOperand &staddropnd = static_cast<RegOperand &>(CreateRegisterOperandOfType(PTY_u64));
8083 SelectAddrof(staddropnd, stopnd);
8084 opndVec.push_back(&staddropnd); /* param 1 */
8085 } else if (sym->GetStorageClass() == kScAuto || sym->GetStorageClass() == kScFormal) {
8086 RegOperand *parm1Reg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8087 AArch64SymbolAlloc *symloc =
8088 static_cast<AArch64SymbolAlloc *>(GetMemlayout()->GetSymAllocInfo(sym->GetStIndex()));
8089 RegOperand *baseOpnd = static_cast<RegOperand *>(GetBaseReg(*symloc));
8090 int32 stoffset = GetBaseOffset(*symloc);
8091 ImmOperand *offsetOpnd1 = &CreateImmOperand(static_cast<int64>(stoffset), k64BitSize, false);
8092 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *parm1Reg, *baseOpnd, *offsetOpnd1));
8093 if (sym->GetStorageClass() == kScFormal) {
8094 MemOperand *ldmopnd =
8095 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, parm1Reg, nullptr,
8096 &GetOrCreateOfstOpnd(0, k32BitSize), static_cast<MIRSymbol *>(nullptr));
8097 RegOperand *tmpreg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8098 RegOperand *vreg2 = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8099 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_a64), *tmpreg, *ldmopnd));
8100 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *vreg2, *tmpreg,
8101 CreateImmOperand(fromOffset, k64BitSize, false)));
8102 parm1Reg = vreg2;
8103 }
8104 opndVec.push_back(parm1Reg); /* param 1 */
8105 } else if (sym->GetStorageClass() == kScPstatic || sym->GetStorageClass() == kScFstatic) {
8106 CHECK_FATAL(sym->GetSKind() != kStConst, "Unsupported sym const for struct param");
8107 StImmOperand *stopnd = &CreateStImmOperand(*sym, 0, 0);
8108 RegOperand &staddropnd = static_cast<RegOperand &>(CreateRegisterOperandOfType(PTY_u64));
8109 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrp, staddropnd, *stopnd));
8110 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xadrpl12, staddropnd, staddropnd, *stopnd));
8111 opndVec.push_back(&staddropnd); /* param 1 */
8112 } else {
8113 CHECK_FATAL(0, "Unsupported sym for struct param");
8114 }
8115 } else {
8116 opndVec.push_back(addropnd); /* param 1 */
8117 }
8118
8119 RegOperand &vreg3 = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8BitSize));
8120 ImmOperand &sizeOpnd = CreateImmOperand(structSize, k64BitSize, false);
8121 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wmovri32, vreg3, sizeOpnd));
8122 opndVec.push_back(&vreg3); /* param 2 */
8123
8124 SelectLibCall("memcpy", opndVec, PTY_a64, PTY_a64);
8125 }
8126
CreateCallStructParamCopyToStack(uint32 numMemOp,const MIRSymbol * sym,RegOperand * addrOpd,int32 copyOffset,int32 fromOffset,const CCLocInfo & ploc)8127 RegOperand *AArch64CGFunc::CreateCallStructParamCopyToStack(uint32 numMemOp, const MIRSymbol *sym, RegOperand *addrOpd,
8128 int32 copyOffset, int32 fromOffset, const CCLocInfo &ploc)
8129 {
8130 /* Create the struct copies. */
8131 MemOperand *ldMopnd = nullptr;
8132 MemOperand *stMopnd = nullptr;
8133 for (uint32 j = 0; j < numMemOp; j++) {
8134 if (sym != nullptr) {
8135 if (sym->GetStorageClass() == kScFormal) {
8136 MemOperand &base = GetOrCreateMemOpnd(*sym, 0, k64BitSize);
8137 RegOperand &vreg = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8138 Insn &ldInsn = GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_i64), vreg, base);
8139 GetCurBB()->AppendInsn(ldInsn);
8140 ldMopnd = &GetOrCreateMemOpnd(
8141 MemOperand::kAddrModeBOi, k64BitSize, &vreg, nullptr,
8142 &GetOrCreateOfstOpnd((j * GetPointerSize() + static_cast<uint64>(fromOffset)), k32BitSize),
8143 nullptr);
8144 } else {
8145 if (CGOptions::IsArm64ilp32()) {
8146 ldMopnd =
8147 &GetOrCreateMemOpnd(*sym, (j * GetPointerSize() + static_cast<int64>(fromOffset)), k32BitSize);
8148 } else {
8149 ldMopnd =
8150 &GetOrCreateMemOpnd(*sym, (j * GetPointerSize() + static_cast<int64>(fromOffset)), k64BitSize);
8151 }
8152 }
8153 } else {
8154 ldMopnd = &GetOrCreateMemOpnd(
8155 MemOperand::kAddrModeBOi, k64BitSize, addrOpd, nullptr,
8156 &GetOrCreateOfstOpnd((j * GetPointerSize() + static_cast<uint64>(fromOffset)), k32BitSize), nullptr);
8157 }
8158 if (CGOptions::IsArm64ilp32()) {
8159 RegOperand *vreg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k4ByteSize));
8160 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(k32BitSize, PTY_i32), *vreg, *ldMopnd));
8161
8162 stMopnd = &CreateMemOpnd(RSP, (static_cast<int64>(copyOffset) + (j * GetPointerSize())), k32BitSize);
8163 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(k32BitSize, PTY_i32), *vreg, *stMopnd));
8164 } else {
8165 RegOperand *vreg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8166 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_i64), *vreg, *ldMopnd));
8167
8168 stMopnd = &CreateMemOpnd(RSP, (static_cast<int64>(copyOffset) + (j * GetPointerSize())), k64BitSize);
8169 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(k64BitSize, PTY_i64), *vreg, *stMopnd));
8170 }
8171 }
8172 /* Create the copy address parameter for the struct */
8173 RegOperand *fpopnd = &GetOrCreatePhysicalRegisterOperand(RSP, k64BitSize, kRegTyInt);
8174 ImmOperand *offset = &CreateImmOperand(copyOffset, k64BitSize, false);
8175 if (ploc.reg0 == kRinvalid) {
8176 RegOperand &res = CreateRegisterOperandOfType(PTY_u64);
8177 SelectAdd(res, *fpopnd, *offset, PTY_u64);
8178 MemOperand &stMopnd2 = CreateMemOpnd(RSP, ploc.memOffset, k64BitSize);
8179 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(k64BitSize, PTY_i64), res, stMopnd2));
8180 return nullptr;
8181 } else {
8182 RegOperand *parmOpnd =
8183 &GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(ploc.reg0), k64BitSize, kRegTyInt);
8184 SelectAdd(*parmOpnd, *fpopnd, *offset, PTY_a64);
8185 return parmOpnd;
8186 }
8187 }
8188
CreateCallStructMemcpyToParamReg(MIRType & structType,int32 structCopyOffset,AArch64CallConvImpl & parmLocator,ListOperand & srcOpnds)8189 void AArch64CGFunc::CreateCallStructMemcpyToParamReg(MIRType &structType, int32 structCopyOffset,
8190 AArch64CallConvImpl &parmLocator, ListOperand &srcOpnds)
8191 {
8192 RegOperand &spReg = GetOrCreatePhysicalRegisterOperand(RSP, k64BitSize, kRegTyInt);
8193 ImmOperand &offsetOpnd = CreateImmOperand(structCopyOffset, k64BitSize, false);
8194
8195 CCLocInfo ploc;
8196 parmLocator.LocateNextParm(structType, ploc);
8197 CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
8198 if (ploc.reg0 != 0) {
8199 RegOperand &res = GetOrCreatePhysicalRegisterOperand(static_cast<AArch64reg>(ploc.reg0), k64BitSize, kRegTyInt);
8200 SelectAdd(res, spReg, offsetOpnd, PTY_a64);
8201 srcOpnds.PushOpnd(res);
8202 } else {
8203 RegOperand &parmOpnd = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
8204 SelectAdd(parmOpnd, spReg, offsetOpnd, PTY_a64);
8205 MemOperand &stmopnd = CreateMemOpnd(RSP, ploc.memOffset, k64BitSize);
8206 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickStInsn(k64BitSize, PTY_i64), parmOpnd, stmopnd));
8207 }
8208 }
8209
SelectParmListForAggregate(BaseNode & argExpr,ListOperand & srcOpnds,AArch64CallConvImpl & parmLocator,int32 & structCopyOffset)8210 void AArch64CGFunc::SelectParmListForAggregate(BaseNode &argExpr, ListOperand &srcOpnds,
8211 AArch64CallConvImpl &parmLocator, int32 &structCopyOffset)
8212 {
8213 uint64 symSize;
8214 int32 rhsOffset = 0;
8215 if (argExpr.GetOpCode() == OP_dread) {
8216 DreadNode &dread = static_cast<DreadNode &>(argExpr);
8217 MIRSymbol *sym = GetBecommon().GetMIRModule().CurFunction()->GetLocalOrGlobalSymbol(dread.GetStIdx());
8218 MIRType *ty = sym->GetType();
8219 if (dread.GetFieldID() != 0) {
8220 MIRStructType *structty = static_cast<MIRStructType *>(ty);
8221 ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(structty->GetFieldTyIdx(dread.GetFieldID()));
8222 rhsOffset = GetBecommon().GetFieldOffset(*structty, dread.GetFieldID()).first;
8223 }
8224 symSize = GetBecommon().GetTypeSize(ty->GetTypeIndex().GetIdx());
8225 if (symSize <= k16ByteSize) {
8226 SelectParmListDreadSmallAggregate(*sym, *ty, srcOpnds, rhsOffset, parmLocator, dread.GetFieldID());
8227 } else if (symSize > kParmMemcpySize) {
8228 CreateCallStructMemcpyToParamReg(*ty, structCopyOffset, parmLocator, srcOpnds);
8229 structCopyOffset += static_cast<int32>(RoundUp(symSize, GetPointerSize()));
8230 } else {
8231 SelectParmListDreadLargeAggregate(*sym, *ty, srcOpnds, parmLocator, structCopyOffset, rhsOffset);
8232 }
8233 } else if (argExpr.GetOpCode() == OP_iread) {
8234 IreadNode &iread = static_cast<IreadNode &>(argExpr);
8235 MIRPtrType *pointerty =
8236 static_cast<MIRPtrType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(iread.GetTyIdx()));
8237 MIRType *ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(pointerty->GetPointedTyIdx());
8238 if (iread.GetFieldID() != 0) {
8239 MIRStructType *structty = static_cast<MIRStructType *>(ty);
8240 ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(structty->GetFieldTyIdx(iread.GetFieldID()));
8241 rhsOffset = GetBecommon().GetFieldOffset(*structty, iread.GetFieldID()).first;
8242 }
8243 symSize = GetBecommon().GetTypeSize(ty->GetTypeIndex().GetIdx());
8244 if (symSize <= k16ByteSize) {
8245 SelectParmListIreadSmallAggregate(iread, *ty, srcOpnds, rhsOffset, parmLocator);
8246 } else if (symSize > kParmMemcpySize) {
8247 RegOperand *ireadOpnd = static_cast<RegOperand *>(HandleExpr(iread, *(iread.Opnd(0))));
8248 if (rhsOffset > 0) {
8249 RegOperand *addrOpnd = &LoadIntoRegister(*ireadOpnd, iread.Opnd(0)->GetPrimType());
8250 regno_t vRegNO = NewVReg(kRegTyInt, k8ByteSize);
8251 RegOperand *result = &CreateVirtualRegisterOperand(vRegNO);
8252 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *result, *addrOpnd,
8253 CreateImmOperand(rhsOffset, k64BitSize, false)));
8254 }
8255
8256 CreateCallStructMemcpyToParamReg(*ty, structCopyOffset, parmLocator, srcOpnds);
8257 structCopyOffset += static_cast<int32>(RoundUp(symSize, GetPointerSize()));
8258 } else {
8259 SelectParmListIreadLargeAggregate(iread, *ty, srcOpnds, parmLocator, structCopyOffset, rhsOffset);
8260 }
8261 } else {
8262 CHECK_FATAL(0, "NYI");
8263 }
8264 }
8265
SelectParmListGetStructReturnSize(StmtNode & naryNode)8266 size_t AArch64CGFunc::SelectParmListGetStructReturnSize(StmtNode &naryNode)
8267 {
8268 if (naryNode.GetOpCode() == OP_call) {
8269 CallNode &callNode = static_cast<CallNode &>(naryNode);
8270 MIRFunction *callFunc = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(callNode.GetPUIdx());
8271 TyIdx retIdx = callFunc->GetReturnTyIdx();
8272 if (callFunc->IsFirstArgReturn()) {
8273 MIRType *ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(callFunc->GetFormalDefVec()[0].formalTyIdx);
8274 return GetBecommon().GetTypeSize(static_cast<MIRPtrType *>(ty)->GetPointedTyIdx());
8275 }
8276 size_t retSize = GetBecommon().GetTypeSize(retIdx.GetIdx());
8277 if ((retSize == 0) && callFunc->IsReturnStruct()) {
8278 TyIdx tyIdx = callFunc->GetFuncRetStructTyIdx();
8279 return GetBecommon().GetTypeSize(tyIdx);
8280 }
8281 return retSize;
8282 } else if (naryNode.GetOpCode() == OP_icall) {
8283 IcallNode &icallNode = static_cast<IcallNode &>(naryNode);
8284 CallReturnVector *p2nrets = &icallNode.GetReturnVec();
8285 if (p2nrets->size() == k1ByteSize) {
8286 StIdx stIdx = (*p2nrets)[0].first;
8287 MIRSymbol *sym = GetBecommon().GetMIRModule().CurFunction()->GetSymTab()->GetSymbolFromStIdx(stIdx.Idx());
8288 if (sym != nullptr) {
8289 return GetBecommon().GetTypeSize(sym->GetTyIdx().GetIdx());
8290 }
8291 }
8292 } else if (naryNode.GetOpCode() == OP_icallproto) {
8293 IcallNode &icallProto = static_cast<IcallNode &>(naryNode);
8294 MIRFuncType *funcTy =
8295 static_cast<MIRFuncType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(icallProto.GetRetTyIdx()));
8296 if (funcTy->FirstArgReturn()) {
8297 MIRType *ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(funcTy->GetNthParamType(0));
8298 return GetBecommon().GetTypeSize(static_cast<MIRPtrType *>(ty)->GetPointedTyIdx());
8299 }
8300 return GetBecommon().GetTypeSize(funcTy->GetRetTyIdx());
8301 }
8302 return 0;
8303 }
8304
SelectParmListPreprocessLargeStruct(BaseNode & argExpr,int32 & structCopyOffset)8305 void AArch64CGFunc::SelectParmListPreprocessLargeStruct(BaseNode &argExpr, int32 &structCopyOffset)
8306 {
8307 uint64 symSize;
8308 int32 rhsOffset = 0;
8309 if (argExpr.GetOpCode() == OP_dread) {
8310 DreadNode &dread = static_cast<DreadNode &>(argExpr);
8311 MIRSymbol *sym = GetBecommon().GetMIRModule().CurFunction()->GetLocalOrGlobalSymbol(dread.GetStIdx());
8312 MIRType *ty = sym->GetType();
8313 if (dread.GetFieldID() != 0) {
8314 MIRStructType *structty = static_cast<MIRStructType *>(ty);
8315 ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(structty->GetFieldTyIdx(dread.GetFieldID()));
8316 rhsOffset = GetBecommon().GetFieldOffset(*structty, dread.GetFieldID()).first;
8317 }
8318 symSize = GetBecommon().GetTypeSize(ty->GetTypeIndex().GetIdx());
8319 if (symSize > kParmMemcpySize) {
8320 CreateCallStructParamMemcpy(sym, nullptr, static_cast<uint32>(symSize), structCopyOffset, rhsOffset);
8321 structCopyOffset += static_cast<int32>(RoundUp(symSize, GetPointerSize()));
8322 } else if (symSize > k16ByteSize) {
8323 uint32 numMemOp = static_cast<uint32>(RoundUp(symSize, GetPointerSize()) / GetPointerSize());
8324 structCopyOffset += static_cast<int32>(numMemOp * GetPointerSize());
8325 }
8326 } else if (argExpr.GetOpCode() == OP_iread) {
8327 IreadNode &iread = static_cast<IreadNode &>(argExpr);
8328 MIRPtrType *pointerty =
8329 static_cast<MIRPtrType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(iread.GetTyIdx()));
8330 MIRType *ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(pointerty->GetPointedTyIdx());
8331 if (iread.GetFieldID() != 0) {
8332 MIRStructType *structty = static_cast<MIRStructType *>(ty);
8333 ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(structty->GetFieldTyIdx(iread.GetFieldID()));
8334 rhsOffset = GetBecommon().GetFieldOffset(*structty, iread.GetFieldID()).first;
8335 }
8336 symSize = GetBecommon().GetTypeSize(ty->GetTypeIndex().GetIdx());
8337 if (symSize > kParmMemcpySize) {
8338 RegOperand *ireadOpnd = static_cast<RegOperand *>(HandleExpr(iread, *(iread.Opnd(0))));
8339 RegOperand *addrOpnd = &LoadIntoRegister(*ireadOpnd, iread.Opnd(0)->GetPrimType());
8340 if (rhsOffset > 0) {
8341 regno_t vRegNO = NewVReg(kRegTyInt, k8ByteSize);
8342 RegOperand *result = &CreateVirtualRegisterOperand(vRegNO);
8343 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xaddrri12, *result, *addrOpnd,
8344 CreateImmOperand(rhsOffset, k64BitSize, false)));
8345 addrOpnd = result;
8346 }
8347
8348 CreateCallStructParamMemcpy(nullptr, addrOpnd, static_cast<uint32>(symSize), structCopyOffset, rhsOffset);
8349 structCopyOffset += static_cast<int32>(RoundUp(symSize, GetPointerSize()));
8350 } else if (symSize > k16ByteSize) {
8351 uint32 numMemOp = static_cast<uint32>(RoundUp(symSize, GetPointerSize()) / GetPointerSize());
8352 structCopyOffset += static_cast<int32>(numMemOp * GetPointerSize());
8353 }
8354 }
8355 }
8356
8357 /* preprocess call in parmlist */
MarkParmListCall(BaseNode & expr)8358 bool AArch64CGFunc::MarkParmListCall(BaseNode &expr)
8359 {
8360 if (!CGOptions::IsPIC()) {
8361 return false;
8362 }
8363 switch (expr.GetOpCode()) {
8364 case OP_addrof: {
8365 auto &addrNode = static_cast<AddrofNode &>(expr);
8366 MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(addrNode.GetStIdx());
8367 if (symbol->IsThreadLocal()) {
8368 return true;
8369 }
8370 break;
8371 }
8372 default: {
8373 for (auto i = 0; i < expr.GetNumOpnds(); i++) {
8374 if (expr.Opnd(i)) {
8375 if (MarkParmListCall(*expr.Opnd(i))) {
8376 return true;
8377 }
8378 }
8379 }
8380 break;
8381 }
8382 }
8383 return false;
8384 }
8385
SelectParmListPreprocess(const StmtNode & naryNode,size_t start,std::set<size_t> & specialArgs)8386 void AArch64CGFunc::SelectParmListPreprocess(const StmtNode &naryNode, size_t start, std::set<size_t> &specialArgs)
8387 {
8388 size_t i = start;
8389 int32 structCopyOffset = GetMaxParamStackSize() - GetStructCopySize();
8390 for (; i < naryNode.NumOpnds(); ++i) {
8391 BaseNode *argExpr = naryNode.Opnd(i);
8392 PrimType primType = argExpr->GetPrimType();
8393 if (MarkParmListCall(*argExpr)) {
8394 (void)specialArgs.emplace(i);
8395 }
8396 DEBUG_ASSERT(primType != PTY_void, "primType should not be void");
8397 if (primType != PTY_agg) {
8398 continue;
8399 }
8400 SelectParmListPreprocessLargeStruct(*argExpr, structCopyOffset);
8401 }
8402 }
8403
8404 /*
8405 SelectParmList generates an instrunction for each of the parameters
8406 to load the parameter value into the corresponding register.
8407 We return a list of registers to the call instruction because
8408 they may be needed in the register allocation phase.
8409 */
SelectParmList(StmtNode & naryNode,ListOperand & srcOpnds,bool isCallNative)8410 void AArch64CGFunc::SelectParmList(StmtNode &naryNode, ListOperand &srcOpnds, bool isCallNative)
8411 {
8412 size_t i = 0;
8413 if (naryNode.GetOpCode() == OP_icall || naryNode.GetOpCode() == OP_icallproto || isCallNative) {
8414 i++;
8415 }
8416 std::set<size_t> specialArgs;
8417 SelectParmListPreprocess(naryNode, i, specialArgs);
8418 bool specialArg = false;
8419 bool firstArgReturn = false;
8420 MIRFunction *callee = nullptr;
8421 if (dynamic_cast<CallNode *>(&naryNode) != nullptr) {
8422 auto calleePuIdx = static_cast<CallNode &>(naryNode).GetPUIdx();
8423 callee = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(calleePuIdx);
8424 firstArgReturn = callee->IsFirstArgReturn();
8425 } else if (naryNode.GetOpCode() == OP_icallproto) {
8426 IcallNode *icallnode = &static_cast<IcallNode &>(naryNode);
8427 MIRFuncType *funcType =
8428 static_cast<MIRFuncType *>(GlobalTables::GetTypeTable().GetTypeFromTyIdx(icallnode->GetRetTyIdx()));
8429 firstArgReturn = funcType->FirstArgReturn();
8430 }
8431 BB *curBBrecord = GetCurBB();
8432 BB *tmpBB = nullptr;
8433 if (!specialArgs.empty()) {
8434 tmpBB = CreateNewBB();
8435 specialArg = true;
8436 }
8437 AArch64CallConvImpl parmLocator(GetBecommon());
8438 CCLocInfo ploc;
8439 int32 structCopyOffset = GetMaxParamStackSize() - GetStructCopySize();
8440 std::vector<Insn *> insnForStackArgs;
8441 uint32 stackArgsCount = 0;
8442 for (uint32 pnum = 0; i < naryNode.NumOpnds(); ++i, ++pnum) {
8443 if (specialArg) {
8444 DEBUG_ASSERT(tmpBB, "need temp bb for lower priority args");
8445 SetCurBB(specialArgs.count(i) ? *curBBrecord : *tmpBB);
8446 }
8447 bool is64x1vec = false;
8448 MIRType *ty = nullptr;
8449 BaseNode *argExpr = naryNode.Opnd(i);
8450 PrimType primType = argExpr->GetPrimType();
8451 DEBUG_ASSERT(primType != PTY_void, "primType should not be void");
8452 if (callee != nullptr && pnum < callee->GetFormalCount() && callee->GetFormal(pnum) != nullptr) {
8453 is64x1vec = callee->GetFormal(pnum)->GetAttr(ATTR_oneelem_simd);
8454 }
8455 switch (argExpr->op) {
8456 case OP_dread: {
8457 DreadNode *dNode = static_cast<DreadNode *>(argExpr);
8458 MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(dNode->GetStIdx());
8459 if (dNode->GetFieldID() != 0) {
8460 MIRStructType *structType = static_cast<MIRStructType *>(symbol->GetType());
8461 DEBUG_ASSERT(structType != nullptr, "SelectParmList: non-zero fieldID for non-structure");
8462 FieldAttrs fa = structType->GetFieldAttrs(dNode->GetFieldID());
8463 is64x1vec = fa.GetAttr(FLDATTR_oneelem_simd);
8464 } else {
8465 is64x1vec = symbol->GetAttr(ATTR_oneelem_simd);
8466 }
8467 break;
8468 }
8469 case OP_iread: {
8470 IreadNode *iNode = static_cast<IreadNode *>(argExpr);
8471 MIRType *type = GlobalTables::GetTypeTable().GetTypeFromTyIdx(iNode->GetTyIdx());
8472 MIRPtrType *ptrTyp = static_cast<MIRPtrType *>(type);
8473 DEBUG_ASSERT(ptrTyp != nullptr, "expect a pointer type at iread node");
8474 MIRType *pointedTy = GlobalTables::GetTypeTable().GetTypeFromTyIdx(ptrTyp->GetPointedTyIdx());
8475 if (iNode->GetFieldID() != 0) {
8476 MIRStructType *structType = static_cast<MIRStructType *>(pointedTy);
8477 FieldAttrs fa = structType->GetFieldAttrs(iNode->GetFieldID());
8478 is64x1vec = fa.GetAttr(FLDATTR_oneelem_simd);
8479 } else {
8480 TypeAttrs ta = static_cast<MIRPtrType *>(ptrTyp)->GetTypeAttrs();
8481 is64x1vec = ta.GetAttr(ATTR_oneelem_simd);
8482 }
8483 break;
8484 }
8485 case OP_constval: {
8486 CallNode *call = safe_cast<CallNode>(&naryNode);
8487 if (call == nullptr) {
8488 break;
8489 }
8490 MIRFunction *fn = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(call->GetPUIdx());
8491 if (fn == nullptr || fn->GetFormalCount() == 0 || fn->GetFormalCount() <= pnum) {
8492 break;
8493 }
8494 is64x1vec = fn->GetFormalDefAt(pnum).formalAttrs.GetAttr(ATTR_oneelem_simd);
8495 break;
8496 }
8497 default:
8498 break;
8499 }
8500 /* use alloca */
8501 if (primType == PTY_agg) {
8502 SelectParmListForAggregate(*argExpr, srcOpnds, parmLocator, structCopyOffset);
8503 continue;
8504 }
8505 ty = GlobalTables::GetTypeTable().GetTypeTable()[static_cast<uint32>(primType)];
8506 RegOperand *expRegOpnd = nullptr;
8507 Operand *opnd = HandleExpr(naryNode, *argExpr);
8508 if (opnd->GetKind() == Operand::kOpdRegister && static_cast<RegOperand *>(opnd)->GetIF64Vec()) {
8509 is64x1vec = true;
8510 }
8511 if (!opnd->IsRegister()) {
8512 opnd = &LoadIntoRegister(*opnd, primType);
8513 }
8514 expRegOpnd = static_cast<RegOperand *>(opnd);
8515
8516 if ((pnum == 0) && firstArgReturn) {
8517 parmLocator.InitCCLocInfo(ploc);
8518 ploc.reg0 = R8;
8519 } else {
8520 parmLocator.LocateNextParm(*ty, ploc);
8521 }
8522 /* is64x1vec should be an int64 value in an FP/simd reg for ABI compliance,
8523 convert R-reg to equivalent V-reg */
8524 PrimType destPrimType = primType;
8525 if (is64x1vec && ploc.reg0 != kRinvalid && ploc.reg0 < R7) {
8526 ploc.reg0 = AArch64Abi::floatParmRegs[static_cast<int>(ploc.reg0) - 1];
8527 destPrimType = PTY_f64;
8528 }
8529
8530 /* skip unused args */
8531 if (callee != nullptr && callee->GetFuncDesc().IsArgUnused(pnum))
8532 continue;
8533
8534 if (ploc.reg0 != kRinvalid) { /* load to the register. */
8535 CHECK_FATAL(expRegOpnd != nullptr, "null ptr check");
8536 RegOperand &parmRegOpnd = GetOrCreatePhysicalRegisterOperand(
8537 static_cast<AArch64reg>(ploc.reg0), expRegOpnd->GetSize(), GetRegTyFromPrimTy(destPrimType));
8538 SelectCopy(parmRegOpnd, destPrimType, *expRegOpnd, primType);
8539 srcOpnds.PushOpnd(parmRegOpnd);
8540 } else { /* store to the memory segment for stack-passsed arguments. */
8541 if (CGOptions::IsBigEndian()) {
8542 if (GetPrimTypeBitSize(primType) < k64BitSize) {
8543 ploc.memOffset = ploc.memOffset + static_cast<int32>(k4BitSize);
8544 }
8545 }
8546 MemOperand &actMemOpnd = CreateMemOpnd(RSP, ploc.memOffset, GetPrimTypeBitSize(primType));
8547 Insn &strInsn = GetInsnBuilder()->BuildInsn(PickStInsn(GetPrimTypeBitSize(primType), primType), *expRegOpnd,
8548 actMemOpnd);
8549 actMemOpnd.SetStackArgMem(true);
8550 if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel1 && stackArgsCount < kShiftAmount12) {
8551 (void)insnForStackArgs.emplace_back(&strInsn);
8552 stackArgsCount++;
8553 } else {
8554 GetCurBB()->AppendInsn(strInsn);
8555 }
8556 }
8557 DEBUG_ASSERT(ploc.reg1 == 0, "SelectCall NYI");
8558 }
8559 if (specialArg) {
8560 DEBUG_ASSERT(tmpBB, "need temp bb for lower priority args");
8561 curBBrecord->InsertAtEnd(*tmpBB);
8562 SetCurBB(*curBBrecord);
8563 }
8564 for (auto &strInsn : insnForStackArgs) {
8565 GetCurBB()->AppendInsn(*strInsn);
8566 }
8567 }
8568
SelectParmListNotC(StmtNode & naryNode,ListOperand & srcOpnds)8569 void AArch64CGFunc::SelectParmListNotC(StmtNode &naryNode, ListOperand &srcOpnds)
8570 {
8571 size_t i = 0;
8572 if (naryNode.GetOpCode() == OP_icall || naryNode.GetOpCode() == OP_icallproto) {
8573 i++;
8574 }
8575
8576 CCImpl &parmLocator = *GetOrCreateLocator(CCImpl::GetCallConvKind(naryNode));
8577 CCLocInfo ploc;
8578 std::vector<Insn *> insnForStackArgs;
8579 uint32 stackArgsCount = 0;
8580 for (uint32 pnum = 0; i < naryNode.NumOpnds(); ++i, ++pnum) {
8581 MIRType *ty = nullptr;
8582 BaseNode *argExpr = naryNode.Opnd(i);
8583 PrimType primType = argExpr->GetPrimType();
8584 DEBUG_ASSERT(primType != PTY_void, "primType should not be void");
8585 /* use alloca */
8586 ty = GlobalTables::GetTypeTable().GetTypeTable()[static_cast<uint32>(primType)];
8587 RegOperand *expRegOpnd = nullptr;
8588 Operand *opnd = HandleExpr(naryNode, *argExpr);
8589 if (!opnd->IsRegister()) {
8590 opnd = &LoadIntoRegister(*opnd, primType);
8591 }
8592 expRegOpnd = static_cast<RegOperand *>(opnd);
8593
8594 parmLocator.LocateNextParm(*ty, ploc);
8595 PrimType destPrimType = primType;
8596 if (ploc.reg0 != kRinvalid) { /* load to the register. */
8597 CHECK_FATAL(expRegOpnd != nullptr, "null ptr check");
8598 RegOperand &parmRegOpnd = GetOrCreatePhysicalRegisterOperand(
8599 static_cast<AArch64reg>(ploc.reg0), expRegOpnd->GetSize(), GetRegTyFromPrimTy(destPrimType));
8600 SelectCopy(parmRegOpnd, destPrimType, *expRegOpnd, primType);
8601 srcOpnds.PushOpnd(parmRegOpnd);
8602 } else { /* store to the memory segment for stack-passsed arguments. */
8603 if (CGOptions::IsBigEndian()) {
8604 if (GetPrimTypeBitSize(primType) < k64BitSize) {
8605 ploc.memOffset = ploc.memOffset + static_cast<int32>(k4BitSize);
8606 }
8607 }
8608 MemOperand &actMemOpnd = CreateMemOpnd(RSP, ploc.memOffset, GetPrimTypeBitSize(primType));
8609 Insn &strInsn = GetInsnBuilder()->BuildInsn(PickStInsn(GetPrimTypeBitSize(primType), primType), *expRegOpnd,
8610 actMemOpnd);
8611 actMemOpnd.SetStackArgMem(true);
8612 if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel1 && stackArgsCount < kShiftAmount12) {
8613 (void)insnForStackArgs.emplace_back(&strInsn);
8614 stackArgsCount++;
8615 } else {
8616 GetCurBB()->AppendInsn(strInsn);
8617 }
8618 }
8619 DEBUG_ASSERT(ploc.reg1 == 0, "SelectCall NYI");
8620 }
8621 for (auto &strInsn : insnForStackArgs) {
8622 GetCurBB()->AppendInsn(*strInsn);
8623 }
8624 }
8625
8626 // based on call conv, choose how to prepare args
SelectParmListWrapper(StmtNode & naryNode,ListOperand & srcOpnds,bool isCallNative)8627 void AArch64CGFunc::SelectParmListWrapper(StmtNode &naryNode, ListOperand &srcOpnds, bool isCallNative)
8628 {
8629 if (CCImpl::GetCallConvKind(naryNode) == kCCall) {
8630 SelectParmList(naryNode, srcOpnds, isCallNative);
8631 } else if (CCImpl::GetCallConvKind(naryNode) == kWebKitJS || CCImpl::GetCallConvKind(naryNode) == kGHC) {
8632 SelectParmListNotC(naryNode, srcOpnds);
8633 } else {
8634 CHECK_FATAL(false, "niy");
8635 }
8636 }
8637 /*
8638 * for MCC_DecRefResetPair(addrof ptr %Reg17_R5592, addrof ptr %Reg16_R6202) or
8639 * MCC_ClearLocalStackRef(addrof ptr %Reg17_R5592), the parameter (addrof ptr xxx) is converted to asm as follow:
8640 * add vreg, x29, #imm
8641 * mov R0/R1, vreg
8642 * this function is used to prepare parameters, the generated vreg is returned, and #imm is saved in offsetValue.
8643 */
SelectClearStackCallParam(const AddrofNode & expr,int64 & offsetValue)8644 Operand *AArch64CGFunc::SelectClearStackCallParam(const AddrofNode &expr, int64 &offsetValue)
8645 {
8646 MIRSymbol *symbol = GetMirModule().CurFunction()->GetLocalOrGlobalSymbol(expr.GetStIdx());
8647 PrimType ptype = expr.GetPrimType();
8648 regno_t vRegNO = NewVReg(kRegTyInt, GetPrimTypeSize(ptype));
8649 Operand &result = CreateVirtualRegisterOperand(vRegNO);
8650 CHECK_FATAL(expr.GetFieldID() == 0, "the fieldID of parameter in clear stack reference call must be 0");
8651 if (!CGOptions::IsQuiet()) {
8652 maple::LogInfo::MapleLogger(kLlErr)
8653 << "Warning: we expect AddrOf with StImmOperand is not used for local variables";
8654 }
8655 auto *symLoc = static_cast<AArch64SymbolAlloc *>(GetMemlayout()->GetSymAllocInfo(symbol->GetStIndex()));
8656 ImmOperand *offset = nullptr;
8657 if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsArgsStkPassed) {
8658 offset = &CreateImmOperand(GetBaseOffset(*symLoc), k64BitSize, false, kUnAdjustVary);
8659 } else if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsRefLocals) {
8660 auto it = immOpndsRequiringOffsetAdjustmentForRefloc.find(symLoc);
8661 if (it != immOpndsRequiringOffsetAdjustmentForRefloc.end()) {
8662 offset = (*it).second;
8663 } else {
8664 offset = &CreateImmOperand(GetBaseOffset(*symLoc), k64BitSize, false);
8665 immOpndsRequiringOffsetAdjustmentForRefloc[symLoc] = offset;
8666 }
8667 } else {
8668 CHECK_FATAL(false, "the symLoc of parameter in clear stack reference call is unreasonable");
8669 }
8670 DEBUG_ASSERT(offset != nullptr, "offset should not be nullptr");
8671 offsetValue = offset->GetValue();
8672 SelectAdd(result, *GetBaseReg(*symLoc), *offset, PTY_u64);
8673 if (GetCG()->GenerateVerboseCG()) {
8674 /* Add a comment */
8675 Insn *insn = GetCurBB()->GetLastInsn();
8676 std::string comm = "local/formal var: ";
8677 comm += symbol->GetName();
8678 insn->SetComment(comm);
8679 }
8680 return &result;
8681 }
8682
8683 /* select paramters for MCC_DecRefResetPair and MCC_ClearLocalStackRef function */
SelectClearStackCallParmList(const StmtNode & naryNode,ListOperand & srcOpnds,std::vector<int64> & stackPostion)8684 void AArch64CGFunc::SelectClearStackCallParmList(const StmtNode &naryNode, ListOperand &srcOpnds,
8685 std::vector<int64> &stackPostion)
8686 {
8687 CHECK_FATAL(false, "should not go here");
8688 AArch64CallConvImpl parmLocator(GetBecommon());
8689 CCLocInfo ploc;
8690 for (size_t i = 0; i < naryNode.NumOpnds(); ++i) {
8691 MIRType *ty = nullptr;
8692 BaseNode *argExpr = naryNode.Opnd(i);
8693 PrimType primType = argExpr->GetPrimType();
8694 DEBUG_ASSERT(primType != PTY_void, "primType check");
8695 /* use alloc */
8696 CHECK_FATAL(primType != PTY_agg, "the type of argument is unreasonable");
8697 ty = GlobalTables::GetTypeTable().GetTypeTable()[static_cast<uint32>(primType)];
8698 CHECK_FATAL(argExpr->GetOpCode() == OP_addrof, "the argument of clear stack call is unreasonable");
8699 auto *expr = static_cast<AddrofNode *>(argExpr);
8700 int64 offsetValue = 0;
8701 Operand *opnd = SelectClearStackCallParam(*expr, offsetValue);
8702 stackPostion.emplace_back(offsetValue);
8703 auto *expRegOpnd = static_cast<RegOperand *>(opnd);
8704 CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
8705 parmLocator.LocateNextParm(*ty, ploc);
8706 CHECK_FATAL(ploc.reg0 != 0, "the parameter of ClearStackCall must be passed by register");
8707 CHECK_FATAL(expRegOpnd != nullptr, "null ptr check");
8708 RegOperand &parmRegOpnd = GetOrCreatePhysicalRegisterOperand(
8709 static_cast<AArch64reg>(ploc.reg0), expRegOpnd->GetSize(), GetRegTyFromPrimTy(primType));
8710 SelectCopy(parmRegOpnd, primType, *expRegOpnd, primType);
8711 srcOpnds.PushOpnd(parmRegOpnd);
8712 DEBUG_ASSERT(ploc.reg1 == 0, "SelectCall NYI");
8713 }
8714 }
8715
8716 /*
8717 * intrinsify Unsafe.getAndAddInt and Unsafe.getAndAddLong
8718 * generate an intrinsic instruction instead of a function call
8719 * intrinsic_get_add_int w0, xt, ws, ws, x1, x2, w3, label
8720 */
IntrinsifyGetAndAddInt(ListOperand & srcOpnds,PrimType pty)8721 void AArch64CGFunc::IntrinsifyGetAndAddInt(ListOperand &srcOpnds, PrimType pty)
8722 {
8723 MapleList<RegOperand *> &opnds = srcOpnds.GetOperands();
8724 /* Unsafe.getAndAddInt has more than 4 parameters */
8725 DEBUG_ASSERT(opnds.size() >= 4, "ensure the operands number");
8726 auto iter = opnds.begin();
8727 RegOperand *objOpnd = *(++iter);
8728 RegOperand *offOpnd = *(++iter);
8729 RegOperand *deltaOpnd = *(++iter);
8730 auto &retVal = static_cast<RegOperand &>(GetTargetRetOperand(pty, -1));
8731 LabelIdx labIdx = CreateLabel();
8732 LabelOperand &targetOpnd = GetOrCreateLabelOperand(labIdx);
8733 RegOperand &tempOpnd0 = CreateRegisterOperandOfType(PTY_i64);
8734 RegOperand &tempOpnd1 = CreateRegisterOperandOfType(pty);
8735 RegOperand &tempOpnd2 = CreateRegisterOperandOfType(PTY_i32);
8736 MOperator mOp = (pty == PTY_i64) ? MOP_get_and_addL : MOP_get_and_addI;
8737 std::vector<Operand *> intrnOpnds;
8738 intrnOpnds.emplace_back(&retVal);
8739 intrnOpnds.emplace_back(&tempOpnd0);
8740 intrnOpnds.emplace_back(&tempOpnd1);
8741 intrnOpnds.emplace_back(&tempOpnd2);
8742 intrnOpnds.emplace_back(objOpnd);
8743 intrnOpnds.emplace_back(offOpnd);
8744 intrnOpnds.emplace_back(deltaOpnd);
8745 intrnOpnds.emplace_back(&targetOpnd);
8746 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, intrnOpnds));
8747 }
8748
8749 /*
8750 * intrinsify Unsafe.getAndSetInt and Unsafe.getAndSetLong
8751 * generate an intrinsic instruction instead of a function call
8752 */
IntrinsifyGetAndSetInt(ListOperand & srcOpnds,PrimType pty)8753 void AArch64CGFunc::IntrinsifyGetAndSetInt(ListOperand &srcOpnds, PrimType pty)
8754 {
8755 MapleList<RegOperand *> &opnds = srcOpnds.GetOperands();
8756 /* Unsafe.getAndSetInt has 4 parameters */
8757 DEBUG_ASSERT(opnds.size() == 4, "ensure the operands number");
8758 auto iter = opnds.begin();
8759 RegOperand *objOpnd = *(++iter);
8760 RegOperand *offOpnd = *(++iter);
8761 RegOperand *newValueOpnd = *(++iter);
8762 auto &retVal = static_cast<RegOperand &>(GetTargetRetOperand(pty, -1));
8763 LabelIdx labIdx = CreateLabel();
8764 LabelOperand &targetOpnd = GetOrCreateLabelOperand(labIdx);
8765 RegOperand &tempOpnd0 = CreateRegisterOperandOfType(PTY_i64);
8766 RegOperand &tempOpnd1 = CreateRegisterOperandOfType(PTY_i32);
8767
8768 MOperator mOp = (pty == PTY_i64) ? MOP_get_and_setL : MOP_get_and_setI;
8769 std::vector<Operand *> intrnOpnds;
8770 intrnOpnds.emplace_back(&retVal);
8771 intrnOpnds.emplace_back(&tempOpnd0);
8772 intrnOpnds.emplace_back(&tempOpnd1);
8773 intrnOpnds.emplace_back(objOpnd);
8774 intrnOpnds.emplace_back(offOpnd);
8775 intrnOpnds.emplace_back(newValueOpnd);
8776 intrnOpnds.emplace_back(&targetOpnd);
8777 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, intrnOpnds));
8778 }
8779
8780 /*
8781 * intrinsify Unsafe.compareAndSwapInt and Unsafe.compareAndSwapLong
8782 * generate an intrinsic instruction instead of a function call
8783 */
IntrinsifyCompareAndSwapInt(ListOperand & srcOpnds,PrimType pty)8784 void AArch64CGFunc::IntrinsifyCompareAndSwapInt(ListOperand &srcOpnds, PrimType pty)
8785 {
8786 MapleList<RegOperand *> &opnds = srcOpnds.GetOperands();
8787 /* Unsafe.compareAndSwapInt has more than 5 parameters */
8788 DEBUG_ASSERT(opnds.size() >= 5, "ensure the operands number");
8789 auto iter = opnds.begin();
8790 RegOperand *objOpnd = *(++iter);
8791 RegOperand *offOpnd = *(++iter);
8792 RegOperand *expectedValueOpnd = *(++iter);
8793 RegOperand *newValueOpnd = *(++iter);
8794 auto &retVal = static_cast<RegOperand &>(GetTargetRetOperand(PTY_i64, -1));
8795 RegOperand &tempOpnd0 = CreateRegisterOperandOfType(PTY_i64);
8796 RegOperand &tempOpnd1 = CreateRegisterOperandOfType(pty);
8797 LabelIdx labIdx1 = CreateLabel();
8798 LabelOperand &label1Opnd = GetOrCreateLabelOperand(labIdx1);
8799 LabelIdx labIdx2 = CreateLabel();
8800 LabelOperand &label2Opnd = GetOrCreateLabelOperand(labIdx2);
8801 MOperator mOp = (pty == PTY_i32) ? MOP_compare_and_swapI : MOP_compare_and_swapL;
8802 std::vector<Operand *> intrnOpnds;
8803 intrnOpnds.emplace_back(&retVal);
8804 intrnOpnds.emplace_back(&tempOpnd0);
8805 intrnOpnds.emplace_back(&tempOpnd1);
8806 intrnOpnds.emplace_back(objOpnd);
8807 intrnOpnds.emplace_back(offOpnd);
8808 intrnOpnds.emplace_back(expectedValueOpnd);
8809 intrnOpnds.emplace_back(newValueOpnd);
8810 intrnOpnds.emplace_back(&label1Opnd);
8811 intrnOpnds.emplace_back(&label2Opnd);
8812 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, intrnOpnds));
8813 }
8814
8815 /*
8816 * the lowest bit of count field is used to indicate whether or not the string is compressed
8817 * if the string is not compressed, jump to jumpLabIdx
8818 */
CheckStringIsCompressed(BB & bb,RegOperand & str,int32 countOffset,PrimType countPty,LabelIdx jumpLabIdx)8819 RegOperand *AArch64CGFunc::CheckStringIsCompressed(BB &bb, RegOperand &str, int32 countOffset, PrimType countPty,
8820 LabelIdx jumpLabIdx)
8821 {
8822 MemOperand &memOpnd = CreateMemOpnd(str, countOffset, str.GetSize());
8823 uint32 bitSize = GetPrimTypeBitSize(countPty);
8824 MOperator loadOp = PickLdInsn(bitSize, countPty);
8825 RegOperand &countOpnd = CreateRegisterOperandOfType(countPty);
8826 bb.AppendInsn(GetInsnBuilder()->BuildInsn(loadOp, countOpnd, memOpnd));
8827 ImmOperand &immValueOne = CreateImmOperand(countPty, 1);
8828 RegOperand &countLowestBitOpnd = CreateRegisterOperandOfType(countPty);
8829 MOperator andOp = bitSize == k64BitSize ? MOP_xandrri13 : MOP_wandrri12;
8830 bb.AppendInsn(GetInsnBuilder()->BuildInsn(andOp, countLowestBitOpnd, countOpnd, immValueOne));
8831 RegOperand &wzr = GetZeroOpnd(bitSize);
8832 MOperator cmpOp = (bitSize == k64BitSize) ? MOP_xcmprr : MOP_wcmprr;
8833 Operand &rflag = GetOrCreateRflag();
8834 bb.AppendInsn(GetInsnBuilder()->BuildInsn(cmpOp, rflag, wzr, countLowestBitOpnd));
8835 bb.AppendInsn(GetInsnBuilder()->BuildInsn(MOP_beq, rflag, GetOrCreateLabelOperand(jumpLabIdx)));
8836 bb.SetKind(BB::kBBIf);
8837 return &countOpnd;
8838 }
8839
8840 /*
8841 * count field stores the length shifted one bit to the left
8842 * if the length is less than eight, jump to jumpLabIdx
8843 */
CheckStringLengthLessThanEight(BB & bb,RegOperand & countOpnd,PrimType countPty,LabelIdx jumpLabIdx)8844 RegOperand *AArch64CGFunc::CheckStringLengthLessThanEight(BB &bb, RegOperand &countOpnd, PrimType countPty,
8845 LabelIdx jumpLabIdx)
8846 {
8847 RegOperand &lengthOpnd = CreateRegisterOperandOfType(countPty);
8848 uint32 bitSize = GetPrimTypeBitSize(countPty);
8849 MOperator lsrOp = (bitSize == k64BitSize) ? MOP_xlsrrri6 : MOP_wlsrrri5;
8850 ImmOperand &immValueOne = CreateImmOperand(countPty, 1);
8851 bb.AppendInsn(GetInsnBuilder()->BuildInsn(lsrOp, lengthOpnd, countOpnd, immValueOne));
8852 constexpr int kConstIntEight = 8;
8853 ImmOperand &immValueEight = CreateImmOperand(countPty, kConstIntEight);
8854 MOperator cmpImmOp = (bitSize == k64BitSize) ? MOP_xcmpri : MOP_wcmpri;
8855 Operand &rflag = GetOrCreateRflag();
8856 bb.AppendInsn(GetInsnBuilder()->BuildInsn(cmpImmOp, rflag, lengthOpnd, immValueEight));
8857 bb.AppendInsn(GetInsnBuilder()->BuildInsn(MOP_blt, rflag, GetOrCreateLabelOperand(jumpLabIdx)));
8858 bb.SetKind(BB::kBBIf);
8859 return &lengthOpnd;
8860 }
8861
GenerateIntrnInsnForStrIndexOf(BB & bb,RegOperand & srcString,RegOperand & patternString,RegOperand & srcCountOpnd,RegOperand & patternLengthOpnd,PrimType countPty,LabelIdx jumpLabIdx)8862 void AArch64CGFunc::GenerateIntrnInsnForStrIndexOf(BB &bb, RegOperand &srcString, RegOperand &patternString,
8863 RegOperand &srcCountOpnd, RegOperand &patternLengthOpnd,
8864 PrimType countPty, LabelIdx jumpLabIdx)
8865 {
8866 RegOperand &srcLengthOpnd = CreateRegisterOperandOfType(countPty);
8867 ImmOperand &immValueOne = CreateImmOperand(countPty, 1);
8868 uint32 bitSize = GetPrimTypeBitSize(countPty);
8869 MOperator lsrOp = (bitSize == k64BitSize) ? MOP_xlsrrri6 : MOP_wlsrrri5;
8870 bb.AppendInsn(GetInsnBuilder()->BuildInsn(lsrOp, srcLengthOpnd, srcCountOpnd, immValueOne));
8871 #ifdef USE_32BIT_REF
8872 const int64 stringBaseObjSize = 16; /* shadow(4)+monitor(4)+count(4)+hash(4) */
8873 #else
8874 const int64 stringBaseObjSize = 20; /* shadow(8)+monitor(4)+count(4)+hash(4) */
8875 #endif /* USE_32BIT_REF */
8876 PrimType pty = (srcString.GetSize() == k64BitSize) ? PTY_i64 : PTY_i32;
8877 ImmOperand &immStringBaseOffset = CreateImmOperand(pty, stringBaseObjSize);
8878 MOperator addOp = (pty == PTY_i64) ? MOP_xaddrri12 : MOP_waddrri12;
8879 RegOperand &srcStringBaseOpnd = CreateRegisterOperandOfType(pty);
8880 bb.AppendInsn(GetInsnBuilder()->BuildInsn(addOp, srcStringBaseOpnd, srcString, immStringBaseOffset));
8881 RegOperand &patternStringBaseOpnd = CreateRegisterOperandOfType(pty);
8882 bb.AppendInsn(GetInsnBuilder()->BuildInsn(addOp, patternStringBaseOpnd, patternString, immStringBaseOffset));
8883 auto &retVal = static_cast<RegOperand &>(GetTargetRetOperand(PTY_i32, -1));
8884 std::vector<Operand *> intrnOpnds;
8885 intrnOpnds.emplace_back(&retVal);
8886 intrnOpnds.emplace_back(&srcStringBaseOpnd);
8887 intrnOpnds.emplace_back(&srcLengthOpnd);
8888 intrnOpnds.emplace_back(&patternStringBaseOpnd);
8889 intrnOpnds.emplace_back(&patternLengthOpnd);
8890 const uint32 tmpRegOperandNum = 6;
8891 for (uint32 i = 0; i < tmpRegOperandNum - 1; ++i) {
8892 RegOperand &tmpOpnd = CreateRegisterOperandOfType(PTY_i64);
8893 intrnOpnds.emplace_back(&tmpOpnd);
8894 }
8895 intrnOpnds.emplace_back(&CreateRegisterOperandOfType(PTY_i32));
8896 const uint32 labelNum = 7;
8897 for (uint32 i = 0; i < labelNum; ++i) {
8898 LabelIdx labIdx = CreateLabel();
8899 LabelOperand &labelOpnd = GetOrCreateLabelOperand(labIdx);
8900 intrnOpnds.emplace_back(&labelOpnd);
8901 }
8902 bb.AppendInsn(GetInsnBuilder()->BuildInsn(MOP_string_indexof, intrnOpnds));
8903 bb.AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xuncond, GetOrCreateLabelOperand(jumpLabIdx)));
8904 bb.SetKind(BB::kBBGoto);
8905 }
8906
8907 /*
8908 * intrinsify String.indexOf
8909 * generate an intrinsic instruction instead of a function call if both the source string and the specified substring
8910 * are compressed and the length of the substring is not less than 8, i.e.
8911 * bl String.indexOf, srcString, patternString ===>>
8912 *
8913 * ldr srcCountOpnd, [srcString, offset]
8914 * and srcCountLowestBitOpnd, srcCountOpnd, #1
8915 * cmp wzr, srcCountLowestBitOpnd
8916 * beq Label.call
8917 * ldr patternCountOpnd, [patternString, offset]
8918 * and patternCountLowestBitOpnd, patternCountOpnd, #1
8919 * cmp wzr, patternCountLowestBitOpnd
8920 * beq Label.call
8921 * lsr patternLengthOpnd, patternCountOpnd, #1
8922 * cmp patternLengthOpnd, #8
8923 * blt Label.call
8924 * lsr srcLengthOpnd, srcCountOpnd, #1
8925 * add srcStringBaseOpnd, srcString, immStringBaseOffset
8926 * add patternStringBaseOpnd, patternString, immStringBaseOffset
8927 * intrinsic_string_indexof retVal, srcStringBaseOpnd, srcLengthOpnd, patternStringBaseOpnd, patternLengthOpnd,
8928 * tmpOpnd1, tmpOpnd2, tmpOpnd3, tmpOpnd4, tmpOpnd5, tmpOpnd6,
8929 * label1, label2, label3, lable3, label4, label5, label6, label7
8930 * b Label.joint
8931 * Label.call:
8932 * bl String.indexOf, srcString, patternString
8933 * Label.joint:
8934 */
IntrinsifyStringIndexOf(ListOperand & srcOpnds,const MIRSymbol & funcSym)8935 void AArch64CGFunc::IntrinsifyStringIndexOf(ListOperand &srcOpnds, const MIRSymbol &funcSym)
8936 {
8937 MapleList<RegOperand *> &opnds = srcOpnds.GetOperands();
8938 /* String.indexOf opnd size must be more than 2 */
8939 DEBUG_ASSERT(opnds.size() >= 2, "ensure the operands number");
8940 auto iter = opnds.begin();
8941 RegOperand *srcString = *iter;
8942 RegOperand *patternString = *(++iter);
8943 GStrIdx gStrIdx = GlobalTables::GetStrTable().GetStrIdxFromName(namemangler::kJavaLangStringStr);
8944 MIRType *type =
8945 GlobalTables::GetTypeTable().GetTypeFromTyIdx(GlobalTables::GetTypeNameTable().GetTyIdxFromGStrIdx(gStrIdx));
8946 auto stringType = static_cast<MIRStructType *>(type);
8947 CHECK_FATAL(stringType != nullptr, "Ljava_2Flang_2FString_3B type can not be null");
8948 FieldID fieldID = GetMirModule().GetMIRBuilder()->GetStructFieldIDFromFieldNameParentFirst(stringType, "count");
8949 MIRType *fieldType = stringType->GetFieldType(fieldID);
8950 PrimType countPty = fieldType->GetPrimType();
8951 int32 offset = GetBecommon().GetFieldOffset(*stringType, fieldID).first;
8952 LabelIdx callBBLabIdx = CreateLabel();
8953 RegOperand *srcCountOpnd = CheckStringIsCompressed(*GetCurBB(), *srcString, offset, countPty, callBBLabIdx);
8954
8955 BB *srcCompressedBB = CreateNewBB();
8956 GetCurBB()->AppendBB(*srcCompressedBB);
8957 RegOperand *patternCountOpnd =
8958 CheckStringIsCompressed(*srcCompressedBB, *patternString, offset, countPty, callBBLabIdx);
8959
8960 BB *patternCompressedBB = CreateNewBB();
8961 RegOperand *patternLengthOpnd =
8962 CheckStringLengthLessThanEight(*patternCompressedBB, *patternCountOpnd, countPty, callBBLabIdx);
8963
8964 BB *intrinsicBB = CreateNewBB();
8965 LabelIdx jointLabIdx = CreateLabel();
8966 GenerateIntrnInsnForStrIndexOf(*intrinsicBB, *srcString, *patternString, *srcCountOpnd, *patternLengthOpnd,
8967 countPty, jointLabIdx);
8968
8969 BB *callBB = CreateNewBB();
8970 callBB->AddLabel(callBBLabIdx);
8971 SetLab2BBMap(callBBLabIdx, *callBB);
8972 SetCurBB(*callBB);
8973 Insn &callInsn = AppendCall(funcSym, srcOpnds);
8974 MIRType *retType = funcSym.GetFunction()->GetReturnType();
8975 if (retType != nullptr) {
8976 callInsn.SetRetSize(static_cast<uint32>(retType->GetSize()));
8977 }
8978 GetFunction().SetHasCall();
8979
8980 BB *jointBB = CreateNewBB();
8981 jointBB->AddLabel(jointLabIdx);
8982 SetLab2BBMap(jointLabIdx, *jointBB);
8983 srcCompressedBB->AppendBB(*patternCompressedBB);
8984 patternCompressedBB->AppendBB(*intrinsicBB);
8985 intrinsicBB->AppendBB(*callBB);
8986 callBB->AppendBB(*jointBB);
8987 SetCurBB(*jointBB);
8988 }
8989
8990 /* Lmbc calls have no argument, they are all explicit iassignspoff or
8991 blkassign. Info collected and to be emitted here */
LmbcSelectParmList(ListOperand * srcOpnds,bool isArgReturn)8992 void AArch64CGFunc::LmbcSelectParmList(ListOperand *srcOpnds, bool isArgReturn)
8993 {
8994 if (GetLmbcArgInfo() == nullptr) {
8995 return; /* no arg */
8996 }
8997 CHECK_FATAL(GetMirModule().GetFlavor() == MIRFlavor::kFlavorLmbc, "To be called for Lmbc model only");
8998 MapleVector<RegOperand *> &args = GetLmbcCallArgs();
8999 MapleVector<PrimType> &types = GetLmbcCallArgTypes();
9000 MapleVector<int32> &offsets = GetLmbcCallArgOffsets();
9001 MapleVector<int32> ®s = GetLmbcCallArgNumOfRegs();
9002 int iCnt = 0;
9003 int fCnt = 0;
9004 for (size_t i = isArgReturn ? 1 : 0; i < args.size(); i++) {
9005 RegType ty = args[i]->GetRegisterType();
9006 PrimType pTy = types[i];
9007 AArch64reg reg;
9008 if (args[i]->IsOfIntClass() && (iCnt + regs[i]) <= static_cast<int32>(k8ByteSize)) {
9009 reg = static_cast<AArch64reg>(R0 + iCnt++);
9010 RegOperand *res = &GetOrCreatePhysicalRegisterOperand(reg, GetPrimTypeSize(pTy) * kBitsPerByte, ty);
9011 SelectCopy(*res, pTy, *args[i], pTy);
9012 srcOpnds->PushOpnd(*res);
9013 } else if (!args[i]->IsOfIntClass() && (fCnt + regs[i]) <= static_cast<int32>(k8ByteSize)) {
9014 reg = static_cast<AArch64reg>(V0 + fCnt++);
9015 RegOperand *res = &GetOrCreatePhysicalRegisterOperand(reg, GetPrimTypeSize(pTy) * kBitsPerByte, ty);
9016 SelectCopy(*res, pTy, *args[i], pTy);
9017 srcOpnds->PushOpnd(*res);
9018 } else {
9019 uint32 pSize = GetPrimTypeSize(pTy);
9020 Operand &memOpd = CreateMemOpnd(RSP, offsets[i], pSize);
9021 GetCurBB()->AppendInsn(
9022 GetInsnBuilder()->BuildInsn(PickStInsn(pSize * kBitsPerByte, pTy), *args[i], memOpd));
9023 }
9024 }
9025 /* Load x8 if 1st arg is for agg return */
9026 if (isArgReturn) {
9027 AArch64reg reg = static_cast<AArch64reg>(R8);
9028 RegOperand *res = &GetOrCreatePhysicalRegisterOperand(reg, GetPrimTypeSize(PTY_a64) * kBitsPerByte, kRegTyInt);
9029 SelectCopy(*res, PTY_a64, *args[0], PTY_a64);
9030 srcOpnds->PushOpnd(*res);
9031 }
9032 ResetLmbcArgInfo(); /* reset */
9033 ResetLmbcArgsInRegs();
9034 ResetLmbcTotalArgs();
9035 }
9036
SelectCall(CallNode & callNode)9037 void AArch64CGFunc::SelectCall(CallNode &callNode)
9038 {
9039 MIRFunction *fn = GlobalTables::GetFunctionTable().GetFunctionFromPuidx(callNode.GetPUIdx());
9040 MIRSymbol *fsym = GetFunction().GetLocalOrGlobalSymbol(fn->GetStIdx(), false);
9041 MIRType *retType = fn->GetReturnType();
9042
9043 if (GetCG()->GenerateVerboseCG()) {
9044 const std::string &comment = fsym->GetName();
9045 GetCurBB()->AppendInsn(CreateCommentInsn(comment));
9046 }
9047
9048 ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
9049 if (GetMirModule().GetFlavor() == MIRFlavor::kFlavorLmbc) {
9050 SetLmbcCallReturnType(nullptr);
9051 bool largeStructRet = false;
9052 if (fn->IsFirstArgReturn()) {
9053 MIRPtrType *ptrTy = static_cast<MIRPtrType *>(
9054 GlobalTables::GetTypeTable().GetTypeFromTyIdx(fn->GetFormalDefVec()[0].formalTyIdx));
9055 MIRType *sTy = GlobalTables::GetTypeTable().GetTypeFromTyIdx(ptrTy->GetPointedTyIdx());
9056 largeStructRet = sTy->GetSize() > k16ByteSize;
9057 SetLmbcCallReturnType(sTy);
9058 } else {
9059 MIRType *ty = fn->GetReturnType();
9060 SetLmbcCallReturnType(ty);
9061 }
9062 LmbcSelectParmList(srcOpnds, largeStructRet);
9063 }
9064
9065 SelectParmListWrapper(callNode, *srcOpnds, false);
9066
9067 const std::string &funcName = fsym->GetName();
9068 if (Globals::GetInstance()->GetOptimLevel() >= CGOptions::kLevel2 &&
9069 funcName == "Ljava_2Flang_2FString_3B_7CindexOf_7C_28Ljava_2Flang_2FString_3B_29I") {
9070 GStrIdx strIdx = GlobalTables::GetStrTable().GetStrIdxFromName(funcName);
9071 MIRSymbol *st = GlobalTables::GetGsymTable().GetSymbolFromStrIdx(strIdx, true);
9072 IntrinsifyStringIndexOf(*srcOpnds, *st);
9073 return;
9074 }
9075
9076 Insn &callInsn = AppendCall(*fsym, *srcOpnds);
9077 GetCurBB()->SetHasCall();
9078 if (retType != nullptr) {
9079 callInsn.SetRetSize(static_cast<uint32>(retType->GetSize()));
9080 callInsn.SetIsCallReturnUnsigned(IsUnsignedInteger(retType->GetPrimType()));
9081 }
9082 const auto &deoptBundleInfo = callNode.GetDeoptBundleInfo();
9083 for (const auto &elem : deoptBundleInfo) {
9084 auto valueKind = elem.second.GetMapleValueKind();
9085 if (valueKind == MapleValue::kPregKind) {
9086 auto *opnd = GetOpndFromPregIdx(elem.second.GetPregIdx());
9087 CHECK_FATAL(opnd != nullptr, "pregIdx has not been assigned Operand");
9088 callInsn.AddDeoptBundleInfo(elem.first, *opnd);
9089 } else if (valueKind == MapleValue::kConstKind) {
9090 auto *opnd = SelectIntConst(static_cast<const MIRIntConst &>(elem.second.GetConstValue()));
9091 callInsn.AddDeoptBundleInfo(elem.first, *opnd);
9092 } else {
9093 CHECK_FATAL(false, "not supported currently");
9094 }
9095 }
9096 AppendStackMapInsn(callInsn);
9097
9098 /* check if this call use stack slot to return */
9099 if (fn->IsFirstArgReturn()) {
9100 SetStackProtectInfo(kRetureStackSlot);
9101 }
9102
9103 GetFunction().SetHasCall();
9104 if (GetMirModule().IsCModule()) { /* do not mark abort BB in C at present */
9105 if (fsym->GetName() == "__builtin_unreachable") {
9106 GetCurBB()->ClearInsns();
9107 GetCurBB()->SetUnreachable(true);
9108 }
9109 return;
9110 }
9111 }
9112
SelectIcall(IcallNode & icallNode,Operand & srcOpnd)9113 void AArch64CGFunc::SelectIcall(IcallNode &icallNode, Operand &srcOpnd)
9114 {
9115 ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
9116 SelectParmListWrapper(icallNode, *srcOpnds, false);
9117
9118 Operand *fptrOpnd = &srcOpnd;
9119 if (fptrOpnd->GetKind() != Operand::kOpdRegister) {
9120 PrimType ty = icallNode.Opnd(0)->GetPrimType();
9121 fptrOpnd = &SelectCopy(srcOpnd, ty, ty);
9122 }
9123 DEBUG_ASSERT(fptrOpnd->IsRegister(), "SelectIcall: function pointer not RegOperand");
9124 RegOperand *regOpnd = static_cast<RegOperand *>(fptrOpnd);
9125 Insn &callInsn = GetInsnBuilder()->BuildInsn(MOP_xblr, *regOpnd, *srcOpnds);
9126
9127 MIRType *retType = icallNode.GetCallReturnType();
9128 if (retType != nullptr) {
9129 callInsn.SetRetSize(static_cast<uint32>(retType->GetSize()));
9130 callInsn.SetIsCallReturnUnsigned(IsUnsignedInteger(retType->GetPrimType()));
9131 }
9132
9133 /* check if this icall use stack slot to return */
9134 CallReturnVector *p2nrets = &icallNode.GetReturnVec();
9135 if (p2nrets->size() == k1ByteSize) {
9136 StIdx stIdx = (*p2nrets)[0].first;
9137 MIRSymbol *sym = GetBecommon().GetMIRModule().CurFunction()->GetSymTab()->GetSymbolFromStIdx(stIdx.Idx());
9138 if (sym != nullptr && (GetBecommon().GetTypeSize(sym->GetTyIdx().GetIdx()) > k16ByteSize)) {
9139 SetStackProtectInfo(kRetureStackSlot);
9140 }
9141 }
9142
9143 GetCurBB()->AppendInsn(callInsn);
9144 GetCurBB()->SetHasCall();
9145 DEBUG_ASSERT(GetCurBB()->GetLastInsn()->IsCall(), "lastInsn should be a call");
9146 GetFunction().SetHasCall();
9147 const auto &deoptBundleInfo = icallNode.GetDeoptBundleInfo();
9148 for (const auto &elem : deoptBundleInfo) {
9149 auto valueKind = elem.second.GetMapleValueKind();
9150 if (valueKind == MapleValue::kPregKind) {
9151 auto *opnd = GetOpndFromPregIdx(elem.second.GetPregIdx());
9152 CHECK_FATAL(opnd != nullptr, "pregIdx has not been assigned Operand");
9153 callInsn.AddDeoptBundleInfo(elem.first, *opnd);
9154 } else if (valueKind == MapleValue::kConstKind) {
9155 auto *opnd = SelectIntConst(static_cast<const MIRIntConst &>(elem.second.GetConstValue()));
9156 callInsn.AddDeoptBundleInfo(elem.first, *opnd);
9157 } else {
9158 CHECK_FATAL(false, "not supported currently");
9159 }
9160 }
9161 AppendStackMapInsn(callInsn);
9162 }
9163
HandleCatch()9164 void AArch64CGFunc::HandleCatch()
9165 {
9166 if (Globals::GetInstance()->GetOptimLevel() >= CGOptions::kLevel1) {
9167 regno_t regNO = uCatch.regNOCatch;
9168 RegOperand &vregOpnd = GetOrCreateVirtualRegisterOperand(regNO);
9169 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(
9170 MOP_xmovrr, vregOpnd, GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, kRegTyInt)));
9171 } else {
9172 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(
9173 PickStInsn(uCatch.opndCatch->GetSize(), PTY_a64),
9174 GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, kRegTyInt), *uCatch.opndCatch));
9175 }
9176 }
9177
SelectMembar(StmtNode & membar)9178 void AArch64CGFunc::SelectMembar(StmtNode &membar)
9179 {
9180 switch (membar.GetOpCode()) {
9181 case OP_membaracquire:
9182 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_dmb_ishld, AArch64CG::kMd[MOP_dmb_ishld]));
9183 break;
9184 case OP_membarrelease:
9185 case OP_membarstoreload:
9186 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_dmb_ish, AArch64CG::kMd[MOP_dmb_ish]));
9187 break;
9188 case OP_membarstorestore:
9189 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_dmb_ishst, AArch64CG::kMd[MOP_dmb_ishst]));
9190 break;
9191 default:
9192 DEBUG_ASSERT(false, "NYI");
9193 break;
9194 }
9195 }
9196
SelectComment(CommentNode & comment)9197 void AArch64CGFunc::SelectComment(CommentNode &comment)
9198 {
9199 GetCurBB()->AppendInsn(CreateCommentInsn(comment.GetComment()));
9200 }
9201
SelectReturn(Operand * opnd0)9202 void AArch64CGFunc::SelectReturn(Operand *opnd0)
9203 {
9204 bool is64x1vec = GetFunction().GetAttr(FUNCATTR_oneelem_simd) ? true : false;
9205 MIRType *floatType = GlobalTables::GetTypeTable().GetDouble();
9206 MIRType *retTyp = is64x1vec ? floatType : GetFunction().GetReturnType();
9207 CCImpl &retLocator = *GetOrCreateLocator(GetCurCallConvKind());
9208 CCLocInfo retMech;
9209 retLocator.InitReturnInfo(*retTyp, retMech);
9210 if ((retMech.GetRegCount() > 0) && (opnd0 != nullptr)) {
9211 RegType regTyp = is64x1vec ? kRegTyFloat : GetRegTyFromPrimTy(retMech.GetPrimTypeOfReg0());
9212 PrimType oriPrimType = is64x1vec ? GetFunction().GetReturnType()->GetPrimType() : retMech.GetPrimTypeOfReg0();
9213 AArch64reg retReg = static_cast<AArch64reg>(retMech.GetReg0());
9214 if (opnd0->IsRegister()) {
9215 RegOperand *regOpnd = static_cast<RegOperand *>(opnd0);
9216 if (regOpnd->GetRegisterNumber() != retMech.GetReg0()) {
9217 RegOperand &retOpnd = GetOrCreatePhysicalRegisterOperand(retReg, regOpnd->GetSize(), regTyp);
9218 SelectCopy(retOpnd, retMech.GetPrimTypeOfReg0(), *regOpnd, oriPrimType);
9219 }
9220 } else if (opnd0->IsMemoryAccessOperand()) {
9221 auto *memopnd = static_cast<MemOperand *>(opnd0);
9222 RegOperand &retOpnd =
9223 GetOrCreatePhysicalRegisterOperand(retReg, GetPrimTypeBitSize(retMech.GetPrimTypeOfReg0()), regTyp);
9224 MOperator mOp = PickLdInsn(memopnd->GetSize(), retMech.GetPrimTypeOfReg0());
9225 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, retOpnd, *memopnd));
9226 } else if (opnd0->IsConstImmediate()) {
9227 ImmOperand *immOpnd = static_cast<ImmOperand *>(opnd0);
9228 if (!is64x1vec) {
9229 RegOperand &retOpnd =
9230 GetOrCreatePhysicalRegisterOperand(retReg, GetPrimTypeBitSize(retMech.GetPrimTypeOfReg0()),
9231 GetRegTyFromPrimTy(retMech.GetPrimTypeOfReg0()));
9232 SelectCopy(retOpnd, retMech.GetPrimTypeOfReg0(), *immOpnd, retMech.GetPrimTypeOfReg0());
9233 } else {
9234 PrimType rType = GetFunction().GetReturnType()->GetPrimType();
9235 RegOperand *reg = &CreateRegisterOperandOfType(rType);
9236 SelectCopy(*reg, rType, *immOpnd, rType);
9237 RegOperand &retOpnd = GetOrCreatePhysicalRegisterOperand(retReg, GetPrimTypeBitSize(PTY_f64),
9238 GetRegTyFromPrimTy(PTY_f64));
9239 Insn &insn = GetInsnBuilder()->BuildInsn(MOP_xvmovdr, retOpnd, *reg);
9240 GetCurBB()->AppendInsn(insn);
9241 }
9242 } else {
9243 CHECK_FATAL(false, "nyi");
9244 }
9245 }
9246 GetExitBBsVec().emplace_back(GetCurBB());
9247 }
9248
GetOrCreateSpecialRegisterOperand(PregIdx sregIdx,PrimType primType)9249 RegOperand &AArch64CGFunc::GetOrCreateSpecialRegisterOperand(PregIdx sregIdx, PrimType primType)
9250 {
9251 AArch64reg reg = R0;
9252 switch (sregIdx) {
9253 case kSregSp:
9254 reg = RSP;
9255 break;
9256 case kSregFp:
9257 reg = RFP;
9258 break;
9259 case kSregGp: {
9260 MIRSymbol *sym = GetCG()->GetGP();
9261 if (sym == nullptr) {
9262 sym = GetFunction().GetSymTab()->CreateSymbol(kScopeLocal);
9263 std::string strBuf("__file__local__GP");
9264 sym->SetNameStrIdx(GetMirModule().GetMIRBuilder()->GetOrCreateStringIndex(strBuf));
9265 GetCG()->SetGP(sym);
9266 }
9267 RegOperand &result = GetOrCreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
9268 SelectAddrof(result, CreateStImmOperand(*sym, 0, 0));
9269 return result;
9270 }
9271 case kSregThrownval: { /* uses x0 == R0 */
9272 DEBUG_ASSERT(uCatch.regNOCatch > 0, "regNOCatch should greater than 0.");
9273 if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) {
9274 RegOperand ®Opnd = GetOrCreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8BitSize));
9275 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(uCatch.opndCatch->GetSize(), PTY_a64),
9276 regOpnd, *uCatch.opndCatch));
9277 return regOpnd;
9278 } else {
9279 return GetOrCreateVirtualRegisterOperand(uCatch.regNOCatch);
9280 }
9281 }
9282 case kSregRetval0:
9283 if (!IsPrimitiveInteger(primType) || IsPrimitiveVectorFloat(primType)) {
9284 reg = V0;
9285 }
9286 break;
9287 case kSregMethodhdl:
9288 if (methodHandleVreg == regno_t(-1)) {
9289 methodHandleVreg = NewVReg(kRegTyInt, k8BitSize);
9290 }
9291 return GetOrCreateVirtualRegisterOperand(methodHandleVreg);
9292 default:
9293 DEBUG_ASSERT(false, "Special pseudo registers NYI");
9294 break;
9295 }
9296 return GetOrCreatePhysicalRegisterOperand(reg, k64BitSize, kRegTyInt);
9297 }
9298
GetOrCreatePhysicalRegisterOperand(std::string & asmAttr)9299 RegOperand &AArch64CGFunc::GetOrCreatePhysicalRegisterOperand(std::string &asmAttr)
9300 {
9301 DEBUG_ASSERT(!asmAttr.empty(), "Get inline asm string failed in GetOrCreatePhysicalRegisterOperand");
9302 RegType rKind = kRegTyUndef;
9303 uint32 rSize = 0;
9304 /* Get Register Type and Size */
9305 switch (asmAttr[0]) {
9306 case 'x': {
9307 rKind = kRegTyInt;
9308 rSize = k64BitSize;
9309 break;
9310 }
9311 case 'w': {
9312 rKind = kRegTyInt;
9313 rSize = k32BitSize;
9314 break;
9315 }
9316 default: {
9317 LogInfo::MapleLogger() << "Unsupport asm string : " << asmAttr << "\n";
9318 CHECK_FATAL(false, "Have not support this kind of register ");
9319 }
9320 }
9321 AArch64reg rNO = kRinvalid;
9322 /* Get Register Number */
9323 uint32 regNumPos = 1;
9324 char numberChar = asmAttr[regNumPos++];
9325 if (numberChar >= '0' && numberChar <= '9') {
9326 uint32 val = static_cast<uint32>(numberChar - '0');
9327 if (regNumPos < asmAttr.length()) {
9328 char numberCharSecond = asmAttr[regNumPos++];
9329 DEBUG_ASSERT(regNumPos == asmAttr.length(), "Invalid asm attribute");
9330 if (numberCharSecond >= '0' && numberCharSecond <= '9') {
9331 val = val * kDecimalMax + static_cast<uint32>((numberCharSecond - '0'));
9332 }
9333 }
9334 rNO = static_cast<AArch64reg>(static_cast<uint32>(R0) + val);
9335 if (val > (kAsmInputRegPrefixOpnd + 1)) {
9336 LogInfo::MapleLogger() << "Unsupport asm string : " << asmAttr << "\n";
9337 CHECK_FATAL(false, "have not support this kind of register ");
9338 }
9339 } else if (numberChar == 0) {
9340 return CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
9341 } else {
9342 CHECK_FATAL(false, "Unexpect input in GetOrCreatePhysicalRegisterOperand");
9343 }
9344 return GetOrCreatePhysicalRegisterOperand(rNO, rSize, rKind);
9345 }
9346
GetOrCreatePhysicalRegisterOperand(AArch64reg regNO,uint32 size,RegType kind,uint32 flag)9347 RegOperand &AArch64CGFunc::GetOrCreatePhysicalRegisterOperand(AArch64reg regNO, uint32 size, RegType kind, uint32 flag)
9348 {
9349 uint64 aarch64PhyRegIdx = regNO;
9350 DEBUG_ASSERT(flag == 0, "Do not expect flag here");
9351 if (size <= k32BitSize) {
9352 size = k32BitSize;
9353 aarch64PhyRegIdx = aarch64PhyRegIdx << 1;
9354 } else if (size <= k64BitSize) {
9355 size = k64BitSize;
9356 aarch64PhyRegIdx = (aarch64PhyRegIdx << 1) + 1;
9357 } else {
9358 size = (size == k128BitSize) ? k128BitSize : k64BitSize;
9359 aarch64PhyRegIdx = aarch64PhyRegIdx << k4BitShift;
9360 }
9361 RegOperand *phyRegOpnd = nullptr;
9362 auto phyRegIt = phyRegOperandTable.find(aarch64PhyRegIdx);
9363 if (phyRegIt != phyRegOperandTable.end()) {
9364 phyRegOpnd = phyRegOperandTable[aarch64PhyRegIdx];
9365 } else {
9366 phyRegOpnd = memPool->New<RegOperand>(regNO, size, kind, flag);
9367 phyRegOperandTable.emplace(aarch64PhyRegIdx, phyRegOpnd);
9368 }
9369 return *phyRegOpnd;
9370 }
9371
GetLabelOperand(LabelIdx labIdx) const9372 const LabelOperand *AArch64CGFunc::GetLabelOperand(LabelIdx labIdx) const
9373 {
9374 const MapleUnorderedMap<LabelIdx, LabelOperand *>::const_iterator it = hashLabelOpndTable.find(labIdx);
9375 if (it != hashLabelOpndTable.end()) {
9376 return it->second;
9377 }
9378 return nullptr;
9379 }
9380
GetOrCreateLabelOperand(LabelIdx labIdx)9381 LabelOperand &AArch64CGFunc::GetOrCreateLabelOperand(LabelIdx labIdx)
9382 {
9383 MapleUnorderedMap<LabelIdx, LabelOperand *>::iterator it = hashLabelOpndTable.find(labIdx);
9384 if (it != hashLabelOpndTable.end()) {
9385 return *(it->second);
9386 }
9387 const char *funcName = GetShortFuncName().c_str();
9388 LabelOperand *res = memPool->New<LabelOperand>(funcName, labIdx);
9389 hashLabelOpndTable[labIdx] = res;
9390 return *res;
9391 }
9392
GetOrCreateLabelOperand(BB & bb)9393 LabelOperand &AArch64CGFunc::GetOrCreateLabelOperand(BB &bb)
9394 {
9395 LabelIdx labelIdx = bb.GetLabIdx();
9396 if (labelIdx == MIRLabelTable::GetDummyLabel()) {
9397 labelIdx = CreateLabel();
9398 bb.AddLabel(labelIdx);
9399 SetLab2BBMap(labelIdx, bb);
9400 }
9401 return GetOrCreateLabelOperand(labelIdx);
9402 }
9403
GetAggCopySize(uint32 offset1,uint32 offset2,uint32 alignment) const9404 uint32 AArch64CGFunc::GetAggCopySize(uint32 offset1, uint32 offset2, uint32 alignment) const
9405 {
9406 /* Generating a larger sized mem op than alignment if allowed by aggregate starting address */
9407 uint32 offsetAlign1 = (offset1 == 0) ? k8ByteSize : offset1;
9408 uint32 offsetAlign2 = (offset2 == 0) ? k8ByteSize : offset2;
9409 uint32 alignOffset =
9410 1U << (std::min(__builtin_ffs(static_cast<int>(offsetAlign1)), __builtin_ffs(static_cast<int>(offsetAlign2))) -
9411 1);
9412 if (alignOffset == k8ByteSize || alignOffset == k4ByteSize || alignOffset == k2ByteSize) {
9413 return alignOffset;
9414 } else if (alignOffset > k8ByteSize) {
9415 return k8ByteSize;
9416 } else {
9417 return alignment;
9418 }
9419 }
9420
GetOrCreateOfstOpnd(uint64 offset,uint32 size)9421 OfstOperand &AArch64CGFunc::GetOrCreateOfstOpnd(uint64 offset, uint32 size)
9422 {
9423 uint64 aarch64OfstRegIdx = offset;
9424 aarch64OfstRegIdx = (aarch64OfstRegIdx << 1);
9425 if (size == k64BitSize) {
9426 ++aarch64OfstRegIdx;
9427 }
9428 DEBUG_ASSERT(size == k32BitSize || size == k64BitSize, "ofStOpnd size check");
9429 auto it = hashOfstOpndTable.find(aarch64OfstRegIdx);
9430 if (it != hashOfstOpndTable.end()) {
9431 return *it->second;
9432 }
9433 OfstOperand *res = &CreateOfstOpnd(offset, size);
9434 hashOfstOpndTable[aarch64OfstRegIdx] = res;
9435 return *res;
9436 }
9437
SelectAddrofAfterRa(Operand & result,StImmOperand & stImm,std::vector<Insn * > & rematInsns)9438 void AArch64CGFunc::SelectAddrofAfterRa(Operand &result, StImmOperand &stImm, std::vector<Insn *> &rematInsns)
9439 {
9440 const MIRSymbol *symbol = stImm.GetSymbol();
9441 DEBUG_ASSERT((symbol->GetStorageClass() != kScAuto) || (symbol->GetStorageClass() != kScFormal), "");
9442 Operand *srcOpnd = &result;
9443 rematInsns.emplace_back(&GetInsnBuilder()->BuildInsn(MOP_xadrp, result, stImm));
9444 if (CGOptions::IsPIC() && symbol->NeedPIC()) {
9445 /* ldr x0, [x0, #:got_lo12:Ljava_2Flang_2FSystem_3B_7Cout] */
9446 OfstOperand &offset = CreateOfstOpnd(*stImm.GetSymbol(), stImm.GetOffset(), stImm.GetRelocs());
9447 MemOperand &memOpnd = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, GetPointerSize() * kBitsPerByte,
9448 static_cast<RegOperand *>(srcOpnd), nullptr, &offset, nullptr);
9449 rematInsns.emplace_back(
9450 &GetInsnBuilder()->BuildInsn(memOpnd.GetSize() == k64BitSize ? MOP_xldr : MOP_wldr, result, memOpnd));
9451
9452 if (stImm.GetOffset() > 0) {
9453 ImmOperand &immOpnd = CreateImmOperand(stImm.GetOffset(), result.GetSize(), false);
9454 rematInsns.emplace_back(&GetInsnBuilder()->BuildInsn(MOP_xaddrri12, result, result, immOpnd));
9455 return;
9456 }
9457 } else {
9458 rematInsns.emplace_back(&GetInsnBuilder()->BuildInsn(MOP_xadrpl12, result, *srcOpnd, stImm));
9459 }
9460 }
9461
GetOrCreateMemOpndAfterRa(const MIRSymbol & symbol,int32 offset,uint32 size,bool needLow12,RegOperand * regOp,std::vector<Insn * > & rematInsns)9462 MemOperand &AArch64CGFunc::GetOrCreateMemOpndAfterRa(const MIRSymbol &symbol, int32 offset, uint32 size, bool needLow12,
9463 RegOperand *regOp, std::vector<Insn *> &rematInsns)
9464 {
9465 MIRStorageClass storageClass = symbol.GetStorageClass();
9466 if ((storageClass == kScGlobal) || (storageClass == kScExtern)) {
9467 StImmOperand &stOpnd = CreateStImmOperand(symbol, offset, 0);
9468 RegOperand &stAddrOpnd = *regOp;
9469 SelectAddrofAfterRa(stAddrOpnd, stOpnd, rematInsns);
9470 /* MemOperand::AddrMode_B_OI */
9471 return *CreateMemOperand(MemOperand::kAddrModeBOi, size, stAddrOpnd, nullptr,
9472 &GetOrCreateOfstOpnd(0, k32BitSize), &symbol);
9473 } else if ((storageClass == kScPstatic) || (storageClass == kScFstatic)) {
9474 if (symbol.GetSKind() == kStConst) {
9475 DEBUG_ASSERT(offset == 0, "offset should be 0 for constant literals");
9476 return *CreateMemOperand(MemOperand::kAddrModeLiteral, size, symbol);
9477 } else {
9478 if (needLow12) {
9479 StImmOperand &stOpnd = CreateStImmOperand(symbol, offset, 0);
9480 RegOperand &stAddrOpnd = *regOp;
9481 SelectAddrofAfterRa(stAddrOpnd, stOpnd, rematInsns);
9482 return *CreateMemOperand(MemOperand::kAddrModeBOi, size, stAddrOpnd, nullptr,
9483 &GetOrCreateOfstOpnd(0, k32BitSize), &symbol);
9484 } else {
9485 StImmOperand &stOpnd = CreateStImmOperand(symbol, offset, 0);
9486 RegOperand &stAddrOpnd = *regOp;
9487 /* adrp x1, _PTR__cinf_Ljava_2Flang_2FSystem_3B */
9488 Insn &insn = GetInsnBuilder()->BuildInsn(MOP_xadrp, stAddrOpnd, stOpnd);
9489 rematInsns.emplace_back(&insn);
9490 /* ldr x1, [x1, #:lo12:_PTR__cinf_Ljava_2Flang_2FSystem_3B] */
9491 return *CreateMemOperand(MemOperand::kAddrModeLo12Li, size, stAddrOpnd, nullptr,
9492 &GetOrCreateOfstOpnd(static_cast<uint64>(offset), k32BitSize), &symbol);
9493 }
9494 }
9495 } else {
9496 CHECK_FATAL(false, "NYI");
9497 }
9498 }
9499
GetOrCreateMemOpnd(const MIRSymbol & symbol,int64 offset,uint32 size,bool forLocalRef,bool needLow12,RegOperand * regOp)9500 MemOperand &AArch64CGFunc::GetOrCreateMemOpnd(const MIRSymbol &symbol, int64 offset, uint32 size, bool forLocalRef,
9501 bool needLow12, RegOperand *regOp)
9502 {
9503 MIRStorageClass storageClass = symbol.GetStorageClass();
9504 if ((storageClass == kScAuto) || (storageClass == kScFormal)) {
9505 AArch64SymbolAlloc *symLoc =
9506 static_cast<AArch64SymbolAlloc *>(GetMemlayout()->GetSymAllocInfo(symbol.GetStIndex()));
9507 if (forLocalRef) {
9508 auto p = GetMemlayout()->GetLocalRefLocMap().find(symbol.GetStIdx());
9509 CHECK_FATAL(p != GetMemlayout()->GetLocalRefLocMap().end(), "sym loc should have been defined");
9510 symLoc = static_cast<AArch64SymbolAlloc *>(p->second);
9511 }
9512 DEBUG_ASSERT(symLoc != nullptr, "sym loc should have been defined");
9513 /* At this point, we don't know which registers the callee needs to save. */
9514 DEBUG_ASSERT((IsFPLRAddedToCalleeSavedList() || (SizeOfCalleeSaved() == 0)),
9515 "CalleeSaved won't be known until after Register Allocation");
9516 StIdx idx = symbol.GetStIdx();
9517 auto it = memOpndsRequiringOffsetAdjustment.find(idx);
9518 DEBUG_ASSERT((!IsFPLRAddedToCalleeSavedList() ||
9519 ((it != memOpndsRequiringOffsetAdjustment.end()) || (storageClass == kScFormal))),
9520 "Memory operand of this symbol should have been added to the hash table");
9521 int32 stOffset = GetBaseOffset(*symLoc);
9522 if (it != memOpndsRequiringOffsetAdjustment.end()) {
9523 if (GetMemlayout()->IsLocalRefLoc(symbol)) {
9524 if (!forLocalRef) {
9525 return *(it->second);
9526 }
9527 } else if (mirModule.IsJavaModule()) {
9528 return *(it->second);
9529 } else {
9530 Operand *offOpnd = (it->second)->GetOffset();
9531 if (((static_cast<OfstOperand *>(offOpnd))->GetOffsetValue() == (stOffset + offset)) &&
9532 (it->second->GetSize() == size)) {
9533 return *(it->second);
9534 }
9535 }
9536 }
9537 it = memOpndsForStkPassedArguments.find(idx);
9538 if (it != memOpndsForStkPassedArguments.end()) {
9539 if (GetMemlayout()->IsLocalRefLoc(symbol)) {
9540 if (!forLocalRef) {
9541 return *(it->second);
9542 }
9543 } else {
9544 return *(it->second);
9545 }
9546 }
9547
9548 RegOperand *baseOpnd = static_cast<RegOperand *>(GetBaseReg(*symLoc));
9549 int32 totalOffset = stOffset + static_cast<int32>(offset);
9550 /* needs a fresh copy of ImmOperand as we may adjust its offset at a later stage. */
9551 OfstOperand *offsetOpnd = nullptr;
9552 if (CGOptions::IsBigEndian()) {
9553 if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsArgsStkPassed && size < k64BitSize) {
9554 offsetOpnd = &CreateOfstOpnd(k4BitSize + static_cast<uint32>(totalOffset), k64BitSize);
9555 } else {
9556 offsetOpnd = &CreateOfstOpnd(static_cast<uint64>(static_cast<int64>(totalOffset)), k64BitSize);
9557 }
9558 } else {
9559 offsetOpnd = &CreateOfstOpnd(static_cast<uint64>(static_cast<int64>(totalOffset)), k64BitSize);
9560 }
9561 if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsArgsStkPassed &&
9562 MemOperand::IsPIMMOffsetOutOfRange(totalOffset, size)) {
9563 ImmOperand *offsetOprand;
9564 offsetOprand = &CreateImmOperand(totalOffset, k64BitSize, true, kUnAdjustVary);
9565 Operand *resImmOpnd = &SelectCopy(*offsetOprand, PTY_i64, PTY_i64);
9566 return *CreateMemOperand(MemOperand::kAddrModeBOrX, size, *baseOpnd, static_cast<RegOperand &>(*resImmOpnd),
9567 nullptr, symbol, true);
9568 } else {
9569 if (symLoc->GetMemSegment()->GetMemSegmentKind() == kMsArgsStkPassed) {
9570 offsetOpnd->SetVary(kUnAdjustVary);
9571 }
9572 MemOperand *res = CreateMemOperand(MemOperand::kAddrModeBOi, size, *baseOpnd, nullptr, offsetOpnd, &symbol);
9573 if ((symbol.GetType()->GetKind() != kTypeClass) && !forLocalRef) {
9574 memOpndsRequiringOffsetAdjustment[idx] = res;
9575 }
9576 return *res;
9577 }
9578 } else if ((storageClass == kScGlobal) || (storageClass == kScExtern)) {
9579 StImmOperand &stOpnd = CreateStImmOperand(symbol, offset, 0);
9580 if (!regOp) {
9581 regOp = static_cast<RegOperand *>(&CreateRegisterOperandOfType(PTY_u64));
9582 }
9583 RegOperand &stAddrOpnd = *regOp;
9584 SelectAddrof(stAddrOpnd, stOpnd);
9585 /* MemOperand::AddrMode_B_OI */
9586 return *CreateMemOperand(MemOperand::kAddrModeBOi, size, stAddrOpnd, nullptr,
9587 &GetOrCreateOfstOpnd(0, k32BitSize), &symbol);
9588 } else if ((storageClass == kScPstatic) || (storageClass == kScFstatic)) {
9589 if (symbol.GetSKind() == kStConst) {
9590 DEBUG_ASSERT(offset == 0, "offset should be 0 for constant literals");
9591 return *CreateMemOperand(MemOperand::kAddrModeLiteral, size, symbol);
9592 } else {
9593 /* not guaranteed align for uninitialized symbol */
9594 if (needLow12 || (!symbol.IsConst() && CGOptions::IsPIC())) {
9595 StImmOperand &stOpnd = CreateStImmOperand(symbol, offset, 0);
9596 if (!regOp) {
9597 regOp = static_cast<RegOperand *>(&CreateRegisterOperandOfType(PTY_u64));
9598 }
9599 RegOperand &stAddrOpnd = *regOp;
9600 SelectAddrof(stAddrOpnd, stOpnd);
9601 return *CreateMemOperand(MemOperand::kAddrModeBOi, size, stAddrOpnd, nullptr,
9602 &GetOrCreateOfstOpnd(0, k32BitSize), &symbol);
9603 } else {
9604 StImmOperand &stOpnd = CreateStImmOperand(symbol, offset, 0);
9605 if (!regOp) {
9606 regOp = static_cast<RegOperand *>(&CreateRegisterOperandOfType(PTY_u64));
9607 }
9608 RegOperand &stAddrOpnd = *regOp;
9609 /* adrp x1, _PTR__cinf_Ljava_2Flang_2FSystem_3B */
9610 Insn &insn = GetInsnBuilder()->BuildInsn(MOP_xadrp, stAddrOpnd, stOpnd);
9611 GetCurBB()->AppendInsn(insn);
9612 /* ldr x1, [x1, #:lo12:_PTR__cinf_Ljava_2Flang_2FSystem_3B] */
9613 return *CreateMemOperand(MemOperand::kAddrModeLo12Li, size, stAddrOpnd, nullptr,
9614 &GetOrCreateOfstOpnd(static_cast<uint64>(offset), k32BitSize), &symbol);
9615 }
9616 }
9617 } else {
9618 CHECK_FATAL(false, "NYI");
9619 }
9620 }
9621
HashMemOpnd(MemOperand & tMemOpnd)9622 MemOperand &AArch64CGFunc::HashMemOpnd(MemOperand &tMemOpnd)
9623 {
9624 auto it = hashMemOpndTable.find(tMemOpnd);
9625 if (it != hashMemOpndTable.end()) {
9626 return *(it->second);
9627 }
9628 auto *res = memPool->New<MemOperand>(tMemOpnd);
9629 hashMemOpndTable[tMemOpnd] = res;
9630 return *res;
9631 }
9632
GetOrCreateMemOpnd(MemOperand::AArch64AddressingMode mode,uint32 size,RegOperand * base,RegOperand * index,ImmOperand * offset,const MIRSymbol * st)9633 MemOperand &AArch64CGFunc::GetOrCreateMemOpnd(MemOperand::AArch64AddressingMode mode, uint32 size, RegOperand *base,
9634 RegOperand *index, ImmOperand *offset, const MIRSymbol *st)
9635 {
9636 DEBUG_ASSERT(base != nullptr, "nullptr check");
9637 MemOperand tMemOpnd(mode, size, *base, index, offset, st);
9638 if (base->GetRegisterNumber() == RFP || base->GetRegisterNumber() == RSP) {
9639 tMemOpnd.SetStackMem(true);
9640 }
9641 return HashMemOpnd(tMemOpnd);
9642 }
9643
GetOrCreateMemOpnd(MemOperand::AArch64AddressingMode mode,uint32 size,RegOperand * base,RegOperand * index,int32 shift,bool isSigned)9644 MemOperand &AArch64CGFunc::GetOrCreateMemOpnd(MemOperand::AArch64AddressingMode mode, uint32 size, RegOperand *base,
9645 RegOperand *index, int32 shift, bool isSigned)
9646 {
9647 DEBUG_ASSERT(base != nullptr, "nullptr check");
9648 MemOperand tMemOpnd(mode, size, *base, *index, shift, isSigned);
9649 if (base->GetRegisterNumber() == RFP || base->GetRegisterNumber() == RSP) {
9650 tMemOpnd.SetStackMem(true);
9651 }
9652 return HashMemOpnd(tMemOpnd);
9653 }
9654
GetOrCreateMemOpnd(MemOperand & oldMem)9655 MemOperand &AArch64CGFunc::GetOrCreateMemOpnd(MemOperand &oldMem)
9656 {
9657 return HashMemOpnd(oldMem);
9658 }
9659
9660 /* offset: base offset from FP or SP */
CreateMemOpnd(RegOperand & baseOpnd,int64 offset,uint32 size)9661 MemOperand &AArch64CGFunc::CreateMemOpnd(RegOperand &baseOpnd, int64 offset, uint32 size)
9662 {
9663 OfstOperand &offsetOpnd = CreateOfstOpnd(static_cast<uint64>(offset), k32BitSize);
9664 /* do not need to check bit size rotate of sign immediate */
9665 bool checkSimm = (offset > kMinSimm64 && offset < kMaxSimm64Pair);
9666 if (!checkSimm && !ImmOperand::IsInBitSizeRot(kMaxImmVal12Bits, offset)) {
9667 Operand *resImmOpnd = &SelectCopy(CreateImmOperand(offset, k32BitSize, true), PTY_i32, PTY_i32);
9668 return *CreateMemOperand(MemOperand::kAddrModeBOrX, size, baseOpnd, static_cast<RegOperand *>(resImmOpnd),
9669 nullptr, nullptr);
9670 } else {
9671 return *CreateMemOperand(MemOperand::kAddrModeBOi, size, baseOpnd, nullptr, &offsetOpnd, nullptr);
9672 }
9673 }
9674
9675 /* offset: base offset + #:lo12:Label+immediate */
CreateMemOpnd(RegOperand & baseOpnd,int64 offset,uint32 size,const MIRSymbol & sym)9676 MemOperand &AArch64CGFunc::CreateMemOpnd(RegOperand &baseOpnd, int64 offset, uint32 size, const MIRSymbol &sym)
9677 {
9678 OfstOperand &offsetOpnd = CreateOfstOpnd(static_cast<uint64>(offset), k32BitSize);
9679 DEBUG_ASSERT(ImmOperand::IsInBitSizeRot(kMaxImmVal12Bits, offset), "");
9680 return *CreateMemOperand(MemOperand::kAddrModeBOi, size, baseOpnd, nullptr, &offsetOpnd, &sym);
9681 }
9682
GenStructParamIndex(RegOperand & base,const BaseNode & indexExpr,int shift,PrimType baseType,PrimType targetType)9683 RegOperand &AArch64CGFunc::GenStructParamIndex(RegOperand &base, const BaseNode &indexExpr, int shift,
9684 PrimType baseType, PrimType targetType)
9685 {
9686 RegOperand *index = &LoadIntoRegister(*HandleExpr(indexExpr, *(indexExpr.Opnd(0))), PTY_a64);
9687 RegOperand *srcOpnd = &CreateRegisterOperandOfType(PTY_a64);
9688 ImmOperand *imm = &CreateImmOperand(PTY_a64, shift);
9689 SelectShift(*srcOpnd, *index, *imm, kShiftLeft, PTY_a64);
9690 RegOperand *result = &CreateRegisterOperandOfType(PTY_a64);
9691 SelectAdd(*result, base, *srcOpnd, PTY_a64);
9692
9693 OfstOperand *offopnd = &CreateOfstOpnd(0, k32BitSize);
9694 MemOperand &mo = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, result, nullptr, offopnd, nullptr);
9695 RegOperand &structAddr = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
9696 GetCurBB()->AppendInsn(
9697 GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(baseType), targetType), structAddr, mo));
9698 return structAddr;
9699 }
9700
9701 /*
9702 * case 1: iread a64 <* <* void>> 0 (add a64 (
9703 * addrof a64 $__reg_jni_func_tab$$libcore_all_bytecode,
9704 * mul a64 (
9705 * cvt a64 i32 (constval i32 21),
9706 * constval a64 8)))
9707 *
9708 * case 2 : iread u32 <* u8> 0 (add a64 (regread a64 %61, constval a64 3))
9709 * case 3 : iread u32 <* u8> 0 (add a64 (regread a64 %61, regread a64 %65))
9710 * case 4 : iread u32 <* u8> 0 (add a64 (cvt a64 i32(regread %n)))
9711 */
CheckAndCreateExtendMemOpnd(PrimType ptype,const BaseNode & addrExpr,int64 offset,AArch64isa::MemoryOrdering memOrd)9712 MemOperand *AArch64CGFunc::CheckAndCreateExtendMemOpnd(PrimType ptype, const BaseNode &addrExpr, int64 offset,
9713 AArch64isa::MemoryOrdering memOrd)
9714 {
9715 aggParamReg = nullptr;
9716 if (memOrd != AArch64isa::kMoNone || addrExpr.GetOpCode() != OP_add || offset != 0) {
9717 return nullptr;
9718 }
9719 BaseNode *baseExpr = addrExpr.Opnd(0);
9720 BaseNode *addendExpr = addrExpr.Opnd(1);
9721
9722 if (baseExpr->GetOpCode() == OP_regread) {
9723 /* case 2 */
9724 if (addendExpr->GetOpCode() == OP_constval) {
9725 DEBUG_ASSERT(addrExpr.GetNumOpnds() == kOpndNum2, "Unepect expr operand in CheckAndCreateExtendMemOpnd");
9726 ConstvalNode *constOfstNode = static_cast<ConstvalNode *>(addendExpr);
9727 DEBUG_ASSERT(constOfstNode->GetConstVal()->GetKind() == kConstInt, "expect MIRIntConst");
9728 MIRIntConst *intOfst = safe_cast<MIRIntConst>(constOfstNode->GetConstVal());
9729 CHECK_FATAL(intOfst != nullptr, "just checking");
9730 /* discard large offset and negative offset */
9731 if (intOfst->GetExtValue() > INT32_MAX || intOfst->IsNegative()) {
9732 return nullptr;
9733 }
9734 uint32 scale = static_cast<uint32>(intOfst->GetExtValue());
9735 OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(scale, k32BitSize);
9736 uint32 dsize = GetPrimTypeBitSize(ptype);
9737 MemOperand *memOpnd =
9738 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, GetPrimTypeBitSize(ptype),
9739 SelectRegread(*static_cast<RegreadNode *>(baseExpr)), nullptr, &ofstOpnd, nullptr);
9740 return IsOperandImmValid(PickLdInsn(dsize, ptype), memOpnd, kInsnSecondOpnd) ? memOpnd : nullptr;
9741 /* case 3 */
9742 } else if (addendExpr->GetOpCode() == OP_regread) {
9743 CHECK_FATAL(addrExpr.GetNumOpnds() == kOpndNum2, "Unepect expr operand in CheckAndCreateExtendMemOpnd");
9744 if (GetPrimTypeSize(baseExpr->GetPrimType()) != GetPrimTypeSize(addendExpr->GetPrimType())) {
9745 return nullptr;
9746 }
9747
9748 auto *baseReg = SelectRegread(*static_cast<RegreadNode *>(baseExpr));
9749 auto *indexReg = SelectRegread(*static_cast<RegreadNode *>(addendExpr));
9750 MemOperand *memOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, GetPrimTypeBitSize(ptype), baseReg,
9751 indexReg, nullptr, nullptr);
9752 return memOpnd;
9753 /* case 4 */
9754 } else if (addendExpr->GetOpCode() == OP_cvt && addendExpr->GetNumOpnds() == 1) {
9755 int shiftAmount = 0;
9756 BaseNode *cvtRegreadNode = addendExpr->Opnd(kInsnFirstOpnd);
9757 if (cvtRegreadNode->GetOpCode() == OP_regread && cvtRegreadNode->IsLeaf()) {
9758 uint32 fromSize = GetPrimTypeBitSize(cvtRegreadNode->GetPrimType());
9759 uint32 toSize = GetPrimTypeBitSize(addendExpr->GetPrimType());
9760
9761 if (toSize < fromSize) {
9762 return nullptr;
9763 }
9764
9765 MemOperand *memOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, GetPrimTypeBitSize(ptype),
9766 SelectRegread(*static_cast<RegreadNode *>(baseExpr)),
9767 SelectRegread(*static_cast<RegreadNode *>(cvtRegreadNode)),
9768 shiftAmount, toSize != fromSize);
9769 return memOpnd;
9770 }
9771 }
9772 }
9773 if (addendExpr->GetOpCode() != OP_mul || !IsPrimitiveInteger(ptype)) {
9774 return nullptr;
9775 }
9776 BaseNode *indexExpr, *scaleExpr;
9777 indexExpr = addendExpr->Opnd(0);
9778 scaleExpr = addendExpr->Opnd(1);
9779 if (scaleExpr->GetOpCode() != OP_constval) {
9780 return nullptr;
9781 }
9782 ConstvalNode *constValNode = static_cast<ConstvalNode *>(scaleExpr);
9783 CHECK_FATAL(constValNode->GetConstVal()->GetKind() == kConstInt, "expect MIRIntConst");
9784 MIRIntConst *mirIntConst = safe_cast<MIRIntConst>(constValNode->GetConstVal());
9785 CHECK_FATAL(mirIntConst != nullptr, "just checking");
9786 int32 scale = mirIntConst->GetExtValue();
9787 if (scale < 0) {
9788 return nullptr;
9789 }
9790 uint32 unsignedScale = static_cast<uint32>(scale);
9791 if (unsignedScale != GetPrimTypeSize(ptype) || indexExpr->GetOpCode() != OP_cvt) {
9792 return nullptr;
9793 }
9794 /* 8 is 1 << 3; 4 is 1 << 2; 2 is 1 << 1; 1 is 1 << 0 */
9795 int32 shift = (unsignedScale == 8) ? 3 : ((unsignedScale == 4) ? 2 : ((unsignedScale == 2) ? 1 : 0));
9796 RegOperand &base = static_cast<RegOperand &>(LoadIntoRegister(*HandleExpr(addrExpr, *baseExpr), PTY_a64));
9797 TypeCvtNode *typeCvtNode = static_cast<TypeCvtNode *>(indexExpr);
9798 PrimType fromType = typeCvtNode->FromType();
9799 PrimType toType = typeCvtNode->GetPrimType();
9800 if (isAggParamInReg) {
9801 aggParamReg = &GenStructParamIndex(base, *indexExpr, shift, ptype, fromType);
9802 return nullptr;
9803 }
9804 MemOperand *memOpnd = nullptr;
9805 if ((fromType == PTY_i32) && (toType == PTY_a64)) {
9806 RegOperand &index =
9807 static_cast<RegOperand &>(LoadIntoRegister(*HandleExpr(*indexExpr, *indexExpr->Opnd(0)), PTY_i32));
9808 memOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, GetPrimTypeBitSize(ptype), &base, &index, shift, true);
9809 } else if ((fromType == PTY_u32) && (toType == PTY_a64)) {
9810 RegOperand &index =
9811 static_cast<RegOperand &>(LoadIntoRegister(*HandleExpr(*indexExpr, *indexExpr->Opnd(0)), PTY_u32));
9812 memOpnd =
9813 &GetOrCreateMemOpnd(MemOperand::kAddrModeBOrX, GetPrimTypeBitSize(ptype), &base, &index, shift, false);
9814 }
9815 return memOpnd;
9816 }
9817
CreateNonExtendMemOpnd(PrimType ptype,const BaseNode & parent,BaseNode & addrExpr,int64 offset)9818 MemOperand &AArch64CGFunc::CreateNonExtendMemOpnd(PrimType ptype, const BaseNode &parent, BaseNode &addrExpr,
9819 int64 offset)
9820 {
9821 Operand *addrOpnd = nullptr;
9822 if ((addrExpr.GetOpCode() == OP_add || addrExpr.GetOpCode() == OP_sub) &&
9823 addrExpr.Opnd(1)->GetOpCode() == OP_constval) {
9824 addrOpnd = HandleExpr(addrExpr, *addrExpr.Opnd(0));
9825 ConstvalNode *constOfstNode = static_cast<ConstvalNode *>(addrExpr.Opnd(1));
9826 DEBUG_ASSERT(constOfstNode->GetConstVal()->GetKind() == kConstInt, "expect MIRIntConst");
9827 MIRIntConst *intOfst = safe_cast<MIRIntConst>(constOfstNode->GetConstVal());
9828 CHECK_FATAL(intOfst != nullptr, "just checking");
9829 offset = (addrExpr.GetOpCode() == OP_add) ? offset + intOfst->GetSXTValue() : offset - intOfst->GetSXTValue();
9830 } else {
9831 addrOpnd = HandleExpr(parent, addrExpr);
9832 }
9833 addrOpnd = static_cast<RegOperand *>(&LoadIntoRegister(*addrOpnd, PTY_a64));
9834 Insn *lastInsn = GetCurBB() == nullptr ? nullptr : GetCurBB()->GetLastInsn();
9835 if ((addrExpr.GetOpCode() == OP_CG_array_elem_add) && (offset == 0) && lastInsn &&
9836 (lastInsn->GetMachineOpcode() == MOP_xadrpl12) &&
9837 (&lastInsn->GetOperand(kInsnFirstOpnd) == &lastInsn->GetOperand(kInsnSecondOpnd))) {
9838 Operand &opnd = lastInsn->GetOperand(kInsnThirdOpnd);
9839 StImmOperand &stOpnd = static_cast<StImmOperand &>(opnd);
9840
9841 OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(static_cast<uint64>(stOpnd.GetOffset()), k32BitSize);
9842 MemOperand &tmpMemOpnd =
9843 GetOrCreateMemOpnd(MemOperand::kAddrModeLo12Li, GetPrimTypeBitSize(ptype),
9844 static_cast<RegOperand *>(addrOpnd), nullptr, &ofstOpnd, stOpnd.GetSymbol());
9845 GetCurBB()->RemoveInsn(*GetCurBB()->GetLastInsn());
9846 return tmpMemOpnd;
9847 } else {
9848 OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(static_cast<uint64>(offset), k64BitSize);
9849 return GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, GetPrimTypeBitSize(ptype),
9850 static_cast<RegOperand *>(addrOpnd), nullptr, &ofstOpnd, nullptr);
9851 }
9852 }
9853
9854 /*
9855 * Create a memory operand with specified data type and memory ordering, making
9856 * use of aarch64 extend register addressing mode when possible.
9857 */
CreateMemOpnd(PrimType ptype,const BaseNode & parent,BaseNode & addrExpr,int64 offset,AArch64isa::MemoryOrdering memOrd)9858 MemOperand &AArch64CGFunc::CreateMemOpnd(PrimType ptype, const BaseNode &parent, BaseNode &addrExpr, int64 offset,
9859 AArch64isa::MemoryOrdering memOrd)
9860 {
9861 MemOperand *memOpnd = CheckAndCreateExtendMemOpnd(ptype, addrExpr, offset, memOrd);
9862 if (memOpnd != nullptr) {
9863 return *memOpnd;
9864 }
9865 return CreateNonExtendMemOpnd(ptype, parent, addrExpr, offset);
9866 }
9867
CreateMemOpndOrNull(PrimType ptype,const BaseNode & parent,BaseNode & addrExpr,int64 offset,AArch64isa::MemoryOrdering memOrd)9868 MemOperand *AArch64CGFunc::CreateMemOpndOrNull(PrimType ptype, const BaseNode &parent, BaseNode &addrExpr, int64 offset,
9869 AArch64isa::MemoryOrdering memOrd)
9870 {
9871 MemOperand *memOpnd = CheckAndCreateExtendMemOpnd(ptype, addrExpr, offset, memOrd);
9872 if (memOpnd != nullptr) {
9873 return memOpnd;
9874 } else if (aggParamReg != nullptr) {
9875 return nullptr;
9876 }
9877 return &CreateNonExtendMemOpnd(ptype, parent, addrExpr, offset);
9878 }
9879
GetOrCreateFuncNameOpnd(const MIRSymbol & symbol) const9880 Operand &AArch64CGFunc::GetOrCreateFuncNameOpnd(const MIRSymbol &symbol) const
9881 {
9882 return *memPool->New<FuncNameOperand>(symbol);
9883 }
9884
GetOrCreateRflag()9885 Operand &AArch64CGFunc::GetOrCreateRflag()
9886 {
9887 if (rcc == nullptr) {
9888 rcc = &CreateRflagOperand();
9889 }
9890 return *rcc;
9891 }
9892
GetRflag() const9893 const Operand *AArch64CGFunc::GetRflag() const
9894 {
9895 return rcc;
9896 }
9897
GetOrCreatevaryreg()9898 Operand &AArch64CGFunc::GetOrCreatevaryreg()
9899 {
9900 if (vary == nullptr) {
9901 regno_t vRegNO = NewVReg(kRegTyVary, k8ByteSize);
9902 vary = &CreateVirtualRegisterOperand(vRegNO);
9903 }
9904 return *vary;
9905 }
9906
9907 /* the first operand in opndvec is return opnd */
SelectLibCall(const std::string & funcName,std::vector<Operand * > & opndVec,PrimType primType,PrimType retPrimType,bool is2ndRet)9908 void AArch64CGFunc::SelectLibCall(const std::string &funcName, std::vector<Operand *> &opndVec, PrimType primType,
9909 PrimType retPrimType, bool is2ndRet)
9910 {
9911 std::vector<PrimType> pt;
9912 pt.push_back(retPrimType);
9913 for (size_t i = 0; i < opndVec.size(); ++i) {
9914 pt.push_back(primType);
9915 }
9916 SelectLibCallNArg(funcName, opndVec, pt, retPrimType, is2ndRet);
9917 return;
9918 }
9919
SelectLibCallNArg(const std::string & funcName,std::vector<Operand * > & opndVec,std::vector<PrimType> pt,PrimType retPrimType,bool is2ndRet)9920 void AArch64CGFunc::SelectLibCallNArg(const std::string &funcName, std::vector<Operand *> &opndVec,
9921 std::vector<PrimType> pt, PrimType retPrimType, bool is2ndRet)
9922 {
9923 std::string newName = funcName;
9924 // Check whether we have a maple version of libcall and we want to use it instead.
9925 if (!CGOptions::IsDuplicateAsmFileEmpty() && asmMap.find(funcName) != asmMap.end()) {
9926 newName = asmMap.at(funcName);
9927 }
9928 MIRSymbol *st = GlobalTables::GetGsymTable().CreateSymbol(kScopeGlobal);
9929 st->SetNameStrIdx(newName);
9930 st->SetStorageClass(kScExtern);
9931 st->SetSKind(kStFunc);
9932 /* setup the type of the callee function */
9933 std::vector<TyIdx> vec;
9934 std::vector<TypeAttrs> vecAt;
9935 for (size_t i = 1; i < opndVec.size(); ++i) {
9936 (void)vec.emplace_back(GlobalTables::GetTypeTable().GetTypeTable()[static_cast<size_t>(pt[i])]->GetTypeIndex());
9937 vecAt.emplace_back(TypeAttrs());
9938 }
9939
9940 MIRType *retType = GlobalTables::GetTypeTable().GetTypeTable().at(static_cast<size_t>(retPrimType));
9941 st->SetTyIdx(GetBecommon().BeGetOrCreateFunctionType(retType->GetTypeIndex(), vec, vecAt)->GetTypeIndex());
9942
9943 if (GetCG()->GenerateVerboseCG()) {
9944 const std::string &comment = "lib call : " + newName;
9945 GetCurBB()->AppendInsn(CreateCommentInsn(comment));
9946 }
9947 // only create c lib call here
9948 AArch64CallConvImpl parmLocator(GetBecommon());
9949 CCLocInfo ploc;
9950 /* setup actual parameters */
9951 ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
9952 for (size_t i = 1; i < opndVec.size(); ++i) {
9953 DEBUG_ASSERT(pt[i] != PTY_void, "primType check");
9954 MIRType *ty;
9955 ty = GlobalTables::GetTypeTable().GetTypeTable()[static_cast<size_t>(pt[i])];
9956 Operand *stOpnd = opndVec[i];
9957 if (stOpnd->GetKind() != Operand::kOpdRegister) {
9958 stOpnd = &SelectCopy(*stOpnd, pt[i], pt[i]);
9959 }
9960 RegOperand *expRegOpnd = static_cast<RegOperand *>(stOpnd);
9961 parmLocator.LocateNextParm(*ty, ploc);
9962 if (ploc.reg0 != 0) { /* load to the register */
9963 RegOperand &parmRegOpnd = GetOrCreatePhysicalRegisterOperand(
9964 static_cast<AArch64reg>(ploc.reg0), expRegOpnd->GetSize(), GetRegTyFromPrimTy(pt[i]));
9965 SelectCopy(parmRegOpnd, pt[i], *expRegOpnd, pt[i]);
9966 srcOpnds->PushOpnd(parmRegOpnd);
9967 }
9968 DEBUG_ASSERT(ploc.reg1 == 0, "SelectCall NYI");
9969 }
9970
9971 MIRSymbol *sym = GetFunction().GetLocalOrGlobalSymbol(st->GetStIdx(), false);
9972 Insn &callInsn = AppendCall(*sym, *srcOpnds);
9973 MIRType *callRetType = GlobalTables::GetTypeTable().GetTypeTable().at(static_cast<int32>(retPrimType));
9974 if (callRetType != nullptr) {
9975 callInsn.SetRetSize(static_cast<uint32>(callRetType->GetSize()));
9976 callInsn.SetIsCallReturnUnsigned(IsUnsignedInteger(callRetType->GetPrimType()));
9977 }
9978 GetFunction().SetHasCall();
9979 /* get return value */
9980 Operand *opnd0 = opndVec[0];
9981 CCLocInfo retMech;
9982 parmLocator.InitReturnInfo(*(GlobalTables::GetTypeTable().GetTypeTable().at(retPrimType)), retMech);
9983 if (retMech.GetRegCount() <= 0) {
9984 CHECK_FATAL(false, "should return from register");
9985 }
9986 if (!opnd0->IsRegister()) {
9987 CHECK_FATAL(false, "nyi");
9988 }
9989 RegOperand *regOpnd = static_cast<RegOperand *>(opnd0);
9990 AArch64reg regNum = static_cast<AArch64reg>(is2ndRet ? retMech.GetReg1() : retMech.GetReg0());
9991 if (regOpnd->GetRegisterNumber() != regNum) {
9992 RegOperand &retOpnd =
9993 GetOrCreatePhysicalRegisterOperand(regNum, regOpnd->GetSize(), GetRegTyFromPrimTy(retPrimType));
9994 SelectCopy(*opnd0, retPrimType, retOpnd, retPrimType);
9995 }
9996 }
9997
GetBaseReg(const AArch64SymbolAlloc & symAlloc)9998 Operand *AArch64CGFunc::GetBaseReg(const AArch64SymbolAlloc &symAlloc)
9999 {
10000 MemSegmentKind sgKind = symAlloc.GetMemSegment()->GetMemSegmentKind();
10001 DEBUG_ASSERT(((sgKind == kMsArgsRegPassed) || (sgKind == kMsLocals) || (sgKind == kMsRefLocals) ||
10002 (sgKind == kMsArgsToStkPass) || (sgKind == kMsArgsStkPassed)),
10003 "NYI");
10004
10005 if (sgKind == kMsArgsStkPassed) {
10006 return &GetOrCreatevaryreg();
10007 }
10008
10009 if (fsp == nullptr) {
10010 fsp = &GetOrCreatePhysicalRegisterOperand(RFP, GetPointerSize() * kBitsPerByte, kRegTyInt);
10011 }
10012 return fsp;
10013 }
10014
GetBaseOffset(const SymbolAlloc & sa)10015 int32 AArch64CGFunc::GetBaseOffset(const SymbolAlloc &sa)
10016 {
10017 const AArch64SymbolAlloc *symAlloc = static_cast<const AArch64SymbolAlloc *>(&sa);
10018 /* Call Frame layout of AArch64
10019 * Refer to V2 in aarch64_memlayout.h.
10020 * Do Not change this unless you know what you do
10021 */
10022 const int32 sizeofFplr = 2 * kIntregBytelen;
10023 MemSegmentKind sgKind = symAlloc->GetMemSegment()->GetMemSegmentKind();
10024 AArch64MemLayout *memLayout = static_cast<AArch64MemLayout *>(this->GetMemlayout());
10025 if (sgKind == kMsArgsStkPassed) { /* for callees */
10026 int32 offset = static_cast<int32>(symAlloc->GetOffset());
10027 return offset;
10028 } else if (sgKind == kMsArgsRegPassed) {
10029 int32 baseOffset = memLayout->GetSizeOfLocals() + symAlloc->GetOffset() + memLayout->GetSizeOfRefLocals();
10030 return baseOffset + sizeofFplr;
10031 } else if (sgKind == kMsRefLocals) {
10032 int32 baseOffset = symAlloc->GetOffset() + memLayout->GetSizeOfLocals();
10033 return baseOffset + sizeofFplr;
10034 } else if (sgKind == kMsLocals) {
10035 int32 baseOffset = symAlloc->GetOffset();
10036 return baseOffset + sizeofFplr;
10037 } else if (sgKind == kMsSpillReg) {
10038 if (GetCG()->IsLmbc()) {
10039 return symAlloc->GetOffset() + memLayout->SizeOfArgsToStackPass();
10040 }
10041 int32 baseOffset = symAlloc->GetOffset() + memLayout->SizeOfArgsRegisterPassed() +
10042 memLayout->GetSizeOfLocals() + memLayout->GetSizeOfRefLocals();
10043 return baseOffset + sizeofFplr;
10044 } else if (sgKind == kMsArgsToStkPass) { /* this is for callers */
10045 return static_cast<int32>(symAlloc->GetOffset());
10046 } else {
10047 CHECK_FATAL(false, "sgKind check");
10048 }
10049 return 0;
10050 }
10051
AppendCall(const MIRSymbol & funcSymbol)10052 void AArch64CGFunc::AppendCall(const MIRSymbol &funcSymbol)
10053 {
10054 ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
10055 AppendCall(funcSymbol, *srcOpnds);
10056 }
10057
DBGFixCallFrameLocationOffsets()10058 void AArch64CGFunc::DBGFixCallFrameLocationOffsets()
10059 {
10060 for (DBGExprLoc *el : GetDbgCallFrameLocations()) {
10061 if (el->GetSimpLoc()->GetDwOp() == DW_OP_fbreg) {
10062 SymbolAlloc *symloc = static_cast<SymbolAlloc *>(el->GetSymLoc());
10063 int32 offset = GetBaseOffset(*symloc) - GetDbgCallFrameOffset();
10064 el->SetFboffset(offset);
10065 }
10066 }
10067 }
10068
SelectAddAfterInsn(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType,bool isDest,Insn & insn)10069 void AArch64CGFunc::SelectAddAfterInsn(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType, bool isDest,
10070 Insn &insn)
10071 {
10072 uint32 dsize = GetPrimTypeBitSize(primType);
10073 bool is64Bits = (dsize == k64BitSize);
10074 DEBUG_ASSERT(opnd0.GetKind() == Operand::kOpdRegister, "Spill memory operand should based on register");
10075 DEBUG_ASSERT((opnd1.GetKind() == Operand::kOpdImmediate || opnd1.GetKind() == Operand::kOpdOffset),
10076 "Spill memory operand should be with a immediate offset.");
10077
10078 ImmOperand *immOpnd = static_cast<ImmOperand *>(&opnd1);
10079
10080 MOperator mOpCode = MOP_undef;
10081 Insn *curInsn = &insn;
10082 /* lower 24 bits has 1, higher bits are all 0 */
10083 if (immOpnd->IsInBitSize(kMaxImmVal24Bits, 0)) {
10084 /* lower 12 bits and higher 12 bits both has 1 */
10085 Operand *newOpnd0 = &opnd0;
10086 if (!(immOpnd->IsInBitSize(kMaxImmVal12Bits, 0) || immOpnd->IsInBitSize(kMaxImmVal12Bits, kMaxImmVal12Bits))) {
10087 /* process higher 12 bits */
10088 ImmOperand &immOpnd2 =
10089 CreateImmOperand(static_cast<int64>(static_cast<uint64>(immOpnd->GetValue()) >> kMaxImmVal12Bits),
10090 immOpnd->GetSize(), immOpnd->IsSignedValue());
10091 mOpCode = is64Bits ? MOP_xaddrri24 : MOP_waddrri24;
10092 BitShiftOperand &shiftopnd = CreateBitShiftOperand(BitShiftOperand::kLSL, kShiftAmount12, k64BitSize);
10093 Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, opnd0, immOpnd2, shiftopnd);
10094 DEBUG_ASSERT(IsOperandImmValid(mOpCode, &immOpnd2, kInsnThirdOpnd), "immOpnd2 appears invalid");
10095 if (isDest) {
10096 insn.GetBB()->InsertInsnAfter(insn, newInsn);
10097 } else {
10098 insn.GetBB()->InsertInsnBefore(insn, newInsn);
10099 }
10100 /* get lower 12 bits value */
10101 immOpnd->ModuloByPow2(static_cast<int32>(kMaxImmVal12Bits));
10102 newOpnd0 = &resOpnd;
10103 curInsn = &newInsn;
10104 }
10105 /* process lower 12 bits value */
10106 mOpCode = is64Bits ? MOP_xaddrri12 : MOP_waddrri12;
10107 Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, *newOpnd0, *immOpnd);
10108 DEBUG_ASSERT(IsOperandImmValid(mOpCode, immOpnd, kInsnThirdOpnd), "immOpnd appears invalid");
10109 if (isDest) {
10110 insn.GetBB()->InsertInsnAfter(*curInsn, newInsn);
10111 } else {
10112 insn.GetBB()->InsertInsnBefore(insn, newInsn);
10113 }
10114 } else {
10115 /* load into register */
10116 RegOperand &movOpnd = GetOrCreatePhysicalRegisterOperand(R16, dsize, kRegTyInt);
10117 mOpCode = is64Bits ? MOP_xmovri64 : MOP_wmovri32;
10118 Insn &movInsn = GetInsnBuilder()->BuildInsn(mOpCode, movOpnd, *immOpnd);
10119 mOpCode = is64Bits ? MOP_xaddrrr : MOP_waddrrr;
10120 Insn &newInsn = GetInsnBuilder()->BuildInsn(mOpCode, resOpnd, opnd0, movOpnd);
10121 if (isDest) {
10122 (void)insn.GetBB()->InsertInsnAfter(insn, newInsn);
10123 (void)insn.GetBB()->InsertInsnAfter(insn, movInsn);
10124 } else {
10125 (void)insn.GetBB()->InsertInsnBefore(insn, movInsn);
10126 (void)insn.GetBB()->InsertInsnBefore(insn, newInsn);
10127 }
10128 }
10129 }
10130
AdjustMemOperandIfOffsetOutOfRange(MemOperand * memOpnd,regno_t vrNum,bool isDest,Insn & insn,AArch64reg regNum,bool & isOutOfRange)10131 MemOperand *AArch64CGFunc::AdjustMemOperandIfOffsetOutOfRange(MemOperand *memOpnd, regno_t vrNum, bool isDest,
10132 Insn &insn, AArch64reg regNum, bool &isOutOfRange)
10133 {
10134 if (vrNum >= vRegTable.size()) {
10135 CHECK_FATAL(false, "index out of range in AArch64CGFunc::AdjustMemOperandIfOffsetOutOfRange");
10136 }
10137 uint32 dataSize = GetOrCreateVirtualRegisterOperand(vrNum).GetSize();
10138 if (IsImmediateOffsetOutOfRange(*memOpnd, dataSize) && CheckIfSplitOffsetWithAdd(*memOpnd, dataSize)) {
10139 isOutOfRange = true;
10140 memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, dataSize, regNum, isDest, &insn);
10141 } else {
10142 isOutOfRange = false;
10143 }
10144 return memOpnd;
10145 }
10146
FreeSpillRegMem(regno_t vrNum)10147 void AArch64CGFunc::FreeSpillRegMem(regno_t vrNum)
10148 {
10149 MemOperand *memOpnd = nullptr;
10150
10151 auto p = spillRegMemOperands.find(vrNum);
10152 if (p != spillRegMemOperands.end()) {
10153 memOpnd = p->second;
10154 }
10155
10156 if ((memOpnd == nullptr) && IsVRegNOForPseudoRegister(vrNum)) {
10157 auto pSecond = pRegSpillMemOperands.find(GetPseudoRegIdxFromVirtualRegNO(vrNum));
10158 if (pSecond != pRegSpillMemOperands.end()) {
10159 memOpnd = pSecond->second;
10160 }
10161 }
10162
10163 if (memOpnd == nullptr) {
10164 DEBUG_ASSERT(false, "free spillreg have no mem");
10165 return;
10166 }
10167
10168 uint32 size = memOpnd->GetSize();
10169 MapleUnorderedMap<uint32, SpillMemOperandSet *>::iterator iter;
10170 if ((iter = reuseSpillLocMem.find(size)) != reuseSpillLocMem.end()) {
10171 iter->second->Add(*memOpnd);
10172 } else {
10173 reuseSpillLocMem[size] = memPool->New<SpillMemOperandSet>(*GetFuncScopeAllocator());
10174 reuseSpillLocMem[size]->Add(*memOpnd);
10175 }
10176 }
10177
GetOrCreatSpillMem(regno_t vrNum,uint32 memSize)10178 MemOperand *AArch64CGFunc::GetOrCreatSpillMem(regno_t vrNum, uint32 memSize)
10179 {
10180 /* NOTES: must used in RA, not used in other place. */
10181 if (IsVRegNOForPseudoRegister(vrNum)) {
10182 auto p = pRegSpillMemOperands.find(GetPseudoRegIdxFromVirtualRegNO(vrNum));
10183 if (p != pRegSpillMemOperands.end()) {
10184 return p->second;
10185 }
10186 }
10187
10188 auto p = spillRegMemOperands.find(vrNum);
10189 if (p == spillRegMemOperands.end()) {
10190 if (vrNum >= vRegTable.size()) {
10191 CHECK_FATAL(false, "index out of range in AArch64CGFunc::FreeSpillRegMem");
10192 }
10193 uint32 memBitSize = (memSize <= k32BitSize) ? k32BitSize :
10194 (memSize <= k64BitSize) ? k64BitSize : k128BitSize;
10195 auto it = reuseSpillLocMem.find(memBitSize);
10196 if (it != reuseSpillLocMem.end()) {
10197 MemOperand *memOpnd = it->second->GetOne();
10198 if (memOpnd != nullptr) {
10199 (void)spillRegMemOperands.emplace(std::pair<regno_t, MemOperand*>(vrNum, memOpnd));
10200 return memOpnd;
10201 }
10202 }
10203
10204 RegOperand &baseOpnd = GetOrCreateStackBaseRegOperand();
10205 int64 offset = GetOrCreatSpillRegLocation(vrNum, memBitSize / kBitsPerByte);
10206 MemOperand *memOpnd = nullptr;
10207 OfstOperand *offsetOpnd = &CreateOfstOpnd(static_cast<uint64>(offset), k64BitSize);
10208 memOpnd = CreateMemOperand(MemOperand::kAddrModeBOi, memBitSize, baseOpnd, nullptr, offsetOpnd, nullptr);
10209 (void)spillRegMemOperands.emplace(std::pair<regno_t, MemOperand*>(vrNum, memOpnd));
10210 return memOpnd;
10211 } else {
10212 return p->second;
10213 }
10214 }
10215
GetPseudoRegisterSpillMemoryOperand(PregIdx i)10216 MemOperand *AArch64CGFunc::GetPseudoRegisterSpillMemoryOperand(PregIdx i)
10217 {
10218 MapleUnorderedMap<PregIdx, MemOperand *>::iterator p;
10219 if (GetCG()->GetOptimizeLevel() == CGOptions::kLevel0) {
10220 p = pRegSpillMemOperands.end();
10221 } else {
10222 p = pRegSpillMemOperands.find(i);
10223 }
10224 if (p != pRegSpillMemOperands.end()) {
10225 return p->second;
10226 }
10227 int64 offset = GetPseudoRegisterSpillLocation(i);
10228 MIRPreg *preg = GetFunction().GetPregTab()->PregFromPregIdx(i);
10229 uint32 bitLen = GetPrimTypeSize(preg->GetPrimType()) * kBitsPerByte;
10230 RegOperand &base = GetOrCreateFramePointerRegOperand();
10231
10232 OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(static_cast<uint64>(offset), k32BitSize);
10233 MemOperand &memOpnd = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, bitLen, &base, nullptr, &ofstOpnd, nullptr);
10234 if (IsImmediateOffsetOutOfRange(memOpnd, bitLen)) {
10235 MemOperand &newMemOpnd = SplitOffsetWithAddInstruction(memOpnd, bitLen);
10236 (void)pRegSpillMemOperands.emplace(std::pair<PregIdx, MemOperand *>(i, &newMemOpnd));
10237 return &newMemOpnd;
10238 }
10239 (void)pRegSpillMemOperands.emplace(std::pair<PregIdx, MemOperand *>(i, &memOpnd));
10240 return &memOpnd;
10241 }
10242
GetPseudoRegFromVirtualRegNO(const regno_t vRegNO,bool afterSSA) const10243 MIRPreg *AArch64CGFunc::GetPseudoRegFromVirtualRegNO(const regno_t vRegNO, bool afterSSA) const
10244 {
10245 PregIdx pri = afterSSA ? VRegNOToPRegIdx(vRegNO) : GetPseudoRegIdxFromVirtualRegNO(vRegNO);
10246 if (pri == -1)
10247 return nullptr;
10248 return GetFunction().GetPregTab()->PregFromPregIdx(pri);
10249 }
10250
10251 /* Get the number of return register of current function. */
GetReturnRegisterNumber()10252 AArch64reg AArch64CGFunc::GetReturnRegisterNumber()
10253 {
10254 CCImpl &retLocator = *GetOrCreateLocator(GetCurCallConvKind());
10255 CCLocInfo retMech;
10256 retLocator.InitReturnInfo(*(GetFunction().GetReturnType()), retMech);
10257 if (retMech.GetRegCount() > 0) {
10258 return static_cast<AArch64reg>(retMech.GetReg0());
10259 }
10260 return kRinvalid;
10261 }
10262
CanLazyBinding(const Insn & ldrInsn) const10263 bool AArch64CGFunc::CanLazyBinding(const Insn &ldrInsn) const
10264 {
10265 Operand &memOpnd = ldrInsn.GetOperand(1);
10266 auto &aarchMemOpnd = static_cast<MemOperand &>(memOpnd);
10267 if (aarchMemOpnd.GetAddrMode() != MemOperand::kAddrModeLo12Li) {
10268 return false;
10269 }
10270
10271 const MIRSymbol *sym = aarchMemOpnd.GetSymbol();
10272 CHECK_FATAL(sym != nullptr, "sym can't be nullptr");
10273 if (sym->IsMuidFuncDefTab() || sym->IsMuidFuncUndefTab() || sym->IsMuidDataDefTab() || sym->IsMuidDataUndefTab() ||
10274 (sym->IsReflectionClassInfo() && !sym->IsReflectionArrayClassInfo())) {
10275 return true;
10276 }
10277
10278 return false;
10279 }
10280
10281 /*
10282 * add reg, reg, __PTR_C_STR_...
10283 * ldr reg1, [reg]
10284 * =>
10285 * ldr reg1, [reg, #:lo12:__Ptr_C_STR_...]
10286 */
ConvertAdrpl12LdrToLdr()10287 void AArch64CGFunc::ConvertAdrpl12LdrToLdr()
10288 {
10289 FOR_ALL_BB(bb, this) {
10290 FOR_BB_INSNS_SAFE(insn, bb, nextInsn) {
10291 nextInsn = insn->GetNextMachineInsn();
10292 if (nextInsn == nullptr) {
10293 break;
10294 }
10295 if (!insn->IsMachineInstruction()) {
10296 continue;
10297 }
10298 /* check first insn */
10299 MOperator thisMop = insn->GetMachineOpcode();
10300 if (thisMop != MOP_xadrpl12) {
10301 continue;
10302 }
10303 /* check second insn */
10304 MOperator nextMop = nextInsn->GetMachineOpcode();
10305 if (!(((nextMop >= MOP_wldrsb) && (nextMop <= MOP_dldp)) ||
10306 ((nextMop >= MOP_wstrb) && (nextMop <= MOP_dstp)))) {
10307 continue;
10308 }
10309
10310 /* Check if base register of nextInsn and the dest operand of insn are identical. */
10311 MemOperand *memOpnd = static_cast<MemOperand *>(nextInsn->GetMemOpnd());
10312 CHECK_FATAL(memOpnd != nullptr, "memOpnd can't be nullptr");
10313
10314 /* Only for AddrMode_B_OI addressing mode. */
10315 if (memOpnd->GetAddrMode() != MemOperand::kAddrModeBOi) {
10316 continue;
10317 }
10318
10319 /* Only for intact memory addressing. */
10320 if (!memOpnd->IsIntactIndexed()) {
10321 continue;
10322 }
10323
10324 auto ®Opnd = static_cast<RegOperand &>(insn->GetOperand(0));
10325
10326 /* Check if dest operand of insn is idential with base register of nextInsn. */
10327 RegOperand *baseReg = memOpnd->GetBaseRegister();
10328 CHECK_FATAL(baseReg != nullptr, "baseReg can't be nullptr");
10329 if (baseReg->GetRegisterNumber() != regOpnd.GetRegisterNumber()) {
10330 continue;
10331 }
10332
10333 StImmOperand &stImmOpnd = static_cast<StImmOperand &>(insn->GetOperand(kInsnThirdOpnd));
10334 OfstOperand &ofstOpnd = GetOrCreateOfstOpnd(
10335 static_cast<uint64>(stImmOpnd.GetOffset() + memOpnd->GetOffsetImmediate()->GetOffsetValue()),
10336 k32BitSize);
10337 RegOperand &newBaseOpnd = static_cast<RegOperand &>(insn->GetOperand(kInsnSecondOpnd));
10338 MemOperand &newMemOpnd = GetOrCreateMemOpnd(MemOperand::kAddrModeLo12Li, memOpnd->GetSize(), &newBaseOpnd,
10339 nullptr, &ofstOpnd, stImmOpnd.GetSymbol());
10340 nextInsn->SetOperand(1, newMemOpnd);
10341 bb->RemoveInsn(*insn);
10342 }
10343 }
10344 }
10345
10346 /*
10347 * adrp reg1, __muid_func_undef_tab..
10348 * ldr reg2, [reg1, #:lo12:__muid_func_undef_tab..]
10349 * =>
10350 * intrinsic_adrp_ldr reg2, __muid_func_undef_tab...
10351 */
ConvertAdrpLdrToIntrisic()10352 void AArch64CGFunc::ConvertAdrpLdrToIntrisic()
10353 {
10354 FOR_ALL_BB(bb, this) {
10355 FOR_BB_INSNS_SAFE(insn, bb, nextInsn) {
10356 nextInsn = insn->GetNextMachineInsn();
10357 if (nextInsn == nullptr) {
10358 break;
10359 }
10360 if (!insn->IsMachineInstruction()) {
10361 continue;
10362 }
10363
10364 MOperator firstMop = insn->GetMachineOpcode();
10365 MOperator secondMop = nextInsn->GetMachineOpcode();
10366 if (!((firstMop == MOP_xadrp) && ((secondMop == MOP_wldr) || (secondMop == MOP_xldr)))) {
10367 continue;
10368 }
10369
10370 if (CanLazyBinding(*nextInsn)) {
10371 bb->ReplaceInsn(
10372 *insn, GetInsnBuilder()->BuildInsn(MOP_adrp_ldr, nextInsn->GetOperand(0), insn->GetOperand(1)));
10373 bb->RemoveInsn(*nextInsn);
10374 }
10375 }
10376 }
10377 }
10378
ProcessLazyBinding()10379 void AArch64CGFunc::ProcessLazyBinding()
10380 {
10381 ConvertAdrpl12LdrToLdr();
10382 ConvertAdrpLdrToIntrisic();
10383 }
10384
10385 /*
10386 * Generate global long call
10387 * adrp VRx, symbol
10388 * ldr VRx, [VRx, #:lo12:symbol]
10389 * blr VRx
10390 *
10391 * Input:
10392 * insn : insert new instruction after the 'insn'
10393 * func : the symbol of the function need to be called
10394 * srcOpnds : list operand of the function need to be called
10395 * isCleanCall: when generate clean call insn, set isCleanCall as true
10396 * Return: the 'blr' instruction
10397 */
GenerateGlobalLongCallAfterInsn(const MIRSymbol & func,ListOperand & srcOpnds)10398 Insn &AArch64CGFunc::GenerateGlobalLongCallAfterInsn(const MIRSymbol &func, ListOperand &srcOpnds)
10399 {
10400 MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(func.GetStIdx());
10401 symbol->SetStorageClass(kScGlobal);
10402 RegOperand &tmpReg = CreateRegisterOperandOfType(PTY_u64);
10403 StImmOperand &stOpnd = CreateStImmOperand(*symbol, 0, 0);
10404 OfstOperand &offsetOpnd = CreateOfstOpnd(*symbol, 0);
10405 Insn &adrpInsn = GetInsnBuilder()->BuildInsn(MOP_xadrp, tmpReg, stOpnd);
10406 GetCurBB()->AppendInsn(adrpInsn);
10407 MemOperand &memOrd = GetOrCreateMemOpnd(MemOperand::kAddrModeLo12Li, GetPointerSize() * kBitsPerByte,
10408 static_cast<RegOperand *>(&tmpReg), nullptr, &offsetOpnd, symbol);
10409 Insn &ldrInsn = GetInsnBuilder()->BuildInsn(memOrd.GetSize() == k64BitSize ? MOP_xldr : MOP_wldr, tmpReg, memOrd);
10410 GetCurBB()->AppendInsn(ldrInsn);
10411
10412 Insn &callInsn = GetInsnBuilder()->BuildInsn(MOP_xblr, tmpReg, srcOpnds);
10413 GetCurBB()->AppendInsn(callInsn);
10414 GetCurBB()->SetHasCall();
10415 return callInsn;
10416 }
10417
10418 /*
10419 * Generate local long call
10420 * adrp VRx, symbol
10421 * add VRx, VRx, #:lo12:symbol
10422 * blr VRx
10423 *
10424 * Input:
10425 * insn : insert new instruction after the 'insn'
10426 * func : the symbol of the function need to be called
10427 * srcOpnds : list operand of the function need to be called
10428 * isCleanCall: when generate clean call insn, set isCleanCall as true
10429 * Return: the 'blr' instruction
10430 */
GenerateLocalLongCallAfterInsn(const MIRSymbol & func,ListOperand & srcOpnds)10431 Insn &AArch64CGFunc::GenerateLocalLongCallAfterInsn(const MIRSymbol &func, ListOperand &srcOpnds)
10432 {
10433 RegOperand &tmpReg = CreateRegisterOperandOfType(PTY_u64);
10434 StImmOperand &stOpnd = CreateStImmOperand(func, 0, 0);
10435 Insn &adrpInsn = GetInsnBuilder()->BuildInsn(MOP_xadrp, tmpReg, stOpnd);
10436 GetCurBB()->AppendInsn(adrpInsn);
10437 Insn &addInsn = GetInsnBuilder()->BuildInsn(MOP_xadrpl12, tmpReg, tmpReg, stOpnd);
10438 GetCurBB()->AppendInsn(addInsn);
10439 Insn *callInsn = &GetInsnBuilder()->BuildInsn(MOP_xblr, tmpReg, srcOpnds);
10440 GetCurBB()->AppendInsn(*callInsn);
10441 GetCurBB()->SetHasCall();
10442 return *callInsn;
10443 }
10444
AppendCall(const MIRSymbol & sym,ListOperand & srcOpnds)10445 Insn &AArch64CGFunc::AppendCall(const MIRSymbol &sym, ListOperand &srcOpnds)
10446 {
10447 Insn *callInsn = nullptr;
10448 if (CGOptions::IsLongCalls()) {
10449 MIRFunction *mirFunc = sym.GetFunction();
10450 if (IsDuplicateAsmList(sym) || (mirFunc && mirFunc->GetAttr(FUNCATTR_local))) {
10451 callInsn = &GenerateLocalLongCallAfterInsn(sym, srcOpnds);
10452 } else {
10453 callInsn = &GenerateGlobalLongCallAfterInsn(sym, srcOpnds);
10454 }
10455 } else {
10456 Operand &targetOpnd = GetOrCreateFuncNameOpnd(sym);
10457 callInsn = &GetInsnBuilder()->BuildInsn(MOP_xbl, targetOpnd, srcOpnds);
10458 GetCurBB()->AppendInsn(*callInsn);
10459 GetCurBB()->SetHasCall();
10460 }
10461 return *callInsn;
10462 }
10463
IsDuplicateAsmList(const MIRSymbol & sym) const10464 bool AArch64CGFunc::IsDuplicateAsmList(const MIRSymbol &sym) const
10465 {
10466 if (CGOptions::IsDuplicateAsmFileEmpty()) {
10467 return false;
10468 }
10469
10470 const std::string &name = sym.GetName();
10471 if ((name == "strlen") || (name == "strncmp") || (name == "memcpy") || (name == "memmove") || (name == "strcmp") ||
10472 (name == "memcmp") || (name == "memcmpMpl")) {
10473 return true;
10474 }
10475 return false;
10476 }
10477
SelectMPLProfCounterInc(const IntrinsiccallNode & intrnNode)10478 void AArch64CGFunc::SelectMPLProfCounterInc(const IntrinsiccallNode &intrnNode)
10479 {
10480 if (Options::profileGen) {
10481 DEBUG_ASSERT(intrnNode.NumOpnds() == 1, "must be 1 operand");
10482 BaseNode *arg1 = intrnNode.Opnd(0);
10483 DEBUG_ASSERT(arg1 != nullptr, "nullptr check");
10484 regno_t vRegNO1 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
10485 RegOperand &vReg1 = CreateVirtualRegisterOperand(vRegNO1);
10486 vReg1.SetRegNotBBLocal();
10487 static const MIRSymbol *bbProfileTab = nullptr;
10488
10489 // Ref: MeProfGen::InstrumentFunc on ctrTbl namiLogicalShiftLeftOperandng
10490 std::string ctrTblName = namemangler::kprefixProfCtrTbl + GetMirModule().GetFileName() + "_" + GetName();
10491 std::replace(ctrTblName.begin(), ctrTblName.end(), '.', '_');
10492 std::replace(ctrTblName.begin(), ctrTblName.end(), '-', '_');
10493 std::replace(ctrTblName.begin(), ctrTblName.end(), '/', '_');
10494
10495 if (!bbProfileTab || bbProfileTab->GetName() != ctrTblName) {
10496 bbProfileTab = GetMirModule().GetMIRBuilder()->GetGlobalDecl(ctrTblName);
10497 CHECK_FATAL(bbProfileTab != nullptr, "expect counter table");
10498 }
10499
10500 ConstvalNode *constvalNode = static_cast<ConstvalNode *>(arg1);
10501 MIRConst *mirConst = constvalNode->GetConstVal();
10502 DEBUG_ASSERT(mirConst != nullptr, "nullptr check");
10503 CHECK_FATAL(mirConst->GetKind() == kConstInt, "expect MIRIntConst type");
10504 MIRIntConst *mirIntConst = safe_cast<MIRIntConst>(mirConst);
10505 int64 offset = GetPrimTypeSize(PTY_u64) * mirIntConst->GetExtValue();
10506
10507 if (!CGOptions::IsQuiet()) {
10508 maple::LogInfo::MapleLogger(kLlInfo) << "At counter table offset: " << offset << std::endl;
10509 }
10510 MemOperand *memOpnd = &GetOrCreateMemOpnd(*bbProfileTab, offset, k64BitSize);
10511 if (IsImmediateOffsetOutOfRange(*memOpnd, k64BitSize)) {
10512 memOpnd = &SplitOffsetWithAddInstruction(*memOpnd, k64BitSize);
10513 }
10514 Operand *reg = &SelectCopy(*memOpnd, PTY_u64, PTY_u64);
10515 ImmOperand &one = CreateImmOperand(1, k64BitSize, false);
10516 SelectAdd(*reg, *reg, one, PTY_u64);
10517 SelectCopy(*memOpnd, PTY_u64, *reg, PTY_u64);
10518 return;
10519 }
10520
10521 DEBUG_ASSERT(intrnNode.NumOpnds() == 1, "must be 1 operand");
10522 BaseNode *arg1 = intrnNode.Opnd(0);
10523 DEBUG_ASSERT(arg1 != nullptr, "nullptr check");
10524 regno_t vRegNO1 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
10525 RegOperand &vReg1 = CreateVirtualRegisterOperand(vRegNO1);
10526 vReg1.SetRegNotBBLocal();
10527 static const MIRSymbol *bbProfileTab = nullptr;
10528 if (!bbProfileTab) {
10529 std::string bbProfileName = namemangler::kBBProfileTabPrefixStr + GetMirModule().GetFileNameAsPostfix();
10530 bbProfileTab = GetMirModule().GetMIRBuilder()->GetGlobalDecl(bbProfileName);
10531 CHECK_FATAL(bbProfileTab != nullptr, "expect bb profile tab");
10532 }
10533 ConstvalNode *constvalNode = static_cast<ConstvalNode *>(arg1);
10534 MIRConst *mirConst = constvalNode->GetConstVal();
10535 DEBUG_ASSERT(mirConst != nullptr, "nullptr check");
10536 CHECK_FATAL(mirConst->GetKind() == kConstInt, "expect MIRIntConst type");
10537 MIRIntConst *mirIntConst = safe_cast<MIRIntConst>(mirConst);
10538 int64 idx = GetPrimTypeSize(PTY_u32) * mirIntConst->GetExtValue();
10539 if (!CGOptions::IsQuiet()) {
10540 maple::LogInfo::MapleLogger(kLlErr) << "Id index " << idx << std::endl;
10541 }
10542 StImmOperand &stOpnd = CreateStImmOperand(*bbProfileTab, idx, 0);
10543 Insn &newInsn = GetInsnBuilder()->BuildInsn(MOP_counter, vReg1, stOpnd);
10544 newInsn.SetDoNotRemove(true);
10545 GetCurBB()->AppendInsn(newInsn);
10546 }
10547
SelectMPLClinitCheck(const IntrinsiccallNode & intrnNode)10548 void AArch64CGFunc::SelectMPLClinitCheck(const IntrinsiccallNode &intrnNode)
10549 {
10550 DEBUG_ASSERT(intrnNode.NumOpnds() == 1, "must be 1 operand");
10551 BaseNode *arg = intrnNode.Opnd(0);
10552 Operand *stOpnd = nullptr;
10553 bool bClinitSeperate = false;
10554 DEBUG_ASSERT(CGOptions::IsPIC(), "must be doPIC");
10555 if (arg->GetOpCode() == OP_addrof) {
10556 AddrofNode *addrof = static_cast<AddrofNode *>(arg);
10557 MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(addrof->GetStIdx());
10558 DEBUG_ASSERT(symbol->GetName().find(CLASSINFO_PREFIX_STR) == 0, "must be a symbol with __classinfo__");
10559
10560 if (!symbol->IsMuidDataUndefTab()) {
10561 std::string ptrName = namemangler::kPtrPrefixStr + symbol->GetName();
10562 MIRType *ptrType = GlobalTables::GetTypeTable().GetPtr();
10563 symbol = GetMirModule().GetMIRBuilder()->GetOrCreateGlobalDecl(ptrName, *ptrType);
10564 bClinitSeperate = true;
10565 symbol->SetStorageClass(kScFstatic);
10566 }
10567 stOpnd = &CreateStImmOperand(*symbol, 0, 0);
10568 } else {
10569 arg = arg->Opnd(0);
10570 BaseNode *arg0 = arg->Opnd(0);
10571 BaseNode *arg1 = arg->Opnd(1);
10572 DEBUG_ASSERT(arg0 != nullptr, "nullptr check");
10573 DEBUG_ASSERT(arg1 != nullptr, "nullptr check");
10574 DEBUG_ASSERT(arg0->GetOpCode() == OP_addrof, "expect the operand to be addrof");
10575 AddrofNode *addrof = static_cast<AddrofNode *>(arg0);
10576 MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(addrof->GetStIdx());
10577 DEBUG_ASSERT(addrof->GetFieldID() == 0, "For debug SelectMPLClinitCheck.");
10578 ConstvalNode *constvalNode = static_cast<ConstvalNode *>(arg1);
10579 MIRConst *mirConst = constvalNode->GetConstVal();
10580 DEBUG_ASSERT(mirConst != nullptr, "nullptr check");
10581 CHECK_FATAL(mirConst->GetKind() == kConstInt, "expect MIRIntConst type");
10582 MIRIntConst *mirIntConst = safe_cast<MIRIntConst>(mirConst);
10583 stOpnd = &CreateStImmOperand(*symbol, mirIntConst->GetExtValue(), 0);
10584 }
10585
10586 regno_t vRegNO2 = NewVReg(GetRegTyFromPrimTy(PTY_a64), GetPrimTypeSize(PTY_a64));
10587 RegOperand &vReg2 = CreateVirtualRegisterOperand(vRegNO2);
10588 vReg2.SetRegNotBBLocal();
10589 if (bClinitSeperate) {
10590 /* Seperate MOP_clinit to MOP_adrp_ldr + MOP_clinit_tail. */
10591 Insn &newInsn = GetInsnBuilder()->BuildInsn(MOP_adrp_ldr, vReg2, *stOpnd);
10592 GetCurBB()->AppendInsn(newInsn);
10593 newInsn.SetDoNotRemove(true);
10594 Insn &insn = GetInsnBuilder()->BuildInsn(MOP_clinit_tail, vReg2);
10595 insn.SetDoNotRemove(true);
10596 GetCurBB()->AppendInsn(insn);
10597 } else {
10598 Insn &newInsn = GetInsnBuilder()->BuildInsn(MOP_clinit, vReg2, *stOpnd);
10599 GetCurBB()->AppendInsn(newInsn);
10600 }
10601 }
GenCVaStartIntrin(RegOperand & opnd,uint32 stkSize)10602 void AArch64CGFunc::GenCVaStartIntrin(RegOperand &opnd, uint32 stkSize)
10603 {
10604 /* FPLR only pushed in regalloc() after intrin function */
10605 Operand &stkOpnd = GetOrCreatePhysicalRegisterOperand(RFP, k64BitSize, kRegTyInt);
10606
10607 /* __stack */
10608 ImmOperand *offsOpnd;
10609 if (GetMirModule().GetFlavor() != MIRFlavor::kFlavorLmbc) {
10610 offsOpnd = &CreateImmOperand(0, k64BitSize, true, kUnAdjustVary); /* isvary reset StackFrameSize */
10611 } else {
10612 offsOpnd = &CreateImmOperand(0, k64BitSize, true);
10613 }
10614 ImmOperand *offsOpnd2 = &CreateImmOperand(stkSize, k64BitSize, false);
10615 RegOperand &vReg = CreateVirtualRegisterOperand(NewVReg(kRegTyInt, GetPrimTypeSize(GetLoweredPtrType())));
10616 if (stkSize) {
10617 SelectAdd(vReg, *offsOpnd, *offsOpnd2, GetLoweredPtrType());
10618 SelectAdd(vReg, stkOpnd, vReg, GetLoweredPtrType());
10619 } else {
10620 SelectAdd(vReg, stkOpnd, *offsOpnd, GetLoweredPtrType()); /* stack pointer */
10621 }
10622 OfstOperand *offOpnd = &GetOrCreateOfstOpnd(0, k64BitSize); /* va_list ptr */
10623 /* mem operand in va_list struct (lhs) */
10624 MemOperand *strOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, &opnd, nullptr, offOpnd,
10625 static_cast<MIRSymbol *>(nullptr));
10626 GetCurBB()->AppendInsn(
10627 GetInsnBuilder()->BuildInsn(vReg.GetSize() == k64BitSize ? MOP_xstr : MOP_wstr, vReg, *strOpnd));
10628
10629 /* __gr_top ; it's the same as __stack before the 1st va_arg */
10630 if (CGOptions::IsArm64ilp32()) {
10631 offOpnd = &GetOrCreateOfstOpnd(GetPointerSize(), k64BitSize);
10632 } else {
10633 offOpnd = &GetOrCreateOfstOpnd(k8BitSize, k64BitSize);
10634 }
10635 strOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, &opnd, nullptr, offOpnd,
10636 static_cast<MIRSymbol *>(nullptr));
10637 SelectAdd(vReg, stkOpnd, *offsOpnd, GetLoweredPtrType());
10638 GetCurBB()->AppendInsn(
10639 GetInsnBuilder()->BuildInsn(vReg.GetSize() == k64BitSize ? MOP_xstr : MOP_wstr, vReg, *strOpnd));
10640
10641 /* __vr_top */
10642 int32 grAreaSize = static_cast<int32>(static_cast<AArch64MemLayout *>(GetMemlayout())->GetSizeOfGRSaveArea());
10643 if (CGOptions::IsArm64ilp32()) {
10644 offsOpnd2 = &CreateImmOperand(static_cast<int64>(RoundUp(static_cast<uint64>(grAreaSize), k8ByteSize << 1)),
10645 k64BitSize, false);
10646 } else {
10647 offsOpnd2 = &CreateImmOperand(
10648 static_cast<int64>(RoundUp(static_cast<uint64>(grAreaSize), GetPointerSize() << 1)), k64BitSize, false);
10649 }
10650 SelectSub(vReg, *offsOpnd, *offsOpnd2, GetLoweredPtrType()); /* if 1st opnd is register => sub */
10651 SelectAdd(vReg, stkOpnd, vReg, GetLoweredPtrType());
10652 offOpnd = &GetOrCreateOfstOpnd(GetPointerSize() << 1, k64BitSize);
10653 strOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, &opnd, nullptr, offOpnd,
10654 static_cast<MIRSymbol *>(nullptr));
10655 GetCurBB()->AppendInsn(
10656 GetInsnBuilder()->BuildInsn(vReg.GetSize() == k64BitSize ? MOP_xstr : MOP_wstr, vReg, *strOpnd));
10657
10658 /* __gr_offs */
10659 int32 offs = 0 - grAreaSize;
10660 offsOpnd = &CreateImmOperand(offs, k32BitSize, false);
10661 RegOperand *tmpReg = &CreateRegisterOperandOfType(PTY_i32); /* offs value to be assigned (rhs) */
10662 SelectCopyImm(*tmpReg, *offsOpnd, PTY_i32);
10663 // write __gr_offs : offset from gr_top to next GP register arg
10664 // accroding to typedef struct va_list
10665 // the field offset of __gr_offs is 3 pointer from beginning
10666 offOpnd = &GetOrCreateOfstOpnd(GetPointerSize() * 3, k32BitSize);
10667 strOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k32BitSize, &opnd, nullptr, offOpnd,
10668 static_cast<MIRSymbol *>(nullptr));
10669 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wstr, *tmpReg, *strOpnd));
10670
10671 /* __vr_offs */
10672 offs =
10673 static_cast<int32>(UINT32_MAX - (static_cast<AArch64MemLayout *>(GetMemlayout())->GetSizeOfVRSaveArea() - 1UL));
10674 offsOpnd = &CreateImmOperand(offs, k32BitSize, false);
10675 tmpReg = &CreateRegisterOperandOfType(PTY_i32);
10676 SelectCopyImm(*tmpReg, *offsOpnd, PTY_i32);
10677 // write __vr_offs : offset from vr_top to next FP/SIMD register arg
10678 // accroding to typedef struct va_list
10679 // the field offset of __vr_offs is 3 pointer + sizeof(int) from beginning
10680 offOpnd = &GetOrCreateOfstOpnd((GetPointerSize() * 3 + sizeof(int32)), k32BitSize);
10681 strOpnd = &GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k32BitSize, &opnd, nullptr, offOpnd,
10682 static_cast<MIRSymbol *>(nullptr));
10683 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wstr, *tmpReg, *strOpnd));
10684 }
10685
SelectCVaStart(const IntrinsiccallNode & intrnNode)10686 void AArch64CGFunc::SelectCVaStart(const IntrinsiccallNode &intrnNode)
10687 {
10688 DEBUG_ASSERT(intrnNode.NumOpnds() == 2, "must be 2 operands"); // must be 2 operands
10689 /* 2 operands, but only 1 needed. Don't need to emit code for second operand
10690 *
10691 * va_list is a passed struct with an address, load its address
10692 */
10693 isIntrnCallForC = true;
10694 BaseNode *argExpr = intrnNode.Opnd(0);
10695 Operand *opnd = HandleExpr(intrnNode, *argExpr);
10696 RegOperand &opnd0 = LoadIntoRegister(*opnd, GetLoweredPtrType()); /* first argument of intrinsic */
10697
10698 /* Find beginning of unnamed arg on stack.
10699 * Ex. void foo(int i1, int i2, ... int i8, struct S r, struct S s, ...)
10700 * where struct S has size 32, address of r and s are on stack but they are named.
10701 */
10702 AArch64CallConvImpl parmLocator(GetBecommon());
10703 CCLocInfo pLoc;
10704 uint32 stkSize = 0;
10705 uint32 inReg = 0;
10706 for (uint32 i = 0; i < GetFunction().GetFormalCount(); i++) {
10707 MIRType *ty = GlobalTables::GetTypeTable().GetTypeFromTyIdx(GetFunction().GetNthParamTyIdx(i));
10708 CHECK_FATAL(GetFunction().GetAttr(FUNCATTR_ccall), "only c calling convention support here");
10709 parmLocator.LocateNextParm(*ty, pLoc);
10710 if (pLoc.reg0 == kRinvalid) { /* on stack */
10711 stkSize = static_cast<uint32_t>(pLoc.memOffset + pLoc.memSize);
10712 } else {
10713 inReg++;
10714 }
10715 }
10716 if (GetMirModule().GetFlavor() == MIRFlavor::kFlavorLmbc) {
10717 stkSize += (inReg * k8ByteSize);
10718 }
10719 if (CGOptions::IsArm64ilp32()) {
10720 stkSize = static_cast<uint32>(RoundUp(stkSize, k8ByteSize));
10721 } else {
10722 stkSize = static_cast<uint32>(RoundUp(stkSize, GetPointerSize()));
10723 }
10724
10725 GenCVaStartIntrin(opnd0, stkSize);
10726
10727 return;
10728 }
10729
10730 // output
10731 // add_with_overflow/ sub_with_overflow:
10732 // w1: parm1
10733 // w2: parm2
10734 // adds/subs w0, w1, w2
10735 // cset w3, vs
10736
10737 // mul_with_overflow:
10738 // w1: parm1
10739 // w2: parm2
10740 // smull x0, w0, w1
10741 // cmp x0, w0, sxtw
10742 // cset w4, ne
SelectOverFlowCall(const IntrinsiccallNode & intrnNode)10743 void AArch64CGFunc::SelectOverFlowCall(const IntrinsiccallNode &intrnNode)
10744 {
10745 DEBUG_ASSERT(intrnNode.NumOpnds() == 2, "must be 2 operands"); // must be 2 operands
10746 MIRIntrinsicID intrinsic = intrnNode.GetIntrinsic();
10747 PrimType type = intrnNode.Opnd(0)->GetPrimType();
10748 PrimType type2 = intrnNode.Opnd(1)->GetPrimType();
10749 CHECK_FATAL(type == PTY_i32 || type == PTY_u32, "only support i32 or u32 here");
10750 CHECK_FATAL(type2 == PTY_i32 || type == PTY_u32, "only support i32 or u32 here");
10751 // deal with parms
10752 RegOperand &opnd0 = LoadIntoRegister(*HandleExpr(intrnNode, *intrnNode.Opnd(0)),
10753 intrnNode.Opnd(0)->GetPrimType()); /* first argument of intrinsic */
10754 RegOperand &opnd1 = LoadIntoRegister(*HandleExpr(intrnNode, *intrnNode.Opnd(1)),
10755 intrnNode.Opnd(1)->GetPrimType()); /* first argument of intrinsic */
10756 RegOperand &resReg = CreateRegisterOperandOfType(type);
10757 RegOperand &resReg2 = CreateRegisterOperandOfType(PTY_u8);
10758 Operand &rflag = GetOrCreateRflag();
10759 // arith operation with set flag
10760 if (intrinsic == INTRN_ADD_WITH_OVERFLOW) {
10761 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_waddsrrr, rflag, resReg, opnd0, opnd1));
10762 SelectAArch64CSet(resReg2, GetCondOperand(CC_VS), false);
10763 } else if (intrinsic == INTRN_SUB_WITH_OVERFLOW) {
10764 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wsubsrrr, rflag, resReg, opnd0, opnd1));
10765 SelectAArch64CSet(resReg2, GetCondOperand(CC_VS), false);
10766 } else if (intrinsic == INTRN_MUL_WITH_OVERFLOW) {
10767 // smull
10768 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xsmullrrr, resReg, opnd0, opnd1));
10769 Operand &sxtw = CreateExtendShiftOperand(ExtendShiftOperand::kSXTW, 0, k3BitSize);
10770 Insn &cmpInsn = GetInsnBuilder()->BuildInsn(MOP_xwcmprre, rflag, resReg, resReg, sxtw);
10771 GetCurBB()->AppendInsn(cmpInsn);
10772 SelectAArch64CSet(resReg2, GetCondOperand(CC_NE), false);
10773 } else {
10774 CHECK_FATAL(false, "niy");
10775 }
10776 // store back
10777 auto *retVals = &intrnNode.GetReturnVec();
10778 auto &pair = retVals->at(0);
10779 stIdx2OverflowResult[pair.first] = std::pair<RegOperand*, RegOperand*>(&resReg, &resReg2);
10780 return;
10781 }
10782
10783 /*
10784 * intrinsiccall C___Atomic_store_N(ptr, val, memorder))
10785 * ====> *ptr = val
10786 * let ptr -> x0
10787 * let val -> x1
10788 * implement to asm: str/stlr x1, [x0]
10789 * a store-release would replace str if memorder is not 0
10790 */
SelectCAtomicStoreN(const IntrinsiccallNode & intrinsiccallNode)10791 void AArch64CGFunc::SelectCAtomicStoreN(const IntrinsiccallNode &intrinsiccallNode)
10792 {
10793 auto primType = intrinsiccallNode.Opnd(1)->GetPrimType();
10794 auto *addr = HandleExpr(intrinsiccallNode, *intrinsiccallNode.Opnd(0));
10795 auto *value = HandleExpr(intrinsiccallNode, *intrinsiccallNode.Opnd(1));
10796 auto *memOrderOpnd = intrinsiccallNode.Opnd(kInsnThirdOpnd);
10797 auto *memOrderConst = static_cast<MIRIntConst *>(static_cast<ConstvalNode *>(memOrderOpnd)->GetConstVal());
10798 auto memOrder = static_cast<std::memory_order>(memOrderConst->GetExtValue());
10799 SelectAtomicStore(*value, *addr, primType, PickMemOrder(memOrder, false));
10800 }
10801
SelectAtomicStore(Operand & srcOpnd,Operand & addrOpnd,PrimType primType,AArch64isa::MemoryOrdering memOrder)10802 void AArch64CGFunc::SelectAtomicStore(Operand &srcOpnd, Operand &addrOpnd, PrimType primType,
10803 AArch64isa::MemoryOrdering memOrder)
10804 {
10805 auto &memOpnd = CreateMemOpnd(LoadIntoRegister(addrOpnd, PTY_a64), 0, k64BitSize);
10806 auto mOp = PickStInsn(GetPrimTypeBitSize(primType), primType, memOrder);
10807 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, LoadIntoRegister(srcOpnd, primType), memOpnd));
10808 }
10809
SelectAddrofThreadLocal(Operand & result,StImmOperand & stImm)10810 void AArch64CGFunc::SelectAddrofThreadLocal(Operand &result, StImmOperand &stImm)
10811 {
10812 if (CGOptions::IsPIC()) {
10813 SelectCTlsGlobalDesc(result, stImm);
10814 } else {
10815 SelectCTlsLocalDesc(result, stImm);
10816 }
10817 if (stImm.GetOffset() > 0) {
10818 auto &immOpnd = CreateImmOperand(stImm.GetOffset(), result.GetSize(), false);
10819 SelectAdd(result, result, immOpnd, PTY_u64);
10820 }
10821 }
10822
SelectCTlsLocalDesc(Operand & result,StImmOperand & stImm)10823 void AArch64CGFunc::SelectCTlsLocalDesc(Operand &result, StImmOperand &stImm)
10824 {
10825 auto tpidr = &CreateCommentOperand("tpidr_el0");
10826 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_mrs, result, *tpidr));
10827 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_tls_desc_rel, result, result, stImm));
10828 }
10829
SelectCTlsGlobalDesc(Operand & result,StImmOperand & stImm)10830 void AArch64CGFunc::SelectCTlsGlobalDesc(Operand &result, StImmOperand &stImm)
10831 {
10832 /* according to AArch64 Machine Directives */
10833 auto &r0opnd = GetOrCreatePhysicalRegisterOperand(R0, k64BitSize, GetRegTyFromPrimTy(PTY_u64));
10834 RegOperand *tlsAddr = &CreateRegisterOperandOfType(PTY_u64);
10835 RegOperand *specialFunc = &CreateRegisterOperandOfType(PTY_u64);
10836 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_tls_desc_call, r0opnd, *tlsAddr, stImm));
10837 /* release tls address */
10838 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_pseduo_tls_release, *tlsAddr));
10839 // mrs xn, tpidr_el0
10840 // add x0, x0, xn
10841 auto tpidr = &CreateCommentOperand("tpidr_el0");
10842 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_mrs, *specialFunc, *tpidr));
10843 SelectAdd(result, r0opnd, *specialFunc, PTY_u64);
10844 }
10845
SelectIntrinCall(IntrinsiccallNode & intrinsiccallNode)10846 void AArch64CGFunc::SelectIntrinCall(IntrinsiccallNode &intrinsiccallNode)
10847 {
10848 MIRIntrinsicID intrinsic = intrinsiccallNode.GetIntrinsic();
10849
10850 if (GetCG()->GenerateVerboseCG()) {
10851 std::string comment = GetIntrinsicName(intrinsic);
10852 GetCurBB()->AppendInsn(CreateCommentInsn(comment));
10853 }
10854
10855 /*
10856 * At this moment, we eagerly evaluates all argument expressions. In theory,
10857 * there could be intrinsics that extract meta-information of variables, such as
10858 * their locations, rather than computing their values. Applications
10859 * include building stack maps that help runtime libraries to find the values
10860 * of local variables (See @stackmap in LLVM), in which case knowing their
10861 * locations will suffice.
10862 */
10863 if (intrinsic == INTRN_MPL_CLINIT_CHECK) { /* special case */
10864 SelectMPLClinitCheck(intrinsiccallNode);
10865 return;
10866 }
10867 if (intrinsic == INTRN_MPL_PROF_COUNTER_INC) { /* special case */
10868 SelectMPLProfCounterInc(intrinsiccallNode);
10869 return;
10870 }
10871 if ((intrinsic == INTRN_MPL_CLEANUP_LOCALREFVARS) || (intrinsic == INTRN_MPL_CLEANUP_LOCALREFVARS_SKIP) ||
10872 (intrinsic == INTRN_MPL_CLEANUP_NORETESCOBJS)) {
10873 return;
10874 }
10875 // js
10876 if (intrinsic == INTRN_ADD_WITH_OVERFLOW || intrinsic == INTRN_SUB_WITH_OVERFLOW ||
10877 intrinsic == INTRN_MUL_WITH_OVERFLOW) {
10878 SelectOverFlowCall(intrinsiccallNode);
10879 return;
10880 }
10881 switch (intrinsic) {
10882 case INTRN_C_va_start:
10883 SelectCVaStart(intrinsiccallNode);
10884 return;
10885 case INTRN_C___sync_lock_release_1:
10886 SelectCSyncLockRelease(intrinsiccallNode, PTY_u8);
10887 return;
10888 case INTRN_C___sync_lock_release_2:
10889 SelectCSyncLockRelease(intrinsiccallNode, PTY_u16);
10890 return;
10891 case INTRN_C___sync_lock_release_4:
10892 SelectCSyncLockRelease(intrinsiccallNode, PTY_u32);
10893 return;
10894 case INTRN_C___sync_lock_release_8:
10895 SelectCSyncLockRelease(intrinsiccallNode, PTY_u64);
10896 return;
10897 case INTRN_C___atomic_store_n:
10898 SelectCAtomicStoreN(intrinsiccallNode);
10899 return;
10900 case INTRN_vector_zip_v8u8:
10901 case INTRN_vector_zip_v8i8:
10902 case INTRN_vector_zip_v4u16:
10903 case INTRN_vector_zip_v4i16:
10904 case INTRN_vector_zip_v2u32:
10905 case INTRN_vector_zip_v2i32:
10906 SelectVectorZip(intrinsiccallNode.Opnd(0)->GetPrimType(),
10907 HandleExpr(intrinsiccallNode, *intrinsiccallNode.Opnd(0)),
10908 HandleExpr(intrinsiccallNode, *intrinsiccallNode.Opnd(1)));
10909 return;
10910 case INTRN_C_stack_save:
10911 return;
10912 case INTRN_C_stack_restore:
10913 return;
10914 default:
10915 break;
10916 }
10917 std::vector<Operand *> operands; /* Temporary. Deallocated on return. */
10918 ListOperand *srcOpnds = CreateListOpnd(*GetFuncScopeAllocator());
10919 for (size_t i = 0; i < intrinsiccallNode.NumOpnds(); i++) {
10920 BaseNode *argExpr = intrinsiccallNode.Opnd(i);
10921 Operand *opnd = HandleExpr(intrinsiccallNode, *argExpr);
10922 operands.emplace_back(opnd);
10923 if (!opnd->IsRegister()) {
10924 opnd = &LoadIntoRegister(*opnd, argExpr->GetPrimType());
10925 }
10926 RegOperand *expRegOpnd = static_cast<RegOperand *>(opnd);
10927 srcOpnds->PushOpnd(*expRegOpnd);
10928 }
10929 CallReturnVector *retVals = &intrinsiccallNode.GetReturnVec();
10930
10931 switch (intrinsic) {
10932 case INTRN_MPL_ATOMIC_EXCHANGE_PTR: {
10933 BB *origFtBB = GetCurBB()->GetNext();
10934 Operand *loc = operands[kInsnFirstOpnd];
10935 Operand *newVal = operands[kInsnSecondOpnd];
10936 Operand *memOrd = operands[kInsnThirdOpnd];
10937
10938 MemOrd ord = OperandToMemOrd(*memOrd);
10939 bool isAcquire = MemOrdIsAcquire(ord);
10940 bool isRelease = MemOrdIsRelease(ord);
10941
10942 const PrimType kValPrimType = PTY_a64;
10943
10944 RegOperand &locReg = LoadIntoRegister(*loc, PTY_a64);
10945 /* Because there is no live analysis when -O1 */
10946 if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) {
10947 locReg.SetRegNotBBLocal();
10948 }
10949 MemOperand &locMem = GetOrCreateMemOpnd(MemOperand::kAddrModeBOi, k64BitSize, &locReg, nullptr,
10950 &GetOrCreateOfstOpnd(0, k32BitSize), nullptr);
10951 RegOperand &newValReg = LoadIntoRegister(*newVal, PTY_a64);
10952 if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) {
10953 newValReg.SetRegNotBBLocal();
10954 }
10955 GetCurBB()->SetKind(BB::kBBFallthru);
10956
10957 LabelIdx retryLabIdx = CreateLabeledBB(intrinsiccallNode);
10958
10959 RegOperand *oldVal = SelectLoadExcl(kValPrimType, locMem, isAcquire);
10960 if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) {
10961 oldVal->SetRegNotBBLocal();
10962 }
10963 RegOperand *succ = SelectStoreExcl(kValPrimType, locMem, newValReg, isRelease);
10964 if (Globals::GetInstance()->GetOptimLevel() == CGOptions::kLevel0) {
10965 succ->SetRegNotBBLocal();
10966 }
10967
10968 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wcbnz, *succ, GetOrCreateLabelOperand(retryLabIdx)));
10969 GetCurBB()->SetKind(BB::kBBIntrinsic);
10970 GetCurBB()->SetNext(origFtBB);
10971
10972 SaveReturnValueInLocal(*retVals, 0, kValPrimType, *oldVal, intrinsiccallNode);
10973 break;
10974 }
10975 case INTRN_GET_AND_ADDI: {
10976 IntrinsifyGetAndAddInt(*srcOpnds, PTY_i32);
10977 break;
10978 }
10979 case INTRN_GET_AND_ADDL: {
10980 IntrinsifyGetAndAddInt(*srcOpnds, PTY_i64);
10981 break;
10982 }
10983 case INTRN_GET_AND_SETI: {
10984 IntrinsifyGetAndSetInt(*srcOpnds, PTY_i32);
10985 break;
10986 }
10987 case INTRN_GET_AND_SETL: {
10988 IntrinsifyGetAndSetInt(*srcOpnds, PTY_i64);
10989 break;
10990 }
10991 case INTRN_COMP_AND_SWAPI: {
10992 IntrinsifyCompareAndSwapInt(*srcOpnds, PTY_i32);
10993 break;
10994 }
10995 case INTRN_COMP_AND_SWAPL: {
10996 IntrinsifyCompareAndSwapInt(*srcOpnds, PTY_i64);
10997 break;
10998 }
10999 default: {
11000 CHECK_FATAL(false, "Intrinsic %d: %s not implemented by the AArch64 CG.", intrinsic,
11001 GetIntrinsicName(intrinsic));
11002 break;
11003 }
11004 }
11005 }
11006
SelectCclz(IntrinsicopNode & intrnNode)11007 Operand *AArch64CGFunc::SelectCclz(IntrinsicopNode &intrnNode)
11008 {
11009 BaseNode *argexpr = intrnNode.Opnd(0);
11010 PrimType ptype = argexpr->GetPrimType();
11011 Operand *opnd = HandleExpr(intrnNode, *argexpr);
11012 MOperator mop;
11013
11014 RegOperand &ldDest = CreateRegisterOperandOfType(ptype);
11015 if (opnd->IsMemoryAccessOperand()) {
11016 Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype), ptype), ldDest, *opnd);
11017 GetCurBB()->AppendInsn(insn);
11018 opnd = &ldDest;
11019 } else if (opnd->IsImmediate()) {
11020 SelectCopyImm(ldDest, *static_cast<ImmOperand *>(opnd), ptype);
11021 opnd = &ldDest;
11022 }
11023
11024 if (GetPrimTypeSize(ptype) == k4ByteSize) {
11025 mop = MOP_wclz;
11026 } else {
11027 mop = MOP_xclz;
11028 }
11029 RegOperand &dst = CreateRegisterOperandOfType(ptype);
11030 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mop, dst, *opnd));
11031 return &dst;
11032 }
11033
SelectCctz(IntrinsicopNode & intrnNode)11034 Operand *AArch64CGFunc::SelectCctz(IntrinsicopNode &intrnNode)
11035 {
11036 BaseNode *argexpr = intrnNode.Opnd(0);
11037 PrimType ptype = argexpr->GetPrimType();
11038 Operand *opnd = HandleExpr(intrnNode, *argexpr);
11039
11040 RegOperand &ldDest = CreateRegisterOperandOfType(ptype);
11041 if (opnd->IsMemoryAccessOperand()) {
11042 Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype), ptype), ldDest, *opnd);
11043 GetCurBB()->AppendInsn(insn);
11044 opnd = &ldDest;
11045 } else if (opnd->IsImmediate()) {
11046 SelectCopyImm(ldDest, *static_cast<ImmOperand *>(opnd), ptype);
11047 opnd = &ldDest;
11048 }
11049
11050 MOperator clzmop;
11051 MOperator rbitmop;
11052 if (GetPrimTypeSize(ptype) == k4ByteSize) {
11053 clzmop = MOP_wclz;
11054 rbitmop = MOP_wrbit;
11055 } else {
11056 clzmop = MOP_xclz;
11057 rbitmop = MOP_xrbit;
11058 }
11059 RegOperand &dst1 = CreateRegisterOperandOfType(ptype);
11060 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(rbitmop, dst1, *opnd));
11061 RegOperand &dst2 = CreateRegisterOperandOfType(ptype);
11062 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(clzmop, dst2, dst1));
11063 return &dst2;
11064 }
11065
SelectCpopcount(IntrinsicopNode & intrnNode)11066 Operand *AArch64CGFunc::SelectCpopcount(IntrinsicopNode &intrnNode)
11067 {
11068 CHECK_FATAL(false, "%s NIY", intrnNode.GetIntrinDesc().name);
11069 return nullptr;
11070 }
11071
SelectCparity(IntrinsicopNode & intrnNode)11072 Operand *AArch64CGFunc::SelectCparity(IntrinsicopNode &intrnNode)
11073 {
11074 CHECK_FATAL(false, "%s NIY", intrnNode.GetIntrinDesc().name);
11075 return nullptr;
11076 }
11077
SelectCclrsb(IntrinsicopNode & intrnNode)11078 Operand *AArch64CGFunc::SelectCclrsb(IntrinsicopNode &intrnNode)
11079 {
11080 BaseNode *argexpr = intrnNode.Opnd(0);
11081 PrimType ptype = argexpr->GetPrimType();
11082 Operand *opnd = HandleExpr(intrnNode, *argexpr);
11083
11084 RegOperand &ldDest = CreateRegisterOperandOfType(ptype);
11085 if (opnd->IsMemoryAccessOperand()) {
11086 Insn &insn = GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype), ptype), ldDest, *opnd);
11087 GetCurBB()->AppendInsn(insn);
11088 opnd = &ldDest;
11089 } else if (opnd->IsImmediate()) {
11090 SelectCopyImm(ldDest, *static_cast<ImmOperand *>(opnd), ptype);
11091 opnd = &ldDest;
11092 }
11093
11094 bool is32Bit = (GetPrimTypeSize(ptype) == k4ByteSize);
11095 RegOperand &res = CreateRegisterOperandOfType(ptype);
11096 SelectMvn(res, *opnd, ptype);
11097 SelectAArch64Cmp(*opnd, GetZeroOpnd(is32Bit ? k32BitSize : k64BitSize), true, is32Bit ? k32BitSize : k64BitSize);
11098 SelectAArch64Select(*opnd, res, *opnd, GetCondOperand(CC_LT), true, is32Bit ? k32BitSize : k64BitSize);
11099 MOperator clzmop = (is32Bit ? MOP_wclz : MOP_xclz);
11100 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(clzmop, *opnd, *opnd));
11101 SelectSub(*opnd, *opnd, CreateImmOperand(1, is32Bit ? k32BitSize : k64BitSize, true), ptype);
11102 return opnd;
11103 }
11104
SelectCisaligned(IntrinsicopNode & intrnNode)11105 Operand *AArch64CGFunc::SelectCisaligned(IntrinsicopNode &intrnNode)
11106 {
11107 BaseNode *argexpr0 = intrnNode.Opnd(0);
11108 PrimType ptype0 = argexpr0->GetPrimType();
11109 Operand *opnd0 = HandleExpr(intrnNode, *argexpr0);
11110
11111 RegOperand &ldDest0 = CreateRegisterOperandOfType(ptype0);
11112 if (opnd0->IsMemoryAccessOperand()) {
11113 GetCurBB()->AppendInsn(
11114 GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype0), ptype0), ldDest0, *opnd0));
11115 opnd0 = &ldDest0;
11116 } else if (opnd0->IsImmediate()) {
11117 SelectCopyImm(ldDest0, *static_cast<ImmOperand *>(opnd0), ptype0);
11118 opnd0 = &ldDest0;
11119 }
11120
11121 BaseNode *argexpr1 = intrnNode.Opnd(1);
11122 PrimType ptype1 = argexpr1->GetPrimType();
11123 Operand *opnd1 = HandleExpr(intrnNode, *argexpr1);
11124
11125 RegOperand &ldDest1 = CreateRegisterOperandOfType(ptype1);
11126 if (opnd1->IsMemoryAccessOperand()) {
11127 GetCurBB()->AppendInsn(
11128 GetInsnBuilder()->BuildInsn(PickLdInsn(GetPrimTypeBitSize(ptype1), ptype1), ldDest1, *opnd1));
11129 opnd1 = &ldDest1;
11130 } else if (opnd1->IsImmediate()) {
11131 SelectCopyImm(ldDest1, *static_cast<ImmOperand *>(opnd1), ptype1);
11132 opnd1 = &ldDest1;
11133 }
11134 // mov w4, #1
11135 RegOperand ®0 = CreateRegisterOperandOfType(PTY_i32);
11136 SelectCopyImm(reg0, CreateImmOperand(1, k32BitSize, true), PTY_i32);
11137 // sxtw x4, w4
11138 MOperator mOp = MOP_xsxtw64;
11139 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, reg0, reg0));
11140 // sub x3, x3, x4
11141 SelectSub(*opnd1, *opnd1, reg0, ptype1);
11142 // and x2, x2, x3
11143 SelectBand(*opnd0, *opnd0, *opnd1, ptype1);
11144 // mov w3, #0
11145 // sxtw x3, w3
11146 // cmp x2, x3
11147 SelectAArch64Cmp(*opnd0, GetZeroOpnd(k64BitSize), true, k64BitSize);
11148 // cset w2, EQ
11149 SelectAArch64CSet(*opnd0, GetCondOperand(CC_EQ), false);
11150 return opnd0;
11151 }
11152
SelectArithmeticAndLogical(Operand & resOpnd,Operand & opnd0,Operand & opnd1,PrimType primType,Opcode op)11153 void AArch64CGFunc::SelectArithmeticAndLogical(Operand &resOpnd, Operand &opnd0, Operand &opnd1, PrimType primType,
11154 Opcode op)
11155 {
11156 switch (op) {
11157 case OP_add:
11158 SelectAdd(resOpnd, opnd0, opnd1, primType);
11159 break;
11160 case OP_sub:
11161 SelectSub(resOpnd, opnd0, opnd1, primType);
11162 break;
11163 case OP_band:
11164 SelectBand(resOpnd, opnd0, opnd1, primType);
11165 break;
11166 case OP_bior:
11167 SelectBior(resOpnd, opnd0, opnd1, primType);
11168 break;
11169 case OP_bxor:
11170 SelectBxor(resOpnd, opnd0, opnd1, primType);
11171 break;
11172 default:
11173 CHECK_FATAL(false, "unconcerned opcode for arithmetical and logical insns");
11174 break;
11175 }
11176 }
11177
SelectAArch64CSyncFetch(const IntrinsicopNode & intrinopNode,Opcode op,bool fetchBefore)11178 Operand *AArch64CGFunc::SelectAArch64CSyncFetch(const IntrinsicopNode &intrinopNode, Opcode op, bool fetchBefore)
11179 {
11180 auto primType = intrinopNode.GetPrimType();
11181 /* Create BB which includes atomic built_in function */
11182 LabelIdx atomicBBLabIdx = CreateLabel();
11183 BB *atomicBB = CreateNewBB();
11184 atomicBB->SetKind(BB::kBBIf);
11185 atomicBB->SetAtomicBuiltIn();
11186 atomicBB->AddLabel(atomicBBLabIdx);
11187 SetLab2BBMap(static_cast<int32>(atomicBBLabIdx), *atomicBB);
11188 GetCurBB()->AppendBB(*atomicBB);
11189 /* keep variables inside same BB */
11190 if (GetCG()->GetOptimizeLevel() == CGOptions::kLevel0) {
11191 SetCurBB(*atomicBB);
11192 }
11193 /* handle built_in args */
11194 Operand *addrOpnd = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnFirstOpnd));
11195 Operand *valueOpnd = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnSecondOpnd));
11196 addrOpnd = &LoadIntoRegister(*addrOpnd, intrinopNode.GetNopndAt(kInsnFirstOpnd)->GetPrimType());
11197 valueOpnd = &LoadIntoRegister(*valueOpnd, intrinopNode.GetNopndAt(kInsnSecondOpnd)->GetPrimType());
11198 if (GetCG()->GetOptimizeLevel() != CGOptions::kLevel0) {
11199 SetCurBB(*atomicBB);
11200 }
11201 /* load from pointed address */
11202 auto primTypeP2Size = GetPrimTypeP2Size(primType);
11203 auto *regLoaded = &CreateRegisterOperandOfType(primType);
11204 auto &memOpnd = CreateMemOpnd(*static_cast<RegOperand *>(addrOpnd), 0, GetPrimTypeBitSize(primType));
11205 auto mOpLoad = PickLoadStoreExclInsn(primTypeP2Size, false, false);
11206 atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(mOpLoad, *regLoaded, memOpnd));
11207 /* update loaded value */
11208 auto *regOperated = &CreateRegisterOperandOfType(primType);
11209 SelectArithmeticAndLogical(*regOperated, *regLoaded, *valueOpnd, primType, op);
11210 /* store to pointed address */
11211 auto *accessStatus = &CreateRegisterOperandOfType(PTY_u32);
11212 auto mOpStore = PickLoadStoreExclInsn(primTypeP2Size, true, true);
11213 atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(mOpStore, *accessStatus, *regOperated, memOpnd));
11214 /* check the exclusive accsess status */
11215 auto &atomicBBOpnd = GetOrCreateLabelOperand(*atomicBB);
11216 atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wcbnz, *accessStatus, atomicBBOpnd));
11217
11218 /* Data Memory Barrier */
11219 BB *nextBB = CreateNewBB();
11220 atomicBB->AppendBB(*nextBB);
11221 SetCurBB(*nextBB);
11222 nextBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_dmb_ish, AArch64CG::kMd[MOP_dmb_ish]));
11223 return fetchBefore ? regLoaded : regOperated;
11224 }
11225
SelectCSyncCmpSwap(const IntrinsicopNode & intrinopNode,bool retBool)11226 Operand *AArch64CGFunc::SelectCSyncCmpSwap(const IntrinsicopNode &intrinopNode, bool retBool)
11227 {
11228 PrimType primType = intrinopNode.GetNopndAt(kInsnSecondOpnd)->GetPrimType();
11229 DEBUG_ASSERT(primType == intrinopNode.GetNopndAt(kInsnThirdOpnd)->GetPrimType(), "gcc built_in rule");
11230 LabelIdx atomicBBLabIdx = CreateLabel();
11231 BB *atomicBB = CreateNewBB();
11232 atomicBB->SetKind(BB::kBBIf);
11233 atomicBB->SetAtomicBuiltIn();
11234 atomicBB->AddLabel(atomicBBLabIdx);
11235 SetLab2BBMap(static_cast<int32>(atomicBBLabIdx), *atomicBB);
11236 GetCurBB()->AppendBB(*atomicBB);
11237 if (GetCG()->GetOptimizeLevel() == CGOptions::kLevel0) {
11238 SetCurBB(*atomicBB);
11239 }
11240 /* handle built_in args */
11241 Operand *addrOpnd = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnFirstOpnd));
11242 Operand *oldVal = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnSecondOpnd));
11243 Operand *newVal = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnThirdOpnd));
11244 if (GetCG()->GetOptimizeLevel() != CGOptions::kLevel0) {
11245 SetCurBB(*atomicBB);
11246 }
11247
11248 uint32 primTypeP2Size = GetPrimTypeP2Size(primType);
11249 /* ldxr */
11250 auto *regLoaded = &CreateRegisterOperandOfType(primType);
11251 auto &memOpnd = CreateMemOpnd(LoadIntoRegister(*addrOpnd, primType), 0, GetPrimTypeBitSize(primType));
11252 auto mOpLoad = PickLoadStoreExclInsn(primTypeP2Size, false, false);
11253 atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(mOpLoad, *regLoaded, memOpnd));
11254 Operand *regExtend = &CreateRegisterOperandOfType(primType);
11255 PrimType targetType = (oldVal->GetSize() <= k32BitSize) ? (IsSignedInteger(primType) ? PTY_i32 : PTY_u32)
11256 : (IsSignedInteger(primType) ? PTY_i64 : PTY_u64);
11257 SelectCvtInt2Int(nullptr, regExtend, regLoaded, primType, targetType);
11258 /* cmp */
11259 SelectAArch64Cmp(*regExtend, *oldVal, true, oldVal->GetSize());
11260 /* bne */
11261 Operand &rflag = GetOrCreateRflag();
11262 LabelIdx nextBBLableIdx = CreateLabel();
11263 LabelOperand &targetOpnd = GetOrCreateLabelOperand(nextBBLableIdx);
11264 atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_bne, rflag, targetOpnd));
11265 /* stlxr */
11266 BB *stlxrBB = CreateNewBB();
11267 stlxrBB->SetKind(BB::kBBIf);
11268 atomicBB->AppendBB(*stlxrBB);
11269 SetCurBB(*stlxrBB);
11270 auto *accessStatus = &CreateRegisterOperandOfType(PTY_u32);
11271 auto &newRegVal = LoadIntoRegister(*newVal, primType);
11272 auto mOpStore = PickLoadStoreExclInsn(primTypeP2Size, true, true);
11273 stlxrBB->AppendInsn(GetInsnBuilder()->BuildInsn(mOpStore, *accessStatus, newRegVal, memOpnd));
11274 /* cbnz ==> check the exclusive accsess status */
11275 auto &atomicBBOpnd = GetOrCreateLabelOperand(*atomicBB);
11276 stlxrBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wcbnz, *accessStatus, atomicBBOpnd));
11277 /* Data Memory Barrier */
11278 BB *nextBB = CreateNewBB();
11279 nextBB->AddLabel(nextBBLableIdx);
11280 nextBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_dmb_ish, AArch64CG::kMd[MOP_dmb_ish]));
11281 SetLab2BBMap(static_cast<int32>(nextBBLableIdx), *nextBB);
11282 stlxrBB->AppendBB(*nextBB);
11283 SetCurBB(*nextBB);
11284 /* bool version return true if the comparison is successful and newval is written */
11285 if (retBool) {
11286 auto *retOpnd = &CreateRegisterOperandOfType(PTY_u32);
11287 SelectAArch64CSet(*retOpnd, GetCondOperand(CC_EQ), false);
11288 return retOpnd;
11289 }
11290 /* type version return the contents of *addrOpnd before the operation */
11291 return regLoaded;
11292 }
11293
SelectCSyncFetch(IntrinsicopNode & intrinopNode,Opcode op,bool fetchBefore)11294 Operand *AArch64CGFunc::SelectCSyncFetch(IntrinsicopNode &intrinopNode, Opcode op, bool fetchBefore)
11295 {
11296 return SelectAArch64CSyncFetch(intrinopNode, op, fetchBefore);
11297 }
11298
SelectCSyncBoolCmpSwap(IntrinsicopNode & intrinopNode)11299 Operand *AArch64CGFunc::SelectCSyncBoolCmpSwap(IntrinsicopNode &intrinopNode)
11300 {
11301 return SelectCSyncCmpSwap(intrinopNode, true);
11302 }
11303
SelectCSyncValCmpSwap(IntrinsicopNode & intrinopNode)11304 Operand *AArch64CGFunc::SelectCSyncValCmpSwap(IntrinsicopNode &intrinopNode)
11305 {
11306 return SelectCSyncCmpSwap(intrinopNode);
11307 }
11308
SelectCSyncLockTestSet(IntrinsicopNode & intrinopNode,PrimType pty)11309 Operand *AArch64CGFunc::SelectCSyncLockTestSet(IntrinsicopNode &intrinopNode, PrimType pty)
11310 {
11311 auto primType = intrinopNode.GetPrimType();
11312 Operand *addrOpnd = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnFirstOpnd));
11313 Operand *valueOpnd = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnSecondOpnd));
11314 addrOpnd = &LoadIntoRegister(*addrOpnd, intrinopNode.GetNopndAt(kInsnFirstOpnd)->GetPrimType());
11315 valueOpnd = &LoadIntoRegister(*valueOpnd, intrinopNode.GetNopndAt(kInsnSecondOpnd)->GetPrimType());
11316
11317 /* Create BB which includes atomic built_in function */
11318 LabelIdx atomicBBLabIdx = CreateLabel();
11319 BB *atomicBB = CreateNewBB();
11320 atomicBB->SetKind(BB::kBBIf);
11321 atomicBB->SetAtomicBuiltIn();
11322 atomicBB->AddLabel(atomicBBLabIdx);
11323 SetLab2BBMap(static_cast<int32>(atomicBBLabIdx), *atomicBB);
11324 GetCurBB()->AppendBB(*atomicBB);
11325 SetCurBB(*atomicBB);
11326 /* load from pointed address */
11327 auto primTypeP2Size = GetPrimTypeP2Size(primType);
11328 auto *regLoaded = &CreateRegisterOperandOfType(primType);
11329 auto &memOpnd = CreateMemOpnd(*static_cast<RegOperand *>(addrOpnd), 0, GetPrimTypeBitSize(primType));
11330 auto mOpLoad = PickLoadStoreExclInsn(primTypeP2Size, false, false);
11331 atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(mOpLoad, *regLoaded, memOpnd));
11332 /* store to pointed address */
11333 auto *accessStatus = &CreateRegisterOperandOfType(PTY_u32);
11334 auto mOpStore = PickLoadStoreExclInsn(primTypeP2Size, true, false);
11335 atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(mOpStore, *accessStatus, *valueOpnd, memOpnd));
11336 /* check the exclusive accsess status */
11337 auto &atomicBBOpnd = GetOrCreateLabelOperand(*atomicBB);
11338 atomicBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_wcbnz, *accessStatus, atomicBBOpnd));
11339
11340 /* Data Memory Barrier */
11341 BB *nextBB = CreateNewBB();
11342 atomicBB->AppendBB(*nextBB);
11343 SetCurBB(*nextBB);
11344 nextBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_dmb_ish, AArch64CG::kMd[MOP_dmb_ish]));
11345 return regLoaded;
11346 }
11347
SelectCSyncLockRelease(const IntrinsiccallNode & intrinsiccall,PrimType primType)11348 void AArch64CGFunc::SelectCSyncLockRelease(const IntrinsiccallNode &intrinsiccall, PrimType primType)
11349 {
11350 auto *addrOpnd = HandleExpr(intrinsiccall, *intrinsiccall.GetNopndAt(kInsnFirstOpnd));
11351 auto primTypeBitSize = GetPrimTypeBitSize(primType);
11352 auto mOp = PickStInsn(primTypeBitSize, primType, AArch64isa::kMoRelease);
11353 auto &zero = GetZeroOpnd(primTypeBitSize);
11354 auto &memOpnd = CreateMemOpnd(LoadIntoRegister(*addrOpnd, primType), 0, primTypeBitSize);
11355 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, zero, memOpnd));
11356 }
11357
SelectCSyncSynchronize(IntrinsicopNode & intrinopNode)11358 Operand *AArch64CGFunc::SelectCSyncSynchronize(IntrinsicopNode &intrinopNode)
11359 {
11360 (void)intrinopNode;
11361 CHECK_FATAL(false, "have not implement SelectCSyncSynchronize yet");
11362 return nullptr;
11363 }
11364
PickMemOrder(std::memory_order memOrder,bool isLdr) const11365 AArch64isa::MemoryOrdering AArch64CGFunc::PickMemOrder(std::memory_order memOrder, bool isLdr) const
11366 {
11367 switch (memOrder) {
11368 case std::memory_order_relaxed:
11369 return AArch64isa::kMoNone;
11370 case std::memory_order_consume:
11371 case std::memory_order_acquire:
11372 return isLdr ? AArch64isa::kMoAcquire : AArch64isa::kMoNone;
11373 case std::memory_order_release:
11374 return isLdr ? AArch64isa::kMoNone : AArch64isa::kMoRelease;
11375 case std::memory_order_acq_rel:
11376 case std::memory_order_seq_cst:
11377 return isLdr ? AArch64isa::kMoAcquire : AArch64isa::kMoRelease;
11378 default:
11379 CHECK_FATAL(false, "unexpected memorder");
11380 return AArch64isa::kMoNone;
11381 }
11382 }
11383
11384 /*
11385 * regassign %1 (intrinsicop C___Atomic_Load_N(ptr, memorder))
11386 * ====> %1 = *ptr
11387 * let %1 -> x0
11388 * let ptr -> x1
11389 * implement to asm: ldr/ldar x0, [x1]
11390 * a load-acquire would replace ldr if memorder is not 0
11391 */
SelectCAtomicLoadN(IntrinsicopNode & intrinsicopNode)11392 Operand *AArch64CGFunc::SelectCAtomicLoadN(IntrinsicopNode &intrinsicopNode)
11393 {
11394 auto *addrOpnd = HandleExpr(intrinsicopNode, *intrinsicopNode.Opnd(0));
11395 auto *memOrderOpnd = intrinsicopNode.Opnd(1);
11396 auto primType = intrinsicopNode.GetPrimType();
11397 auto *memOrderConst = static_cast<MIRIntConst *>(static_cast<ConstvalNode *>(memOrderOpnd)->GetConstVal());
11398 auto memOrder = static_cast<std::memory_order>(memOrderConst->GetExtValue());
11399 return SelectAtomicLoad(*addrOpnd, primType, PickMemOrder(memOrder, true));
11400 }
11401
11402 /*
11403 * regassign %1 (intrinsicop C___Atomic_exchange_n(ptr, val, memorder))
11404 * ====> %1 = *ptr; *ptr = val;
11405 * let %1 -> x0
11406 * let ptr -> x1
11407 * let val -> x2
11408 * implement to asm:
11409 * ldr/ldar x0, [x1]
11410 * str/stlr x2, [x1]
11411 * a load-acquire would replace ldr if acquire needed
11412 * a store-relase would replace str if release needed
11413 */
SelectCAtomicExchangeN(IntrinsicopNode & intrinsicopNode)11414 Operand *AArch64CGFunc::SelectCAtomicExchangeN(IntrinsicopNode &intrinsicopNode)
11415 {
11416 auto primType = intrinsicopNode.GetPrimType();
11417 auto *addrOpnd = HandleExpr(intrinsicopNode, *intrinsicopNode.Opnd(0));
11418 auto *valueOpnd = HandleExpr(intrinsicopNode, *intrinsicopNode.Opnd(1));
11419 auto *memOrderOpnd = intrinsicopNode.Opnd(kInsnThirdOpnd);
11420 auto *memOrderConst = static_cast<MIRIntConst *>(static_cast<ConstvalNode *>(memOrderOpnd)->GetConstVal());
11421 auto memOrder = static_cast<std::memory_order>(memOrderConst->GetExtValue());
11422 auto *result = SelectAtomicLoad(*addrOpnd, primType, PickMemOrder(memOrder, true));
11423 SelectAtomicStore(*valueOpnd, *addrOpnd, primType, PickMemOrder(memOrder, false));
11424 return result;
11425 }
11426
SelectAtomicLoad(Operand & addrOpnd,PrimType primType,AArch64isa::MemoryOrdering memOrder)11427 Operand *AArch64CGFunc::SelectAtomicLoad(Operand &addrOpnd, PrimType primType, AArch64isa::MemoryOrdering memOrder)
11428 {
11429 auto mOp = PickLdInsn(GetPrimTypeBitSize(primType), primType, memOrder);
11430 auto &memOpnd = CreateMemOpnd(LoadIntoRegister(addrOpnd, PTY_a64), 0, k64BitSize);
11431 auto *resultOpnd = &CreateRegisterOperandOfType(primType);
11432 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, *resultOpnd, memOpnd));
11433 return resultOpnd;
11434 }
11435
SelectCReturnAddress(IntrinsicopNode & intrinopNode)11436 Operand *AArch64CGFunc::SelectCReturnAddress(IntrinsicopNode &intrinopNode)
11437 {
11438 if (intrinopNode.GetIntrinsic() == INTRN_C__builtin_extract_return_addr) {
11439 DEBUG_ASSERT(intrinopNode.GetNumOpnds() == 1, "expect one parameter");
11440 Operand *addrOpnd = HandleExpr(intrinopNode, *intrinopNode.GetNopndAt(kInsnFirstOpnd));
11441 return &LoadIntoRegister(*addrOpnd, PTY_a64);
11442 } else if (intrinopNode.GetIntrinsic() == INTRN_C__builtin_return_address) {
11443 BaseNode *argexpr0 = intrinopNode.Opnd(0);
11444 while (!argexpr0->IsLeaf()) {
11445 argexpr0 = argexpr0->Opnd(0);
11446 }
11447 CHECK_FATAL(argexpr0->IsConstval(), "Invalid argument of __builtin_return_address");
11448 auto &constNode = static_cast<ConstvalNode &>(*argexpr0);
11449 DEBUG_ASSERT(constNode.GetConstVal()->GetKind() == kConstInt, "expect MIRIntConst does not support float yet");
11450 MIRIntConst *mirIntConst = safe_cast<MIRIntConst>(constNode.GetConstVal());
11451 DEBUG_ASSERT(mirIntConst != nullptr, "nullptr checking");
11452 int64 scale = mirIntConst->GetExtValue();
11453 /*
11454 * Do not support getting return address with a nonzero argument
11455 * inline / tail call opt will destory this behavior
11456 */
11457 CHECK_FATAL(scale == 0, "Do not support recursion");
11458 Operand *resReg = &static_cast<Operand &>(CreateRegisterOperandOfType(PTY_i64));
11459 SelectCopy(*resReg, PTY_i64, GetOrCreatePhysicalRegisterOperand(RLR, k64BitSize, kRegTyInt), PTY_i64);
11460 return resReg;
11461 }
11462 return nullptr;
11463 }
11464
SelectCalignup(IntrinsicopNode & intrnNode)11465 Operand *AArch64CGFunc::SelectCalignup(IntrinsicopNode &intrnNode)
11466 {
11467 return SelectAArch64align(intrnNode, true);
11468 }
11469
SelectCaligndown(IntrinsicopNode & intrnNode)11470 Operand *AArch64CGFunc::SelectCaligndown(IntrinsicopNode &intrnNode)
11471 {
11472 return SelectAArch64align(intrnNode, false);
11473 }
11474
SelectAArch64align(const IntrinsicopNode & intrnNode,bool isUp)11475 Operand *AArch64CGFunc::SelectAArch64align(const IntrinsicopNode &intrnNode, bool isUp)
11476 {
11477 /* Handle Two args */
11478 BaseNode *argexpr0 = intrnNode.Opnd(0);
11479 PrimType ptype0 = argexpr0->GetPrimType();
11480 Operand *opnd0 = HandleExpr(intrnNode, *argexpr0);
11481 PrimType resultPtype = intrnNode.GetPrimType();
11482 RegOperand &ldDest0 = LoadIntoRegister(*opnd0, ptype0);
11483
11484 BaseNode *argexpr1 = intrnNode.Opnd(1);
11485 PrimType ptype1 = argexpr1->GetPrimType();
11486 Operand *opnd1 = HandleExpr(intrnNode, *argexpr1);
11487 RegOperand &arg1 = LoadIntoRegister(*opnd1, ptype1);
11488 DEBUG_ASSERT(IsPrimitiveInteger(ptype0) && IsPrimitiveInteger(ptype1), "align integer type only");
11489 Operand *ldDest1 = &static_cast<Operand &>(CreateRegisterOperandOfType(ptype0));
11490 SelectCvtInt2Int(nullptr, ldDest1, &arg1, ptype1, ptype0);
11491
11492 Operand *resultReg = &static_cast<Operand &>(CreateRegisterOperandOfType(ptype0));
11493 Operand &immReg = CreateImmOperand(1, GetPrimTypeBitSize(ptype0), true);
11494 /* Do alignment x0 -- value to be aligned x1 -- alignment */
11495 if (isUp) {
11496 /* add res, x0, x1 */
11497 SelectAdd(*resultReg, ldDest0, *ldDest1, ptype0);
11498 /* sub res, res, 1 */
11499 SelectSub(*resultReg, *resultReg, immReg, ptype0);
11500 }
11501 Operand *tempReg = &static_cast<Operand &>(CreateRegisterOperandOfType(ptype0));
11502 /* sub temp, x1, 1 */
11503 SelectSub(*tempReg, *ldDest1, immReg, ptype0);
11504 /* mvn temp, temp */
11505 SelectMvn(*tempReg, *tempReg, ptype0);
11506 /* and res, res, temp */
11507 if (isUp) {
11508 SelectBand(*resultReg, *resultReg, *tempReg, ptype0);
11509 } else {
11510 SelectBand(*resultReg, ldDest0, *tempReg, ptype0);
11511 }
11512 if (resultPtype != ptype0) {
11513 SelectCvtInt2Int(&intrnNode, resultReg, resultReg, ptype0, resultPtype);
11514 }
11515 return resultReg;
11516 }
11517
11518 /*
11519 * NOTE: consider moving the following things into aarch64_cg.cpp They may
11520 * serve not only inrinsics, but other MapleIR instructions as well.
11521 * Do it as if we are adding a label in straight-line assembly code.
11522 */
CreateLabeledBB(StmtNode & stmt)11523 LabelIdx AArch64CGFunc::CreateLabeledBB(StmtNode &stmt)
11524 {
11525 LabelIdx labIdx = CreateLabel();
11526 BB *newBB = StartNewBBImpl(false, stmt);
11527 newBB->AddLabel(labIdx);
11528 SetLab2BBMap(labIdx, *newBB);
11529 SetCurBB(*newBB);
11530 return labIdx;
11531 }
11532
11533 /* Save value into the local variable for the index-th return value; */
SaveReturnValueInLocal(CallReturnVector & retVals,size_t index,PrimType primType,Operand & value,StmtNode & parentStmt)11534 void AArch64CGFunc::SaveReturnValueInLocal(CallReturnVector &retVals, size_t index, PrimType primType, Operand &value,
11535 StmtNode &parentStmt)
11536 {
11537 CallReturnPair &pair = retVals.at(index);
11538 BB tempBB(static_cast<uint32>(-1), *GetFuncScopeAllocator());
11539 BB *realCurBB = GetCurBB();
11540 CHECK_FATAL(!pair.second.IsReg(), "NYI");
11541 Operand *destOpnd = &value;
11542 /* for O0 ,corss-BB var is not support, do extra store/load but why new BB */
11543 if (GetCG()->GetOptimizeLevel() == CGOptions::kLevel0) {
11544 MIRSymbol *symbol = GetFunction().GetLocalOrGlobalSymbol(pair.first);
11545 MIRType *sPty = symbol->GetType();
11546 PrimType ty = symbol->GetType()->GetPrimType();
11547 if (sPty->GetKind() == kTypeStruct || sPty->GetKind() == kTypeUnion) {
11548 MIRStructType *structType = static_cast<MIRStructType *>(sPty);
11549 ty = structType->GetFieldType(pair.second.GetFieldID())->GetPrimType();
11550 } else if (sPty->GetKind() == kTypeClass) {
11551 CHECK_FATAL(false, "unsuppotr type for inlineasm / intrinsic");
11552 }
11553 RegOperand &tempReg = CreateVirtualRegisterOperand(NewVReg(GetRegTyFromPrimTy(ty), GetPrimTypeSize(ty)));
11554 SelectCopy(tempReg, ty, value, ty);
11555 destOpnd = &tempReg;
11556 }
11557 SetCurBB(tempBB);
11558 SelectDassign(pair.first, pair.second.GetFieldID(), primType, *destOpnd);
11559
11560 CHECK_FATAL(realCurBB->GetNext() == nullptr, "current BB must has not nextBB");
11561 realCurBB->SetLastStmt(parentStmt);
11562 realCurBB->SetNext(StartNewBBImpl(true, parentStmt));
11563 realCurBB->GetNext()->SetKind(BB::kBBFallthru);
11564 realCurBB->GetNext()->SetPrev(realCurBB);
11565
11566 realCurBB->GetNext()->InsertAtBeginning(*GetCurBB());
11567 /* restore it */
11568 SetCurBB(*realCurBB->GetNext());
11569 }
11570
11571 /* The following are translation of LL/SC and atomic RMW operations */
OperandToMemOrd(Operand & opnd) const11572 MemOrd AArch64CGFunc::OperandToMemOrd(Operand &opnd) const
11573 {
11574 CHECK_FATAL(opnd.IsImmediate(), "Memory order must be an int constant.");
11575 auto immOpnd = static_cast<ImmOperand *>(&opnd);
11576 int32 val = immOpnd->GetValue();
11577 CHECK_FATAL(val >= 0, "val must be non-negtive");
11578 return MemOrdFromU32(static_cast<uint32>(val));
11579 }
11580
11581 /*
11582 * Generate ldxr or ldaxr instruction.
11583 * byte_p2x: power-of-2 size of operand in bytes (0: 1B, 1: 2B, 2: 4B, 3: 8B).
11584 */
PickLoadStoreExclInsn(uint32 byteP2Size,bool store,bool acqRel) const11585 MOperator AArch64CGFunc::PickLoadStoreExclInsn(uint32 byteP2Size, bool store, bool acqRel) const
11586 {
11587 CHECK_FATAL(byteP2Size < kIntByteSizeDimension, "Illegal argument p2size: %d", byteP2Size);
11588
11589 static MOperator operators[4][2][2] = {{{MOP_wldxrb, MOP_wldaxrb}, {MOP_wstxrb, MOP_wstlxrb}},
11590 {{MOP_wldxrh, MOP_wldaxrh}, {MOP_wstxrh, MOP_wstlxrh}},
11591 {{MOP_wldxr, MOP_wldaxr}, {MOP_wstxr, MOP_wstlxr}},
11592 {{MOP_xldxr, MOP_xldaxr}, {MOP_xstxr, MOP_xstlxr}}};
11593
11594 MOperator optr = operators[byteP2Size][store][acqRel];
11595 CHECK_FATAL(optr != MOP_undef, "Unsupported type p2size: %d", byteP2Size);
11596
11597 return optr;
11598 }
11599
SelectLoadExcl(PrimType valPrimType,MemOperand & loc,bool acquire)11600 RegOperand *AArch64CGFunc::SelectLoadExcl(PrimType valPrimType, MemOperand &loc, bool acquire)
11601 {
11602 uint32 p2size = GetPrimTypeP2Size(valPrimType);
11603
11604 RegOperand &result = CreateRegisterOperandOfType(valPrimType);
11605 MOperator mOp = PickLoadStoreExclInsn(p2size, false, acquire);
11606 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, loc));
11607
11608 return &result;
11609 }
11610
SelectStoreExcl(PrimType valPty,MemOperand & loc,RegOperand & newVal,bool release)11611 RegOperand *AArch64CGFunc::SelectStoreExcl(PrimType valPty, MemOperand &loc, RegOperand &newVal, bool release)
11612 {
11613 uint32 p2size = GetPrimTypeP2Size(valPty);
11614
11615 /* the result (success/fail) is to be stored in a 32-bit register */
11616 RegOperand &result = CreateRegisterOperandOfType(PTY_u32);
11617
11618 MOperator mOp = PickLoadStoreExclInsn(p2size, true, release);
11619 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(mOp, result, newVal, loc));
11620
11621 return &result;
11622 }
11623
GetRegisterType(regno_t reg) const11624 RegType AArch64CGFunc::GetRegisterType(regno_t reg) const
11625 {
11626 if (AArch64isa::IsPhysicalRegister(reg)) {
11627 return AArch64isa::GetRegType(static_cast<AArch64reg>(reg));
11628 } else if (reg == kRFLAG) {
11629 return kRegTyCc;
11630 } else {
11631 return CGFunc::GetRegisterType(reg);
11632 }
11633 }
11634
LoadStructCopyBase(const MIRSymbol & symbol,int64 offset,int dataSize)11635 MemOperand &AArch64CGFunc::LoadStructCopyBase(const MIRSymbol &symbol, int64 offset, int dataSize)
11636 {
11637 /* For struct formals > 16 bytes, this is the pointer to the struct copy. */
11638 /* Load the base pointer first. */
11639 RegOperand *vreg = &CreateVirtualRegisterOperand(NewVReg(kRegTyInt, k8ByteSize));
11640 MemOperand *baseMemOpnd = &GetOrCreateMemOpnd(symbol, 0, k64BitSize);
11641 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(PickLdInsn(k64BitSize, PTY_i64), *vreg, *baseMemOpnd));
11642 /* Create the indirect load mem opnd from the base pointer. */
11643 return CreateMemOpnd(*vreg, offset, static_cast<uint32>(dataSize));
11644 }
11645
11646 /* For long branch, insert an unconditional branch.
11647 * From To
11648 * cond_br targe_label reverse_cond_br fallthru_label
11649 * fallthruBB unconditional br target_label
11650 * fallthru_label:
11651 * fallthruBB
11652 */
InsertJumpPad(Insn * insn)11653 void AArch64CGFunc::InsertJumpPad(Insn *insn)
11654 {
11655 BB *bb = insn->GetBB();
11656 DEBUG_ASSERT(bb, "instruction has no bb");
11657 DEBUG_ASSERT(bb->GetKind() == BB::kBBIf || bb->GetKind() == BB::kBBGoto,
11658 "instruction is in neither if bb nor goto bb");
11659 if (bb->GetKind() == BB::kBBGoto) {
11660 return;
11661 }
11662 DEBUG_ASSERT(bb->NumSuccs() == k2ByteSize, "if bb should have 2 successors");
11663
11664 BB *longBrBB = CreateNewBB();
11665
11666 BB *fallthruBB = bb->GetNext();
11667 LabelIdx fallthruLBL = fallthruBB->GetLabIdx();
11668 if (fallthruLBL == 0) {
11669 fallthruLBL = CreateLabel();
11670 SetLab2BBMap(static_cast<int32>(fallthruLBL), *fallthruBB);
11671 fallthruBB->AddLabel(fallthruLBL);
11672 }
11673
11674 BB *targetBB;
11675 if (bb->GetSuccs().front() == fallthruBB) {
11676 targetBB = bb->GetSuccs().back();
11677 } else {
11678 targetBB = bb->GetSuccs().front();
11679 }
11680 LabelIdx targetLBL = targetBB->GetLabIdx();
11681 if (targetLBL == 0) {
11682 targetLBL = CreateLabel();
11683 SetLab2BBMap(static_cast<int32>(targetLBL), *targetBB);
11684 targetBB->AddLabel(targetLBL);
11685 }
11686
11687 // Adjustment on br and CFG
11688 bb->RemoveSuccs(*targetBB);
11689 bb->PushBackSuccs(*longBrBB);
11690 bb->SetNext(longBrBB);
11691 // reverse cond br targeting fallthruBB
11692 uint32 targetIdx = AArch64isa::GetJumpTargetIdx(*insn);
11693 MOperator mOp = AArch64isa::FlipConditionOp(insn->GetMachineOpcode());
11694 insn->SetMOP(AArch64CG::kMd[mOp]);
11695 LabelOperand &fallthruBBLBLOpnd = GetOrCreateLabelOperand(fallthruLBL);
11696 insn->SetOperand(targetIdx, fallthruBBLBLOpnd);
11697
11698 longBrBB->PushBackPreds(*bb);
11699 longBrBB->PushBackSuccs(*targetBB);
11700 LabelOperand &targetLBLOpnd = GetOrCreateLabelOperand(targetLBL);
11701 longBrBB->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xuncond, targetLBLOpnd));
11702 longBrBB->SetPrev(bb);
11703 longBrBB->SetNext(fallthruBB);
11704 longBrBB->SetKind(BB::kBBGoto);
11705
11706 fallthruBB->SetPrev(longBrBB);
11707
11708 targetBB->RemovePreds(*bb);
11709 targetBB->PushBackPreds(*longBrBB);
11710 }
11711
AdjustOneElementVectorOperand(PrimType oType,RegOperand * opnd)11712 RegOperand *AArch64CGFunc::AdjustOneElementVectorOperand(PrimType oType, RegOperand *opnd)
11713 {
11714 RegOperand *resCvt = &CreateRegisterOperandOfType(oType);
11715 Insn *insnCvt = &GetInsnBuilder()->BuildInsn(MOP_xvmovrd, *resCvt, *opnd);
11716 GetCurBB()->AppendInsn(*insnCvt);
11717 return resCvt;
11718 }
11719
SelectOneElementVectorCopy(Operand * src,PrimType sType)11720 RegOperand *AArch64CGFunc::SelectOneElementVectorCopy(Operand *src, PrimType sType)
11721 {
11722 RegOperand *res = &CreateRegisterOperandOfType(PTY_f64);
11723 SelectCopy(*res, PTY_f64, *src, sType);
11724 static_cast<RegOperand *>(res)->SetIF64Vec();
11725 return res;
11726 }
11727
SelectVectorAbs(PrimType rType,Operand * o1)11728 RegOperand *AArch64CGFunc::SelectVectorAbs(PrimType rType, Operand *o1)
11729 {
11730 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
11731 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
11732 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
11733
11734 MOperator mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vabsvv : MOP_vabsuu;
11735 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
11736 vInsn.AddOpndChain(*res).AddOpndChain(*o1);
11737 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1);
11738 GetCurBB()->AppendInsn(vInsn);
11739 return res;
11740 }
11741
SelectVectorAddLong(PrimType rType,Operand * o1,Operand * o2,PrimType otyp,bool isLow)11742 RegOperand *AArch64CGFunc::SelectVectorAddLong(PrimType rType, Operand *o1, Operand *o2, PrimType otyp, bool isLow)
11743 {
11744 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result type */
11745 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
11746 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(otyp); /* vector operand 1 */
11747 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(otyp); /* vector operand 2 */
11748 MOperator mOp;
11749 if (isLow) {
11750 mOp = IsUnsignedInteger(rType) ? MOP_vuaddlvuu : MOP_vsaddlvuu;
11751 } else {
11752 mOp = IsUnsignedInteger(rType) ? MOP_vuaddl2vvv : MOP_vsaddl2vvv;
11753 }
11754 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
11755 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
11756 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
11757 GetCurBB()->AppendInsn(vInsn);
11758 return res;
11759 }
11760
SelectVectorAddWiden(Operand * o1,PrimType otyp1,Operand * o2,PrimType otyp2,bool isLow)11761 RegOperand *AArch64CGFunc::SelectVectorAddWiden(Operand *o1, PrimType otyp1, Operand *o2, PrimType otyp2, bool isLow)
11762 {
11763 RegOperand *res = &CreateRegisterOperandOfType(otyp1); /* restype is same as o1 */
11764 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(otyp1);
11765 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(otyp1); /* vector operand 1 */
11766 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(otyp2); /* vector operand 2 */
11767
11768 MOperator mOp;
11769 if (isLow) {
11770 mOp = IsUnsignedInteger(otyp1) ? MOP_vuaddwvvu : MOP_vsaddwvvu;
11771 } else {
11772 mOp = IsUnsignedInteger(otyp1) ? MOP_vuaddw2vvv : MOP_vsaddw2vvv;
11773 }
11774 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
11775 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
11776 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
11777 GetCurBB()->AppendInsn(vInsn);
11778 return res;
11779 }
11780
SelectVectorImmMov(PrimType rType,Operand * src,PrimType sType)11781 RegOperand *AArch64CGFunc::SelectVectorImmMov(PrimType rType, Operand *src, PrimType sType)
11782 {
11783 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
11784 VectorRegSpec *vecSpec = GetMemoryPool()->New<VectorRegSpec>(rType);
11785 int64 val = static_cast<ImmOperand *>(src)->GetValue();
11786 /* copy the src imm operand to a reg if out of range */
11787 if ((GetVecEleSize(rType) >= k64BitSize) || (GetPrimTypeSize(sType) > k4ByteSize && val != 0) ||
11788 (val < kMinImmVal || val > kMaxImmVal)) {
11789 Operand *reg = &CreateRegisterOperandOfType(sType);
11790 SelectCopy(*reg, sType, *src, sType);
11791 return SelectVectorRegMov(rType, reg, sType);
11792 }
11793
11794 MOperator mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vmovvi : MOP_vmovui;
11795 if (GetVecEleSize(rType) == k8BitSize && val < 0) {
11796 src = &CreateImmOperand(static_cast<uint8>(val), k8BitSize, true);
11797 } else if (val < 0) {
11798 src = &CreateImmOperand(-(val + 1), k8BitSize, true);
11799 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vnotvi : MOP_vnotui;
11800 }
11801
11802 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
11803 vInsn.AddOpndChain(*res).AddOpndChain(*src);
11804 vInsn.PushRegSpecEntry(vecSpec);
11805 GetCurBB()->AppendInsn(vInsn);
11806 return res;
11807 }
11808
SelectVectorRegMov(PrimType rType,Operand * src,PrimType sType)11809 RegOperand *AArch64CGFunc::SelectVectorRegMov(PrimType rType, Operand *src, PrimType sType)
11810 {
11811 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
11812 VectorRegSpec *vecSpec = GetMemoryPool()->New<VectorRegSpec>(rType);
11813
11814 MOperator mOp;
11815 if (GetPrimTypeSize(sType) > k4ByteSize) {
11816 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vxdupvr : MOP_vxdupur;
11817 } else {
11818 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vwdupvr : MOP_vwdupur;
11819 }
11820
11821 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
11822 vInsn.AddOpndChain(*res).AddOpndChain(*src);
11823 vInsn.PushRegSpecEntry(vecSpec);
11824 GetCurBB()->AppendInsn(vInsn);
11825 return res;
11826 }
11827
SelectVectorFromScalar(PrimType rType,Operand * src,PrimType sType)11828 RegOperand *AArch64CGFunc::SelectVectorFromScalar(PrimType rType, Operand *src, PrimType sType)
11829 {
11830 if (!IsPrimitiveVector(rType)) {
11831 return SelectOneElementVectorCopy(src, sType);
11832 } else if (src->IsConstImmediate()) {
11833 return SelectVectorImmMov(rType, src, sType);
11834 } else {
11835 return SelectVectorRegMov(rType, src, sType);
11836 }
11837 }
11838
SelectVectorDup(PrimType rType,Operand * src,bool getLow)11839 RegOperand *AArch64CGFunc::SelectVectorDup(PrimType rType, Operand *src, bool getLow)
11840 {
11841 PrimType oType = rType;
11842 rType = FilterOneElementVectorType(oType);
11843 RegOperand *res = &CreateRegisterOperandOfType(rType);
11844 VectorRegSpec *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(k2ByteSize, k64BitSize, getLow ? 0 : 1);
11845
11846 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(MOP_vduprv, AArch64CG::kMd[MOP_vduprv]);
11847 vInsn.AddOpndChain(*res).AddOpndChain(*src);
11848 vInsn.PushRegSpecEntry(vecSpecSrc);
11849 GetCurBB()->AppendInsn(vInsn);
11850 if (oType != rType) {
11851 res = AdjustOneElementVectorOperand(oType, res);
11852 static_cast<RegOperand *>(res)->SetIF64Vec();
11853 }
11854 return res;
11855 }
11856
SelectVectorGetElement(PrimType rType,Operand * src,PrimType sType,int32 lane)11857 RegOperand *AArch64CGFunc::SelectVectorGetElement(PrimType rType, Operand *src, PrimType sType, int32 lane)
11858 {
11859 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
11860 VectorRegSpec *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(sType, lane); /* vector operand */
11861
11862 MOperator mop;
11863 if (!IsPrimitiveVector(sType)) {
11864 mop = MOP_xmovrr;
11865 } else if (GetPrimTypeBitSize(rType) >= k64BitSize) {
11866 mop = MOP_vxmovrv;
11867 } else {
11868 mop = (GetPrimTypeBitSize(sType) > k64BitSize) ? MOP_vwmovrv : MOP_vwmovru;
11869 }
11870
11871 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
11872 vInsn.AddOpndChain(*res).AddOpndChain(*src);
11873 vInsn.PushRegSpecEntry(vecSpecSrc);
11874 GetCurBB()->AppendInsn(vInsn);
11875 return res;
11876 }
11877
11878 /* adalp o1, o2 instruction accumulates into o1, overwriting the original operand.
11879 Hence we perform c = vadalp(a,b) as
11880 T tmp = a;
11881 return tmp+b;
11882 The return value of vadalp is then assigned to c, leaving value of a intact.
11883 */
SelectVectorPairwiseAdalp(Operand * src1,PrimType sty1,Operand * src2,PrimType sty2)11884 RegOperand *AArch64CGFunc::SelectVectorPairwiseAdalp(Operand *src1, PrimType sty1, Operand *src2, PrimType sty2)
11885 {
11886 VectorRegSpec *vecSpecDest;
11887 RegOperand *res;
11888
11889 if (!IsPrimitiveVector(sty1)) {
11890 RegOperand *resF = SelectOneElementVectorCopy(src1, sty1);
11891 res = &CreateRegisterOperandOfType(PTY_f64);
11892 SelectCopy(*res, PTY_f64, *resF, PTY_f64);
11893 vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(k1ByteSize, k64BitSize);
11894 } else {
11895 res = &CreateRegisterOperandOfType(sty1); /* result type same as sty1 */
11896 SelectCopy(*res, sty1, *src1, sty1);
11897 vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(sty1);
11898 }
11899 VectorRegSpec *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(sty2);
11900
11901 MOperator mop;
11902 if (IsUnsignedInteger(sty1)) {
11903 mop = GetPrimTypeSize(sty1) > k8ByteSize ? MOP_vupadalvv : MOP_vupadaluu;
11904 } else {
11905 mop = GetPrimTypeSize(sty1) > k8ByteSize ? MOP_vspadalvv : MOP_vspadaluu;
11906 }
11907
11908 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
11909 vInsn.AddOpndChain(*res).AddOpndChain(*src2);
11910 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpecSrc);
11911 GetCurBB()->AppendInsn(vInsn);
11912 if (!IsPrimitiveVector(sty1)) {
11913 res = AdjustOneElementVectorOperand(sty1, res);
11914 }
11915 return res;
11916 }
11917
SelectVectorPairwiseAdd(PrimType rType,Operand * src,PrimType sType)11918 RegOperand *AArch64CGFunc::SelectVectorPairwiseAdd(PrimType rType, Operand *src, PrimType sType)
11919 {
11920 PrimType oType = rType;
11921 rType = FilterOneElementVectorType(oType);
11922 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
11923 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
11924 VectorRegSpec *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(sType); /* source operand */
11925
11926 if (rType == PTY_f64) {
11927 vecSpecDest->vecLaneMax = 1;
11928 }
11929
11930 MOperator mop;
11931 if (IsUnsignedInteger(sType)) {
11932 mop = GetPrimTypeSize(sType) > k8ByteSize ? MOP_vupaddvv : MOP_vupadduu;
11933 } else {
11934 mop = GetPrimTypeSize(sType) > k8ByteSize ? MOP_vspaddvv : MOP_vspadduu;
11935 }
11936
11937 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
11938 vInsn.AddOpndChain(*res).AddOpndChain(*src);
11939 /* dest pushed first, popped first */
11940 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpecSrc);
11941 GetCurBB()->AppendInsn(vInsn);
11942 if (oType != rType) {
11943 res = AdjustOneElementVectorOperand(oType, res);
11944 }
11945 return res;
11946 }
11947
SelectVectorSetElement(Operand * eOpnd,PrimType eType,Operand * vOpnd,PrimType vType,int32 lane)11948 RegOperand *AArch64CGFunc::SelectVectorSetElement(Operand *eOpnd, PrimType eType, Operand *vOpnd, PrimType vType,
11949 int32 lane)
11950 {
11951 if (!IsPrimitiveVector(vType)) {
11952 return SelectOneElementVectorCopy(eOpnd, eType);
11953 }
11954 RegOperand *reg = &CreateRegisterOperandOfType(eType); /* vector element type */
11955 SelectCopy(*reg, eType, *eOpnd, eType);
11956 VectorRegSpec *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(vType, lane); /* vector operand == result */
11957
11958 MOperator mOp;
11959 if (GetPrimTypeSize(eType) > k4ByteSize) {
11960 mOp = GetPrimTypeSize(vType) > k8ByteSize ? MOP_vxinsvr : MOP_vxinsur;
11961 } else {
11962 mOp = GetPrimTypeSize(vType) > k8ByteSize ? MOP_vwinsvr : MOP_vwinsur;
11963 }
11964
11965 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
11966 vInsn.AddOpndChain(*vOpnd).AddOpndChain(*reg);
11967 vInsn.PushRegSpecEntry(vecSpecSrc);
11968 GetCurBB()->AppendInsn(vInsn);
11969 return static_cast<RegOperand *>(vOpnd);
11970 }
11971
SelectVectorAbsSubL(PrimType rType,Operand * o1,Operand * o2,PrimType oTy,bool isLow)11972 RegOperand *AArch64CGFunc::SelectVectorAbsSubL(PrimType rType, Operand *o1, Operand *o2, PrimType oTy, bool isLow)
11973 {
11974 RegOperand *res = &CreateRegisterOperandOfType(rType);
11975 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
11976 VectorRegSpec *vecSpecOpd1 = GetMemoryPool()->New<VectorRegSpec>(oTy);
11977 VectorRegSpec *vecSpecOpd2 = GetMemoryPool()->New<VectorRegSpec>(oTy); /* same opnd types */
11978
11979 MOperator mop;
11980 if (isLow) {
11981 mop = IsPrimitiveUnSignedVector(rType) ? MOP_vuabdlvuu : MOP_vsabdlvuu;
11982 } else {
11983 mop = IsPrimitiveUnSignedVector(rType) ? MOP_vuabdl2vvv : MOP_vsabdl2vvv;
11984 }
11985 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
11986 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
11987 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpecOpd1).PushRegSpecEntry(vecSpecOpd2);
11988 GetCurBB()->AppendInsn(vInsn);
11989 return res;
11990 }
11991
SelectVectorMerge(PrimType rType,Operand * o1,Operand * o2,int32 index)11992 RegOperand *AArch64CGFunc::SelectVectorMerge(PrimType rType, Operand *o1, Operand *o2, int32 index)
11993 {
11994 if (!IsPrimitiveVector(rType)) {
11995 static_cast<RegOperand *>(o1)->SetIF64Vec();
11996 return static_cast<RegOperand *>(o1); /* 64x1_t, index equals 0 */
11997 }
11998 RegOperand *res = &CreateRegisterOperandOfType(rType);
11999 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12000 VectorRegSpec *vecSpecOpd1 = GetMemoryPool()->New<VectorRegSpec>(rType);
12001 VectorRegSpec *vecSpecOpd2 = GetMemoryPool()->New<VectorRegSpec>(rType);
12002
12003 ImmOperand *imm = &CreateImmOperand(index, k8BitSize, true);
12004
12005 MOperator mOp = (GetPrimTypeSize(rType) > k8ByteSize) ? MOP_vextvvvi : MOP_vextuuui;
12006 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12007 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2).AddOpndChain(*imm);
12008 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpecOpd1).PushRegSpecEntry(vecSpecOpd2);
12009 GetCurBB()->AppendInsn(vInsn);
12010 return res;
12011 }
12012
SelectVectorReverse(PrimType rType,Operand * src,PrimType sType,uint32 size)12013 RegOperand *AArch64CGFunc::SelectVectorReverse(PrimType rType, Operand *src, PrimType sType, uint32 size)
12014 {
12015 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12016 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12017 VectorRegSpec *vecSpecSrc = GetMemoryPool()->New<VectorRegSpec>(sType); /* vector operand */
12018
12019 MOperator mOp;
12020 if (GetPrimTypeBitSize(rType) == k128BitSize) {
12021 mOp = size >= k64BitSize ? MOP_vrev64qq : (size >= k32BitSize ? MOP_vrev32qq : MOP_vrev16qq);
12022 } else if (GetPrimTypeBitSize(rType) == k64BitSize) {
12023 mOp = size >= k64BitSize ? MOP_vrev64dd : (size >= k32BitSize ? MOP_vrev32dd : MOP_vrev16dd);
12024 } else {
12025 CHECK_FATAL(false, "should not be here");
12026 }
12027 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12028 vInsn.AddOpndChain(*res).AddOpndChain(*src);
12029 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpecSrc);
12030 GetCurBB()->AppendInsn(vInsn);
12031 return res;
12032 }
12033
SelectVectorSum(PrimType rType,Operand * o1,PrimType oType)12034 RegOperand *AArch64CGFunc::SelectVectorSum(PrimType rType, Operand *o1, PrimType oType)
12035 {
12036 RegOperand *res = &CreateRegisterOperandOfType(rType); /* uint32_t result */
12037 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oType);
12038 RegOperand *iOpnd = &CreateRegisterOperandOfType(oType); /* float intermediate result */
12039 uint32 eSize = GetVecEleSize(oType); /* vector opd in bits */
12040 bool is16ByteVec = GetPrimTypeSize(oType) >= k16ByteSize;
12041 MOperator mOp;
12042 if (is16ByteVec) {
12043 mOp = eSize <= k8BitSize
12044 ? MOP_vbaddvrv
12045 : (eSize <= k16BitSize ? MOP_vhaddvrv : (eSize <= k32BitSize ? MOP_vsaddvrv : MOP_vdaddvrv));
12046 } else {
12047 mOp = eSize <= k8BitSize ? MOP_vbaddvru : (eSize <= k16BitSize ? MOP_vhaddvru : MOP_vsaddvru);
12048 }
12049 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12050 vInsn.AddOpndChain(*iOpnd).AddOpndChain(*o1);
12051 vInsn.PushRegSpecEntry(vecSpec1);
12052 GetCurBB()->AppendInsn(vInsn);
12053
12054 mOp = eSize > k32BitSize ? MOP_vxmovrv : MOP_vwmovrv;
12055 VectorInsn &vInsn2 = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12056 auto *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(oType);
12057 vInsn2.AddOpndChain(*res).AddOpndChain(*iOpnd);
12058 vecSpec2->vecLane = 0;
12059 vInsn2.PushRegSpecEntry(vecSpec2);
12060 GetCurBB()->AppendInsn(vInsn2);
12061 return res;
12062 }
12063
PrepareVectorOperands(Operand ** o1,PrimType & oty1,Operand ** o2,PrimType & oty2)12064 void AArch64CGFunc::PrepareVectorOperands(Operand **o1, PrimType &oty1, Operand **o2, PrimType &oty2)
12065 {
12066 /* Only 1 operand can be non vector, otherwise it's a scalar operation, wouldn't come here */
12067 if (IsPrimitiveVector(oty1) == IsPrimitiveVector(oty2)) {
12068 return;
12069 }
12070 PrimType origTyp = !IsPrimitiveVector(oty2) ? oty2 : oty1;
12071 Operand *opd = !IsPrimitiveVector(oty2) ? *o2 : *o1;
12072 PrimType rType = !IsPrimitiveVector(oty2) ? oty1 : oty2; /* Type to dup into */
12073 RegOperand *res = &CreateRegisterOperandOfType(rType);
12074 VectorRegSpec *vecSpec = GetMemoryPool()->New<VectorRegSpec>(rType);
12075
12076 bool immOpnd = false;
12077 if (opd->IsConstImmediate()) {
12078 int64 val = static_cast<ImmOperand *>(opd)->GetValue();
12079 if (val >= kMinImmVal && val <= kMaxImmVal && GetVecEleSize(rType) < k64BitSize) {
12080 immOpnd = true;
12081 } else {
12082 RegOperand *regOpd = &CreateRegisterOperandOfType(origTyp);
12083 SelectCopyImm(*regOpd, origTyp, static_cast<ImmOperand &>(*opd), origTyp);
12084 opd = static_cast<Operand *>(regOpd);
12085 }
12086 }
12087
12088 /* need dup to vector operand */
12089 MOperator mOp;
12090 if (immOpnd) {
12091 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vmovvi : MOP_vmovui; /* a const */
12092 } else {
12093 if (GetPrimTypeSize(origTyp) > k4ByteSize) {
12094 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vxdupvr : MOP_vxdupur;
12095 } else {
12096 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vwdupvr : MOP_vwdupur; /* a scalar var */
12097 }
12098 }
12099 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12100 vInsn.AddOpndChain(*res).AddOpndChain(*opd);
12101 vInsn.PushRegSpecEntry(vecSpec);
12102 GetCurBB()->AppendInsn(vInsn);
12103 if (!IsPrimitiveVector(oty2)) {
12104 *o2 = static_cast<Operand *>(res);
12105 oty2 = rType;
12106 } else {
12107 *o1 = static_cast<Operand *>(res);
12108 oty1 = rType;
12109 }
12110 }
12111
SelectVectorCvt(Operand * res,PrimType rType,Operand * o1,PrimType oType)12112 void AArch64CGFunc::SelectVectorCvt(Operand *res, PrimType rType, Operand *o1, PrimType oType)
12113 {
12114 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12115 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oType); /* vector operand 1 */
12116
12117 MOperator mOp;
12118 VectorInsn *insn;
12119 if (GetPrimTypeSize(rType) > GetPrimTypeSize(oType)) {
12120 /* expand, similar to vmov_XX() intrinsics */
12121 mOp = IsUnsignedInteger(rType) ? MOP_vushllvvi : MOP_vshllvvi;
12122 ImmOperand *imm = &CreateImmOperand(0, k8BitSize, true);
12123 insn = &GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12124 insn->AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*imm);
12125 } else if (GetPrimTypeSize(rType) < GetPrimTypeSize(oType)) {
12126 /* extract, similar to vqmovn_XX() intrinsics */
12127 insn = &GetInsnBuilder()->BuildVectorInsn(MOP_vxtnuv, AArch64CG::kMd[MOP_vxtnuv]);
12128 insn->AddOpndChain(*res).AddOpndChain(*o1);
12129 } else {
12130 CHECK_FATAL(0, "Invalid cvt between 2 operands of the same size");
12131 }
12132 insn->PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1);
12133 GetCurBB()->AppendInsn(*insn);
12134 }
12135
SelectVectorCompareZero(Operand * o1,PrimType oty1,Operand * o2,Opcode opc)12136 RegOperand *AArch64CGFunc::SelectVectorCompareZero(Operand *o1, PrimType oty1, Operand *o2, Opcode opc)
12137 {
12138 if (IsUnsignedInteger(oty1) && (opc != OP_eq && opc != OP_ne)) {
12139 return nullptr; /* no unsigned instr for zero */
12140 }
12141 RegOperand *res = &CreateRegisterOperandOfType(oty1); /* result operand */
12142 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(oty1);
12143 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oty1); /* vector operand 1 */
12144
12145 MOperator mOp;
12146 switch (opc) {
12147 case OP_eq:
12148 case OP_ne:
12149 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vzcmeqvv : MOP_vzcmequu;
12150 break;
12151 case OP_gt:
12152 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vzcmgtvv : MOP_vzcmgtuu;
12153 break;
12154 case OP_ge:
12155 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vzcmgevv : MOP_vzcmgeuu;
12156 break;
12157 case OP_lt:
12158 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vzcmltvv : MOP_vzcmltuu;
12159 break;
12160 case OP_le:
12161 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vzcmlevv : MOP_vzcmleuu;
12162 break;
12163 default:
12164 CHECK_FATAL(0, "Invalid cc in vector compare");
12165 }
12166 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12167 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12168 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1);
12169 GetCurBB()->AppendInsn(vInsn);
12170 if (opc == OP_ne) {
12171 res = SelectVectorNot(oty1, res);
12172 }
12173 return res;
12174 }
12175
12176 /* Neon compare intrinsics always return unsigned vector, MapleIR for comparison always return
12177 signed. Using type of 1st operand for operation here */
SelectVectorCompare(Operand * o1,PrimType oty1,Operand * o2,PrimType oty2,Opcode opc)12178 RegOperand *AArch64CGFunc::SelectVectorCompare(Operand *o1, PrimType oty1, Operand *o2, PrimType oty2, Opcode opc)
12179 {
12180 if (o2->IsConstImmediate() && static_cast<ImmOperand *>(o2)->GetValue() == 0) {
12181 RegOperand *zeroCmp = SelectVectorCompareZero(o1, oty1, o2, opc);
12182 if (zeroCmp != nullptr) {
12183 return zeroCmp;
12184 }
12185 }
12186 PrepareVectorOperands(&o1, oty1, &o2, oty2);
12187 DEBUG_ASSERT(oty1 == oty2, "vector operand type mismatch");
12188
12189 RegOperand *res = &CreateRegisterOperandOfType(oty1); /* result operand */
12190 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(oty1);
12191 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oty1); /* vector operand 1 */
12192 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(oty2); /* vector operand 2 */
12193
12194 MOperator mOp;
12195 switch (opc) {
12196 case OP_eq:
12197 case OP_ne:
12198 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vcmeqvvv : MOP_vcmequuu;
12199 break;
12200 case OP_lt:
12201 case OP_gt:
12202 if (IsUnsignedInteger(oty1)) {
12203 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vcmhivvv : MOP_vcmhiuuu;
12204 } else {
12205 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vcmgtvvv : MOP_vcmgtuuu;
12206 }
12207 break;
12208 case OP_le:
12209 case OP_ge:
12210 if (IsUnsignedInteger(oty1)) {
12211 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vcmhsvvv : MOP_vcmhsuuu;
12212 } else {
12213 mOp = GetPrimTypeSize(oty1) > k8ByteSize ? MOP_vcmgevvv : MOP_vcmgeuuu;
12214 }
12215 break;
12216 default:
12217 CHECK_FATAL(0, "Invalid cc in vector compare");
12218 }
12219 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12220 if (opc == OP_lt || opc == OP_le) {
12221 vInsn.AddOpndChain(*res).AddOpndChain(*o2).AddOpndChain(*o1);
12222 } else {
12223 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12224 }
12225 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
12226 GetCurBB()->AppendInsn(vInsn);
12227 if (opc == OP_ne) {
12228 res = SelectVectorNot(oty1, res);
12229 }
12230 return res;
12231 }
12232
SelectVectorShift(PrimType rType,Operand * o1,PrimType oty1,Operand * o2,PrimType oty2,Opcode opc)12233 RegOperand *AArch64CGFunc::SelectVectorShift(PrimType rType, Operand *o1, PrimType oty1, Operand *o2, PrimType oty2,
12234 Opcode opc)
12235 {
12236 PrepareVectorOperands(&o1, oty1, &o2, oty2);
12237 PrimType resultType = rType;
12238 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12239 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12240 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 2 */
12241
12242 if (!IsPrimitiveVector(rType)) {
12243 o1 = &SelectCopy(*o1, rType, PTY_f64);
12244 o2 = &SelectCopy(*o2, rType, PTY_f64);
12245 resultType = PTY_f64;
12246 }
12247 RegOperand *res = &CreateRegisterOperandOfType(resultType); /* result operand */
12248
12249 /* signed and unsigned shl(v,v) both use sshl or ushl, they are the same */
12250 MOperator mOp;
12251 if (IsPrimitiveUnsigned(rType)) {
12252 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vushlvvv : MOP_vushluuu;
12253 } else {
12254 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vshlvvv : MOP_vshluuu;
12255 }
12256
12257 if (opc != OP_shl) {
12258 o2 = SelectVectorNeg(rType, o2);
12259 }
12260 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12261 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12262 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
12263 GetCurBB()->AppendInsn(vInsn);
12264 return res;
12265 }
12266
ValidShiftConst(PrimType rType)12267 uint32 ValidShiftConst(PrimType rType)
12268 {
12269 switch (rType) {
12270 case PTY_v8u8:
12271 case PTY_v8i8:
12272 case PTY_v16u8:
12273 case PTY_v16i8:
12274 return k8BitSize;
12275 case PTY_v4u16:
12276 case PTY_v4i16:
12277 case PTY_v8u16:
12278 case PTY_v8i16:
12279 return k16BitSize;
12280 case PTY_v2u32:
12281 case PTY_v2i32:
12282 case PTY_v4u32:
12283 case PTY_v4i32:
12284 return k32BitSize;
12285 case PTY_v2u64:
12286 case PTY_v2i64:
12287 return k64BitSize;
12288 default:
12289 CHECK_FATAL(0, "Invalid Shift operand type");
12290 }
12291 return 0;
12292 }
12293
SelectVectorShiftImm(PrimType rType,Operand * o1,Operand * imm,int32 sVal,Opcode opc)12294 RegOperand *AArch64CGFunc::SelectVectorShiftImm(PrimType rType, Operand *o1, Operand *imm, int32 sVal, Opcode opc)
12295 {
12296 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12297 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12298 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12299
12300 if (!imm->IsConstImmediate()) {
12301 CHECK_FATAL(0, "VectorUShiftImm has invalid shift const");
12302 }
12303 uint32 shift = static_cast<uint32>(ValidShiftConst(rType));
12304 bool needDup = false;
12305 if (opc == OP_shl) {
12306 if ((shift == k8BitSize && (sVal < 0 || static_cast<uint32>(sVal) >= shift)) ||
12307 (shift == k16BitSize && (sVal < 0 || static_cast<uint32>(sVal) >= shift)) ||
12308 (shift == k32BitSize && (sVal < 0 || static_cast<uint32>(sVal) >= shift)) ||
12309 (shift == k64BitSize && (sVal < 0 || static_cast<uint32>(sVal) >= shift))) {
12310 needDup = true;
12311 }
12312 } else {
12313 if ((shift == k8BitSize && (sVal < 1 || static_cast<uint32>(sVal) > shift)) ||
12314 (shift == k16BitSize && (sVal < 1 || static_cast<uint32>(sVal) > shift)) ||
12315 (shift == k32BitSize && (sVal < 1 || static_cast<uint32>(sVal) > shift)) ||
12316 (shift == k64BitSize && (sVal < 1 || static_cast<uint32>(sVal) > shift))) {
12317 needDup = true;
12318 }
12319 }
12320 if (needDup) {
12321 /* Dup constant to vector reg */
12322 MOperator mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vmovvi : MOP_vmovui;
12323 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12324 vInsn.AddOpndChain(*res).AddOpndChain(*imm);
12325 vInsn.PushRegSpecEntry(vecSpecDest);
12326 GetCurBB()->AppendInsn(vInsn);
12327 res = SelectVectorShift(rType, o1, rType, res, rType, opc);
12328 return res;
12329 }
12330 MOperator mOp;
12331 if (GetPrimTypeSize(rType) > k8ByteSize) {
12332 if (IsUnsignedInteger(rType)) {
12333 mOp = opc == OP_shl ? MOP_vushlvvi : MOP_vushrvvi;
12334 } else {
12335 mOp = opc == OP_shl ? MOP_vushlvvi : MOP_vshrvvi;
12336 }
12337 } else {
12338 if (IsUnsignedInteger(rType)) {
12339 mOp = opc == OP_shl ? MOP_vushluui : MOP_vushruui;
12340 } else {
12341 mOp = opc == OP_shl ? MOP_vushluui : MOP_vshruui;
12342 }
12343 }
12344 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12345 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*imm);
12346 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1);
12347 GetCurBB()->AppendInsn(vInsn);
12348 return res;
12349 }
12350
SelectVectorTableLookup(PrimType rType,Operand * o1,Operand * o2)12351 RegOperand *AArch64CGFunc::SelectVectorTableLookup(PrimType rType, Operand *o1, Operand *o2)
12352 {
12353 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12354 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType); /* 8B or 16B */
12355 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12356 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 2 */
12357 vecSpec1->compositeOpnds = 1; /* composite operand */
12358
12359 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(MOP_vtbl1vvv, AArch64CG::kMd[MOP_vtbl1vvv]);
12360 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12361 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
12362 GetCurBB()->AppendInsn(vInsn);
12363 return res;
12364 }
12365
SelectVectorMadd(Operand * o1,PrimType oTyp1,Operand * o2,PrimType oTyp2,Operand * o3,PrimType oTyp3)12366 RegOperand *AArch64CGFunc::SelectVectorMadd(Operand *o1, PrimType oTyp1, Operand *o2, PrimType oTyp2, Operand *o3,
12367 PrimType oTyp3)
12368 {
12369 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oTyp1); /* operand 1 and result */
12370 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(oTyp2); /* vector operand 2 */
12371 VectorRegSpec *vecSpec3 = GetMemoryPool()->New<VectorRegSpec>(oTyp3); /* vector operand 2 */
12372
12373 MOperator mop = IsPrimitiveUnSignedVector(oTyp1) ? MOP_vumaddvvv : MOP_vsmaddvvv;
12374 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
12375 vInsn.AddOpndChain(*o1).AddOpndChain(*o2).AddOpndChain(*o3);
12376 vInsn.PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2).PushRegSpecEntry(vecSpec3);
12377 GetCurBB()->AppendInsn(vInsn);
12378 return static_cast<RegOperand *>(o1);
12379 }
12380
SelectVectorMull(PrimType rType,Operand * o1,PrimType oTyp1,Operand * o2,PrimType oTyp2,bool isLow)12381 RegOperand *AArch64CGFunc::SelectVectorMull(PrimType rType, Operand *o1, PrimType oTyp1, Operand *o2, PrimType oTyp2,
12382 bool isLow)
12383 {
12384 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12385 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12386 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oTyp1); /* vector operand 1 */
12387 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(oTyp2); /* vector operand 1 */
12388
12389 MOperator mop;
12390 if (isLow) {
12391 mop = IsPrimitiveUnSignedVector(rType) ? MOP_vumullvvv : MOP_vsmullvvv;
12392 } else {
12393 mop = IsPrimitiveUnSignedVector(rType) ? MOP_vumull2vvv : MOP_vsmull2vvv;
12394 }
12395 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mop, AArch64CG::kMd[mop]);
12396 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12397 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
12398 GetCurBB()->AppendInsn(vInsn);
12399 return res;
12400 }
12401
SelectVectorBinOp(PrimType rType,Operand * o1,PrimType oty1,Operand * o2,PrimType oty2,Opcode opc)12402 RegOperand *AArch64CGFunc::SelectVectorBinOp(PrimType rType, Operand *o1, PrimType oty1, Operand *o2, PrimType oty2,
12403 Opcode opc)
12404 {
12405 PrepareVectorOperands(&o1, oty1, &o2, oty2);
12406 DEBUG_ASSERT(oty1 == oty2, "vector operand type mismatch");
12407
12408 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12409 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12410 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oty1); /* source operand 1 */
12411 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(oty2); /* source operand 2 */
12412
12413 MOperator mOp;
12414 if (opc == OP_add) {
12415 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vaddvvv : MOP_vadduuu;
12416 } else if (opc == OP_sub) {
12417 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vsubvvv : MOP_vsubuuu;
12418 } else if (opc == OP_mul) {
12419 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vmulvvv : MOP_vmuluuu;
12420 } else {
12421 CHECK_FATAL(0, "Invalid opcode for SelectVectorBinOp");
12422 }
12423 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12424 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12425 /* dest pushed first, popped first */
12426 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
12427 GetCurBB()->AppendInsn(vInsn);
12428 return res;
12429 }
12430
SelectVectorBitwiseOp(PrimType rType,Operand * o1,PrimType oty1,Operand * o2,PrimType oty2,Opcode opc)12431 RegOperand *AArch64CGFunc::SelectVectorBitwiseOp(PrimType rType, Operand *o1, PrimType oty1, Operand *o2, PrimType oty2,
12432 Opcode opc)
12433 {
12434 PrepareVectorOperands(&o1, oty1, &o2, oty2);
12435 DEBUG_ASSERT(oty1 == oty2, "vector operand type mismatch");
12436
12437 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12438 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12439 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12440 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12441
12442 MOperator mOp;
12443 if (opc == OP_band) {
12444 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vandvvv : MOP_vanduuu;
12445 } else if (opc == OP_bior) {
12446 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vorvvv : MOP_voruuu;
12447 } else if (opc == OP_bxor) {
12448 mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vxorvvv : MOP_vxoruuu;
12449 } else {
12450 CHECK_FATAL(0, "Invalid opcode for SelectVectorBitwiseOp");
12451 }
12452 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12453 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12454 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1).PushRegSpecEntry(vecSpec2);
12455 GetCurBB()->AppendInsn(vInsn);
12456 return res;
12457 }
12458
SelectVectorNarrow(PrimType rType,Operand * o1,PrimType otyp)12459 RegOperand *AArch64CGFunc::SelectVectorNarrow(PrimType rType, Operand *o1, PrimType otyp)
12460 {
12461 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12462 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12463 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(otyp); /* vector operand */
12464
12465 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(MOP_vxtnuv, AArch64CG::kMd[MOP_vxtnuv]);
12466 vInsn.AddOpndChain(*res).AddOpndChain(*o1);
12467 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1);
12468 GetCurBB()->AppendInsn(vInsn);
12469 return res;
12470 }
12471
SelectVectorNarrow2(PrimType rType,Operand * o1,PrimType oty1,Operand * o2,PrimType oty2)12472 RegOperand *AArch64CGFunc::SelectVectorNarrow2(PrimType rType, Operand *o1, PrimType oty1, Operand *o2, PrimType oty2)
12473 {
12474 (void)oty1; /* 1st opnd was loaded already, type no longer needed */
12475 RegOperand *res = static_cast<RegOperand *>(o1); /* o1 is also the result */
12476 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12477 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(oty2); /* vector opnd2 */
12478
12479 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(MOP_vxtn2uv, AArch64CG::kMd[MOP_vxtn2uv]);
12480 vInsn.AddOpndChain(*res).AddOpndChain(*o2);
12481 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec2);
12482 GetCurBB()->AppendInsn(vInsn);
12483 return res;
12484 }
12485
SelectVectorNot(PrimType rType,Operand * o1)12486 RegOperand *AArch64CGFunc::SelectVectorNot(PrimType rType, Operand *o1)
12487 {
12488 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12489 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12490 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12491
12492 MOperator mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vnotvv : MOP_vnotuu;
12493 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12494 vInsn.AddOpndChain(*res).AddOpndChain(*o1);
12495 vInsn.PushRegSpecEntry(vecSpecDest).PushRegSpecEntry(vecSpec1);
12496 GetCurBB()->AppendInsn(vInsn);
12497 return res;
12498 }
12499
SelectVectorNeg(PrimType rType,Operand * o1)12500 RegOperand *AArch64CGFunc::SelectVectorNeg(PrimType rType, Operand *o1)
12501 {
12502 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12503 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12504 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12505
12506 MOperator mOp = GetPrimTypeSize(rType) > k8ByteSize ? MOP_vnegvv : MOP_vneguu;
12507 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12508 vInsn.AddOpndChain(*res).AddOpndChain(*o1);
12509 vInsn.PushRegSpecEntry(vecSpecDest);
12510 vInsn.PushRegSpecEntry(vecSpec1);
12511 GetCurBB()->AppendInsn(vInsn);
12512 return res;
12513 }
12514
12515 /*
12516 * Called internally for auto-vec, no intrinsics for now
12517 */
SelectVectorSelect(Operand & cond,PrimType rType,Operand & o0,Operand & o1)12518 RegOperand *AArch64CGFunc::SelectVectorSelect(Operand &cond, PrimType rType, Operand &o0, Operand &o1)
12519 {
12520 rType = GetPrimTypeSize(rType) > k8ByteSize ? PTY_v16u8 : PTY_v8u8;
12521 RegOperand *res = &CreateRegisterOperandOfType(rType);
12522 SelectCopy(*res, rType, cond, rType);
12523 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12524 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType);
12525 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(rType);
12526
12527 uint32 mOp = GetPrimTypeBitSize(rType) > k64BitSize ? MOP_vbslvvv : MOP_vbsluuu;
12528 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12529 vInsn.AddOpndChain(*res).AddOpndChain(o0).AddOpndChain(o1);
12530 vInsn.PushRegSpecEntry(vecSpecDest);
12531 vInsn.PushRegSpecEntry(vecSpec1);
12532 vInsn.PushRegSpecEntry(vecSpec2);
12533 GetCurBB()->AppendInsn(vInsn);
12534 return res;
12535 }
12536
SelectVectorShiftRNarrow(PrimType rType,Operand * o1,PrimType oType,Operand * o2,bool isLow)12537 RegOperand *AArch64CGFunc::SelectVectorShiftRNarrow(PrimType rType, Operand *o1, PrimType oType, Operand *o2,
12538 bool isLow)
12539 {
12540 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12541 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12542 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(oType); /* vector operand 1 */
12543
12544 ImmOperand *imm = static_cast<ImmOperand *>(o2);
12545 MOperator mOp;
12546 if (isLow) {
12547 mOp = MOP_vshrnuvi;
12548 } else {
12549 CHECK_FATAL(0, "NYI: vshrn_high_");
12550 }
12551 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12552 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*imm);
12553 vInsn.PushRegSpecEntry(vecSpecDest);
12554 vInsn.PushRegSpecEntry(vecSpec1);
12555 GetCurBB()->AppendInsn(vInsn);
12556 return res;
12557 }
12558
SelectVectorSubWiden(PrimType resType,Operand * o1,PrimType otyp1,Operand * o2,PrimType otyp2,bool isLow,bool isWide)12559 RegOperand *AArch64CGFunc::SelectVectorSubWiden(PrimType resType, Operand *o1, PrimType otyp1, Operand *o2,
12560 PrimType otyp2, bool isLow, bool isWide)
12561 {
12562 RegOperand *res = &CreateRegisterOperandOfType(resType); /* result reg */
12563 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(resType);
12564 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(otyp1); /* vector operand 1 */
12565 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(otyp2); /* vector operand 2 */
12566
12567 MOperator mOp;
12568 if (!isWide) {
12569 if (isLow) {
12570 mOp = IsUnsignedInteger(otyp1) ? MOP_vusublvuu : MOP_vssublvuu;
12571 } else {
12572 mOp = IsUnsignedInteger(otyp1) ? MOP_vusubl2vvv : MOP_vssubl2vvv;
12573 }
12574 } else {
12575 if (isLow) {
12576 mOp = IsUnsignedInteger(otyp1) ? MOP_vusubwvvu : MOP_vssubwvvu;
12577 } else {
12578 mOp = IsUnsignedInteger(otyp1) ? MOP_vusubw2vvv : MOP_vssubw2vvv;
12579 }
12580 }
12581 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12582 vInsn.AddOpndChain(*res).AddOpndChain(*o1).AddOpndChain(*o2);
12583 vInsn.PushRegSpecEntry(vecSpecDest);
12584 vInsn.PushRegSpecEntry(vecSpec1);
12585 vInsn.PushRegSpecEntry(vecSpec2);
12586 GetCurBB()->AppendInsn(vInsn);
12587 return res;
12588 }
12589
SelectVectorZip(PrimType rType,Operand * o1,Operand * o2)12590 void AArch64CGFunc::SelectVectorZip(PrimType rType, Operand *o1, Operand *o2)
12591 {
12592 RegOperand *res1 = &CreateRegisterOperandOfType(rType); /* result operand 1 */
12593 RegOperand *res2 = &CreateRegisterOperandOfType(rType); /* result operand 2 */
12594 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12595 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 1 */
12596 VectorRegSpec *vecSpec2 = GetMemoryPool()->New<VectorRegSpec>(rType); /* vector operand 2 */
12597
12598 VectorInsn &vInsn1 = GetInsnBuilder()->BuildVectorInsn(MOP_vzip1vvv, AArch64CG::kMd[MOP_vzip1vvv]);
12599 vInsn1.AddOpndChain(*res1).AddOpndChain(*o1).AddOpndChain(*o2);
12600 vInsn1.PushRegSpecEntry(vecSpecDest);
12601 vInsn1.PushRegSpecEntry(vecSpec1);
12602 vInsn1.PushRegSpecEntry(vecSpec2);
12603 GetCurBB()->AppendInsn(vInsn1);
12604
12605 VectorInsn &vInsn2 = GetInsnBuilder()->BuildVectorInsn(MOP_vzip2vvv, AArch64CG::kMd[MOP_vzip2vvv]);
12606 vInsn2.AddOpndChain(*res2).AddOpndChain(*o1).AddOpndChain(*o2);
12607 vInsn2.PushRegSpecEntry(vecSpecDest);
12608 vInsn2.PushRegSpecEntry(vecSpec1);
12609 vInsn2.PushRegSpecEntry(vecSpec2);
12610 GetCurBB()->AppendInsn(vInsn2);
12611
12612 if (GetPrimTypeSize(rType) <= k16ByteSize) {
12613 Operand *preg1 = &GetOrCreatePhysicalRegisterOperand(V0, k64BitSize, kRegTyFloat);
12614 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xvmovd, *preg1, *res1));
12615 Operand *preg2 = &GetOrCreatePhysicalRegisterOperand(V1, k64BitSize, kRegTyFloat);
12616 GetCurBB()->AppendInsn(GetInsnBuilder()->BuildInsn(MOP_xvmovd, *preg2, *res2));
12617 }
12618 }
12619
SelectVectorWiden(PrimType rType,Operand * o1,PrimType otyp,bool isLow)12620 RegOperand *AArch64CGFunc::SelectVectorWiden(PrimType rType, Operand *o1, PrimType otyp, bool isLow)
12621 {
12622 RegOperand *res = &CreateRegisterOperandOfType(rType); /* result operand */
12623 VectorRegSpec *vecSpecDest = GetMemoryPool()->New<VectorRegSpec>(rType);
12624 VectorRegSpec *vecSpec1 = GetMemoryPool()->New<VectorRegSpec>(otyp); /* vector operand */
12625
12626 MOperator mOp;
12627 if (isLow) {
12628 mOp = IsPrimitiveUnSignedVector(rType) ? MOP_vuxtlvu : MOP_vsxtlvu;
12629 } else {
12630 mOp = IsPrimitiveUnSignedVector(rType) ? MOP_vuxtl2vv : MOP_vsxtl2vv;
12631 }
12632 VectorInsn &vInsn = GetInsnBuilder()->BuildVectorInsn(mOp, AArch64CG::kMd[mOp]);
12633 vInsn.AddOpndChain(*res).AddOpndChain(*o1);
12634 vInsn.PushRegSpecEntry(vecSpecDest);
12635 vInsn.PushRegSpecEntry(vecSpec1);
12636 GetCurBB()->AppendInsn(vInsn);
12637 return res;
12638 }
12639
12640 /*
12641 * Check the distance between the first insn of BB with the lable(targ_labidx)
12642 * and the insn with targ_id. If the distance greater than kShortBRDistance
12643 * return false.
12644 */
DistanceCheck(const BB & bb,LabelIdx targLabIdx,uint32 targId) const12645 bool AArch64CGFunc::DistanceCheck(const BB &bb, LabelIdx targLabIdx, uint32 targId) const
12646 {
12647 for (auto *tBB : bb.GetSuccs()) {
12648 if (tBB->GetLabIdx() != targLabIdx) {
12649 continue;
12650 }
12651 Insn *tInsn = tBB->GetFirstInsn();
12652 while (tInsn == nullptr || !tInsn->IsMachineInstruction()) {
12653 if (tInsn == nullptr) {
12654 tBB = tBB->GetNext();
12655 if (tBB == nullptr) { /* tailcallopt may make the target block empty */
12656 return true;
12657 }
12658 tInsn = tBB->GetFirstInsn();
12659 } else {
12660 tInsn = tInsn->GetNext();
12661 }
12662 }
12663 uint32 tmp = (tInsn->GetId() > targId) ? (tInsn->GetId() - targId) : (targId - tInsn->GetId());
12664 return (tmp < kShortBRDistance);
12665 }
12666 CHECK_FATAL(false, "CFG error");
12667 }
12668 } /* namespace maplebe */
12669