1 /**
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 /*
16 Encoder (implementation of math and mem Low-level emitters)
17 */
18
19 #include <aarch32/disasm-aarch32.h>
20 #include "compiler/optimizer/code_generator/relocations.h"
21 #include "operands.h"
22 #include "target/aarch32/target.h"
23 #include "lib_helpers.inl"
24 #include "encode.h"
25
26 #ifndef PANDA_TARGET_MACOS
27 #include "elf.h"
28 #endif // PANDA_TARGET_MACOS
29
30 #include <iomanip>
31
32 namespace panda::compiler::aarch32 {
BindLabel(LabelId id)33 void Aarch32LabelHolder::BindLabel(LabelId id)
34 {
35 static_cast<Aarch32Encoder *>(GetEncoder())->GetMasm()->Bind(labels_[id]);
36 }
37
Aarch32Encoder(ArenaAllocator * allocator)38 Aarch32Encoder::Aarch32Encoder(ArenaAllocator *allocator) : Encoder(allocator, Arch::AARCH32)
39 {
40 labels_ = allocator->New<Aarch32LabelHolder>(this);
41 if (labels_ == nullptr) {
42 SetFalseResult();
43 }
44 EnableLrAsTempReg(true);
45 }
46
~Aarch32Encoder()47 Aarch32Encoder::~Aarch32Encoder()
48 {
49 auto labels = static_cast<Aarch32LabelHolder *>(GetLabels())->labels_;
50 for (auto label : labels) {
51 label->~Label();
52 }
53 if (masm_ != nullptr) {
54 masm_->~MacroAssembler();
55 masm_ = nullptr;
56 }
57 }
58
InitMasm()59 bool Aarch32Encoder::InitMasm()
60 {
61 if (masm_ == nullptr) {
62 auto allocator = GetAllocator();
63
64 // Initialize Masm
65 masm_ = allocator->New<vixl::aarch32::MacroAssembler>(allocator);
66 if (masm_ == nullptr || !masm_->IsValid()) {
67 SetFalseResult();
68 return false;
69 }
70
71 ASSERT(masm_);
72 for (auto reg_code : AARCH32_TMP_REG) {
73 masm_->GetScratchRegisterList()->Combine(vixl::aarch32::Register(reg_code));
74 }
75 for (auto vreg_code : AARCH32_TMP_VREG) {
76 masm_->GetScratchVRegisterList()->Combine(vixl::aarch32::SRegister(vreg_code));
77 }
78
79 // Make sure that the compiler uses the same scratch registers as the assembler
80 CHECK_EQ(RegMask(masm_->GetScratchRegisterList()->GetList()), GetTarget().GetTempRegsMask());
81 CHECK_EQ(RegMask(masm_->GetScratchVRegisterList()->GetList()), GetTarget().GetTempVRegsMask());
82 }
83 return true;
84 }
85
Finalize()86 void Aarch32Encoder::Finalize()
87 {
88 GetMasm()->FinalizeCode();
89 }
90
EncodeJump(LabelHolder::LabelId id)91 void Aarch32Encoder::EncodeJump(LabelHolder::LabelId id)
92 {
93 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
94 GetMasm()->B(label);
95 }
96
EncodeJump(LabelHolder::LabelId id,Reg src0,Reg src1,Condition cc)97 void Aarch32Encoder::EncodeJump(LabelHolder::LabelId id, Reg src0, Reg src1, Condition cc)
98 {
99 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
100 CompareHelper(src0, src1, &cc);
101 GetMasm()->B(Convert(cc), label);
102 }
103
EncodeJumpTest(LabelHolder::LabelId id,Reg src0,Reg src1,Condition cc)104 void Aarch32Encoder::EncodeJumpTest(LabelHolder::LabelId id, Reg src0, Reg src1, Condition cc)
105 {
106 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
107 TestHelper(src0, src1, cc);
108 GetMasm()->B(ConvertTest(cc), label);
109 }
110
EncodeBitTestAndBranch(LabelHolder::LabelId id,Reg reg,uint32_t bit_pos,bool bit_value)111 void Aarch32Encoder::EncodeBitTestAndBranch(LabelHolder::LabelId id, Reg reg, uint32_t bit_pos, bool bit_value)
112 {
113 ASSERT(reg.IsScalar() && reg.GetSize() > bit_pos);
114 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
115 if (reg.GetSize() == DOUBLE_WORD_SIZE) {
116 if (bit_pos < WORD_SIZE) {
117 GetMasm()->tst(VixlReg(reg), VixlImm(1U << bit_pos));
118 } else {
119 GetMasm()->tst(VixlRegU(reg), VixlImm(1U << (bit_pos - WORD_SIZE)));
120 }
121 } else {
122 GetMasm()->tst(VixlReg(reg), VixlImm(1U << bit_pos));
123 }
124 if (bit_value) {
125 GetMasm()->B(Convert(Condition::NE), label);
126 } else {
127 GetMasm()->B(Convert(Condition::EQ), label);
128 }
129 }
130
CompareImmHelper(Reg src,Imm imm,Condition * cc)131 bool Aarch32Encoder::CompareImmHelper(Reg src, Imm imm, Condition *cc)
132 {
133 auto value = GetIntValue(imm);
134 ASSERT(src.IsScalar());
135 ASSERT(value != 0);
136 ASSERT(-static_cast<int64_t>(UINT32_MAX) <= value && value <= UINT32_MAX);
137 ASSERT(CanEncodeImmAddSubCmp(value, src.GetSize(), IsConditionSigned(*cc)));
138
139 return value < 0 ? CompareNegImmHelper(src, value, cc) : ComparePosImmHelper(src, value, cc);
140 }
141
TestImmHelper(Reg src,Imm imm,Condition cc)142 void Aarch32Encoder::TestImmHelper(Reg src, Imm imm, [[maybe_unused]] Condition cc)
143 {
144 auto value = GetIntValue(imm);
145 ASSERT(src.IsScalar());
146 ASSERT(cc == Condition::TST_EQ || cc == Condition::TST_NE);
147 ASSERT(CanEncodeImmLogical(value, src.GetSize()));
148
149 if (src.GetSize() <= WORD_SIZE) {
150 GetMasm()->Tst(VixlReg(src), VixlImm(value));
151 } else {
152 GetMasm()->Tst(VixlRegU(src), VixlImm(0x0));
153 GetMasm()->Tst(Convert(Condition::EQ), VixlReg(src), VixlImm(value));
154 }
155 }
156
CompareNegImmHelper(Reg src,int64_t value,const Condition * cc)157 bool Aarch32Encoder::CompareNegImmHelper(Reg src, int64_t value, const Condition *cc)
158 {
159 if (src.GetSize() <= WORD_SIZE) {
160 GetMasm()->Cmn(VixlReg(src), VixlImm(-value));
161 } else {
162 if (!IsConditionSigned(*cc)) {
163 GetMasm()->Cmn(VixlRegU(src), VixlImm(0x1));
164 GetMasm()->Cmn(Convert(Condition::EQ), VixlReg(src), VixlImm(-value));
165 } else {
166 // There are no effective implementation in this case
167 // Can't get here because of logic behind CanEncodeImmAddSubCmp
168 UNREACHABLE();
169 SetFalseResult();
170 return false;
171 }
172 }
173 return true;
174 }
175
ComparePosImmHelper(Reg src,int64_t value,Condition * cc)176 bool Aarch32Encoder::ComparePosImmHelper(Reg src, int64_t value, Condition *cc)
177 {
178 if (src.GetSize() <= WORD_SIZE) {
179 GetMasm()->Cmp(VixlReg(src), VixlImm(value));
180 } else {
181 if (!IsConditionSigned(*cc)) {
182 GetMasm()->Cmp(VixlRegU(src), VixlImm(0x0));
183 GetMasm()->Cmp(Convert(Condition::EQ), VixlReg(src), VixlImm(value));
184 } else {
185 bool swap = false;
186 switch (*cc) {
187 case Condition::GT:
188 swap = true;
189 *cc = Condition::LT;
190 break;
191 case Condition::LE:
192 swap = true;
193 *cc = Condition::GE;
194 break;
195 case Condition::GE:
196 case Condition::LT:
197 break;
198 default:
199 UNREACHABLE();
200 }
201
202 ScopedTmpRegU32 tmp_reg(this);
203 if (swap) {
204 GetMasm()->Rsbs(VixlReg(tmp_reg), VixlReg(src), VixlImm(value));
205 GetMasm()->Rscs(VixlReg(tmp_reg), VixlRegU(src), vixl::aarch32::Operand(0x0));
206 } else {
207 GetMasm()->Cmp(VixlReg(src), VixlImm(value));
208 GetMasm()->Sbcs(VixlReg(tmp_reg), VixlRegU(src), vixl::aarch32::Operand(0x0));
209 }
210 }
211 }
212 return true;
213 }
214
EncodeJump(LabelHolder::LabelId id,Reg src,Imm imm,Condition cc)215 void Aarch32Encoder::EncodeJump(LabelHolder::LabelId id, Reg src, Imm imm, Condition cc)
216 {
217 auto value = GetIntValue(imm);
218 if (value == 0) {
219 EncodeJump(id, src, cc);
220 return;
221 }
222
223 if (!CompareImmHelper(src, imm, &cc)) {
224 return;
225 }
226
227 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
228 GetMasm()->B(Convert(cc), label);
229 }
230
EncodeJumpTest(LabelHolder::LabelId id,Reg src,Imm imm,Condition cc)231 void Aarch32Encoder::EncodeJumpTest(LabelHolder::LabelId id, Reg src, Imm imm, Condition cc)
232 {
233 ASSERT(src.IsScalar());
234
235 TestImmHelper(src, imm, cc);
236 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
237 GetMasm()->B(ConvertTest(cc), label);
238 }
239
CompareZeroHelper(Reg src,Condition * cc)240 void Aarch32Encoder::CompareZeroHelper(Reg src, Condition *cc)
241 {
242 ASSERT(src.IsScalar());
243 if (src.GetSize() <= WORD_SIZE) {
244 GetMasm()->Cmp(VixlReg(src), vixl::aarch32::Operand(0x0));
245 } else {
246 ScopedTmpRegU32 tmp_reg(this);
247 uint32_t imm = 0x0;
248
249 switch (*cc) {
250 case Condition::EQ:
251 case Condition::NE:
252 GetMasm()->Orrs(VixlReg(tmp_reg), VixlReg(src), VixlRegU(src));
253 break;
254 case Condition::LE:
255 imm = 0x1;
256 *cc = Condition::LT;
257 /* fallthrough */
258 [[fallthrough]];
259 case Condition::LT:
260 GetMasm()->Cmp(VixlReg(src), vixl::aarch32::Operand(imm));
261 GetMasm()->Sbcs(VixlReg(tmp_reg), VixlRegU(src), vixl::aarch32::Operand(0x0));
262 break;
263 case Condition::GT:
264 imm = 0x1;
265 *cc = Condition::GE;
266 /* fallthrough */
267 [[fallthrough]];
268 case Condition::GE:
269 GetMasm()->Cmp(VixlReg(src), vixl::aarch32::Operand(imm));
270 GetMasm()->Sbcs(VixlReg(tmp_reg), VixlRegU(src), vixl::aarch32::Operand(0x0));
271 break;
272 default:
273 UNREACHABLE();
274 }
275 }
276 }
277
EncodeJump(LabelHolder::LabelId id,Reg src,Condition cc)278 void Aarch32Encoder::EncodeJump(LabelHolder::LabelId id, Reg src, Condition cc)
279 {
280 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
281 ASSERT(src.IsScalar());
282
283 switch (cc) {
284 case Condition::LO:
285 // LO is always false
286 return;
287 case Condition::HS:
288 // HS is always true
289 GetMasm()->B(label);
290 return;
291 case Condition::LS:
292 // LS is same as EQ
293 cc = Condition::EQ;
294 break;
295 case Condition::HI:
296 // HI is same as NE
297 cc = Condition::NE;
298 break;
299 default:
300 break;
301 }
302
303 CompareZeroHelper(src, &cc);
304
305 GetMasm()->B(Convert(cc), label);
306 }
307
EncodeJump(Reg dst)308 void Aarch32Encoder::EncodeJump(Reg dst)
309 {
310 GetMasm()->Bx(VixlReg(dst));
311 }
312
EncodeJump(RelocationInfo * relocation)313 void Aarch32Encoder::EncodeJump([[maybe_unused]] RelocationInfo *relocation)
314 {
315 #ifdef PANDA_TARGET_MACOS
316 LOG(FATAL, COMPILER) << "Not supported in Macos build";
317 #else
318 auto buffer = GetMasm()->GetBuffer();
319 relocation->offset = GetCursorOffset();
320 relocation->addend = 0;
321 relocation->type = R_ARM_CALL;
322 static constexpr uint32_t CALL_WITH_ZERO_OFFSET = 0xeafffffe;
323 buffer->Emit32(CALL_WITH_ZERO_OFFSET);
324 #endif
325 }
326
EncodeNop()327 void Aarch32Encoder::EncodeNop()
328 {
329 GetMasm()->Nop();
330 }
331
MakeCall(compiler::RelocationInfo * relocation)332 void Aarch32Encoder::MakeCall([[maybe_unused]] compiler::RelocationInfo *relocation)
333 {
334 #ifdef PANDA_TARGET_MACOS
335 LOG(FATAL, COMPILER) << "Not supported in Macos build";
336 #else
337 auto buffer = GetMasm()->GetBuffer();
338 relocation->offset = GetCursorOffset();
339 relocation->addend = 0;
340 relocation->type = R_ARM_CALL;
341 static constexpr uint32_t CALL_WITH_ZERO_OFFSET = 0xebfffffe;
342 buffer->Emit32(CALL_WITH_ZERO_OFFSET);
343 #endif
344 }
345
MakeCall(const void * entry_point)346 void Aarch32Encoder::MakeCall(const void *entry_point)
347 {
348 ScopedTmpRegU32 tmp_reg(this);
349
350 auto entry = static_cast<int32_t>(reinterpret_cast<int64_t>(entry_point));
351 EncodeMov(tmp_reg, Imm(entry));
352 GetMasm()->Blx(VixlReg(tmp_reg));
353 }
354
MakeCall(MemRef entry_point)355 void Aarch32Encoder::MakeCall(MemRef entry_point)
356 {
357 ScopedTmpRegU32 tmp_reg(this);
358
359 EncodeLdr(tmp_reg, false, entry_point);
360 GetMasm()->Blx(VixlReg(tmp_reg));
361 }
362
MakeCall(Reg reg)363 void Aarch32Encoder::MakeCall(Reg reg)
364 {
365 GetMasm()->Blx(VixlReg(reg));
366 }
367
MakeCallAot(intptr_t offset)368 void Aarch32Encoder::MakeCallAot([[maybe_unused]] intptr_t offset)
369 {
370 // Unimplemented
371 SetFalseResult();
372 }
373
MakeCallByOffset(intptr_t offset)374 void Aarch32Encoder::MakeCallByOffset([[maybe_unused]] intptr_t offset)
375 {
376 // Unimplemented
377 SetFalseResult();
378 }
379
MakeLoadAotTable(intptr_t offset,Reg reg)380 void Aarch32Encoder::MakeLoadAotTable([[maybe_unused]] intptr_t offset, [[maybe_unused]] Reg reg)
381 {
382 // Unimplemented
383 SetFalseResult();
384 }
385
MakeLoadAotTableAddr(intptr_t offset,Reg addr,Reg val)386 void Aarch32Encoder::MakeLoadAotTableAddr([[maybe_unused]] intptr_t offset, [[maybe_unused]] Reg addr,
387 [[maybe_unused]] Reg val)
388 {
389 // Unimplemented
390 SetFalseResult();
391 }
392
EncodeReturn()393 void Aarch32Encoder::EncodeReturn()
394 {
395 GetMasm()->Bx(vixl::aarch32::lr);
396 }
397
EncodeAbort()398 void Aarch32Encoder::EncodeAbort()
399 {
400 GetMasm()->Udf(0);
401 }
402
EncodeMul(Reg dst,Reg src,Imm imm)403 void Aarch32Encoder::EncodeMul([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src, [[maybe_unused]] Imm imm)
404 {
405 SetFalseResult();
406 }
407
408 /* static */
IsNeedToPrepareMemLdS(MemRef mem,const TypeInfo & mem_type,bool is_signed)409 bool Aarch32Encoder::IsNeedToPrepareMemLdS(MemRef mem, const TypeInfo &mem_type, bool is_signed)
410 {
411 bool has_index = mem.HasIndex();
412 bool has_shift = mem.HasScale();
413 bool has_offset = mem.HasDisp();
414 ASSERT(mem.HasBase());
415 // VLDR and VSTR has base and offset. The offset must be a multiple of 4, and lie in the range -1020 to +1020.
416 if (mem_type.IsFloat()) {
417 if (has_index) {
418 return true;
419 }
420 if (has_offset) {
421 auto offset = mem.GetDisp();
422 constexpr int32_t IMM_4 = 4;
423 if (!((offset >= -VMEM_OFFSET) && (offset <= VMEM_OFFSET) && ((offset % IMM_4) == 0))) {
424 return true;
425 }
426 }
427 return false;
428 }
429 int32_t max_offset = MEM_SMALL_OFFSET;
430 bool has_lsl = false;
431 if (!is_signed && (mem_type == INT32_TYPE || mem_type == INT8_TYPE)) {
432 has_lsl = true;
433 max_offset = MEM_BIG_OFFSET;
434 }
435 // LDR/STR with register offset(for A32)
436 // | mem type | shift |
437 // | ---------- | ----------- |
438 // |word or byte| 0 to 31 |
439 // | others | -- |
440 //
441 if (has_index) {
442 // MemRef with index and offser isn't supported
443 ASSERT(!has_offset);
444 if (has_shift) {
445 ASSERT(mem.GetScale() != 0);
446 [[maybe_unused]] constexpr int32_t MAX_SHIFT = 3;
447 ASSERT(mem.GetScale() <= MAX_SHIFT);
448 if (!has_lsl) {
449 return true;
450 }
451 }
452 return false;
453 }
454 // LDR/STR with immediate offset(for A32)
455 // | mem type | offset size |
456 // | ---------- | ----------- |
457 // |word or byte|-4095 to 4095|
458 // | others | -255 to 255 |
459 //
460 if (has_offset) {
461 ASSERT(mem.GetDisp() != 0);
462 [[maybe_unused]] auto offset = mem.GetDisp();
463 if (!((offset >= -max_offset) && (offset <= max_offset))) {
464 return true;
465 }
466 }
467 return false;
468 }
469
470 /**
471 * The function construct additional instruction for encode memory instructions and returns MemOperand for ldr/str
472 * LDR/STR with immediate offset(for A32)
473 * | mem type | offset size |
474 * | ---------- | ----------- |
475 * |word or byte|-4095 to 4095|
476 * | others | -255 to 255 |
477 *
478 * LDR/STR with register offset(for A32)
479 * | mem type | shift |
480 * | ---------- | ----------- |
481 * |word or byte| 0 to 31 |
482 * | others | -- |
483 *
484 * VLDR and VSTR has base and offset. The offset must be a multiple of 4, and lie in the range -1020 to +1020.
485 */
PrepareMemLdS(MemRef mem,const TypeInfo & mem_type,vixl::aarch32::Register tmp,bool is_signed,bool copy_sp)486 vixl::aarch32::MemOperand Aarch32Encoder::PrepareMemLdS(MemRef mem, const TypeInfo &mem_type,
487 vixl::aarch32::Register tmp, bool is_signed, bool copy_sp)
488 {
489 bool has_index = mem.HasIndex();
490 bool has_shift = mem.HasScale();
491 bool has_offset = mem.HasDisp();
492 ASSERT(mem.HasBase());
493 auto base_reg = VixlReg(mem.GetBase());
494 if (copy_sp) {
495 if (base_reg.IsSP()) {
496 GetMasm()->Mov(tmp, base_reg);
497 base_reg = tmp;
498 }
499 }
500 // VLDR and VSTR has base and offset. The offset must be a multiple of 4, and lie in the range -1020 to +1020.
501 if (mem_type.IsFloat()) {
502 return PrepareMemLdSForFloat(mem, tmp);
503 }
504 int32_t max_offset = MEM_SMALL_OFFSET;
505 bool has_lsl = false;
506 if (!is_signed && (mem_type == INT32_TYPE || mem_type == INT8_TYPE)) {
507 has_lsl = true;
508 max_offset = MEM_BIG_OFFSET;
509 }
510 // LDR/STR with register offset(for A32)
511 // | mem type | shift |
512 // | ---------- | ----------- |
513 // |word or byte| 0 to 31 |
514 // | others | -- |
515 //
516 if (has_index) {
517 // MemRef with index and offser isn't supported
518 ASSERT(!has_offset);
519 auto index_reg = mem.GetIndex();
520 if (has_shift) {
521 ASSERT(mem.GetScale() != 0);
522 auto shift = mem.GetScale();
523 [[maybe_unused]] constexpr int32_t MAX_SHIFT = 3;
524 ASSERT(mem.GetScale() <= MAX_SHIFT);
525 if (has_lsl) {
526 return vixl::aarch32::MemOperand(base_reg, VixlReg(index_reg), vixl::aarch32::LSL, shift);
527 }
528 // from:
529 // mem: base, index, scale
530 // to:
531 // add tmp, base, index, scale
532 // mem tmp
533 GetMasm()->Add(tmp, base_reg, vixl::aarch32::Operand(VixlReg(index_reg), vixl::aarch32::LSL, shift));
534 return vixl::aarch32::MemOperand(tmp);
535 }
536 return vixl::aarch32::MemOperand(base_reg, VixlReg(index_reg));
537 }
538 // LDR/STR with immediate offset(for A32):
539 // | mem type | offset size |
540 // | ---------- | ----------- |
541 // |word or byte|-4095 to 4095|
542 // | others | -255 to 255 |
543 //
544 if (has_offset) {
545 ASSERT(mem.GetDisp() != 0);
546 auto offset = mem.GetDisp();
547 if ((offset >= -max_offset) && (offset <= max_offset)) {
548 return vixl::aarch32::MemOperand(base_reg, offset);
549 }
550 // from:
551 // mem: base, offset
552 // to:
553 // add tmp, base, offset
554 // mem tmp
555 GetMasm()->Add(tmp, base_reg, VixlImm(offset));
556 base_reg = tmp;
557 }
558 return vixl::aarch32::MemOperand(base_reg);
559 }
560
PrepareMemLdSForFloat(MemRef mem,vixl::aarch32::Register tmp)561 vixl::aarch32::MemOperand Aarch32Encoder::PrepareMemLdSForFloat(MemRef mem, vixl::aarch32::Register tmp)
562 {
563 bool has_index = mem.HasIndex();
564 bool has_shift = mem.HasScale();
565 bool has_offset = mem.HasDisp();
566 auto base_reg = VixlReg(mem.GetBase());
567 if (has_index) {
568 auto index_reg = mem.GetIndex();
569 auto scale = mem.GetScale();
570 // from:
571 // vmem: base, index, scale, offset
572 // to:
573 // add tmp, base, index, scale
574 // vmem tmp, offset
575 if (has_shift) {
576 ASSERT(scale != 0);
577 [[maybe_unused]] constexpr int32_t MAX_SHIFT = 3;
578 ASSERT(scale > 0 && scale <= MAX_SHIFT);
579 GetMasm()->Add(tmp, base_reg, vixl::aarch32::Operand(VixlReg(index_reg), vixl::aarch32::LSL, scale));
580 } else {
581 ASSERT(scale == 0);
582 GetMasm()->Add(tmp, base_reg, VixlReg(index_reg));
583 }
584 base_reg = tmp;
585 }
586 if (has_offset) {
587 ASSERT(mem.GetDisp() != 0);
588 auto offset = mem.GetDisp();
589 constexpr int32_t IMM_4 = 4;
590 if ((offset >= -VMEM_OFFSET) && (offset <= VMEM_OFFSET) && ((offset % IMM_4) == 0)) {
591 return vixl::aarch32::MemOperand(base_reg, offset);
592 }
593 // from:
594 // vmem: base, offset
595 // to:
596 // add tmp, base, offset
597 // vmem tmp
598 GetMasm()->Add(tmp, base_reg, VixlImm(offset));
599 base_reg = tmp;
600 }
601 return vixl::aarch32::MemOperand(base_reg);
602 }
603
EncodeFpToBits(Reg dst,Reg src)604 void Aarch32Encoder::EncodeFpToBits(Reg dst, Reg src)
605 {
606 ASSERT(dst.IsScalar() && src.IsFloat());
607 if (dst.GetSize() == WORD_SIZE) {
608 ASSERT(src.GetSize() == WORD_SIZE);
609
610 constexpr uint32_t NANF = 0x7fc00000U;
611
612 GetMasm()->Vcmp(VixlVReg(src), VixlVReg(src));
613 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
614 GetMasm()->Vmov(VixlReg(dst), VixlVReg(src).S());
615 GetMasm()->Mov(Convert(Condition::NE), VixlReg(dst), VixlImm(NANF));
616 } else {
617 ASSERT(src.GetSize() == DOUBLE_WORD_SIZE);
618
619 constexpr uint32_t NAND_HIGH = 0x7ff80000U;
620
621 GetMasm()->Vcmp(VixlVReg(src), VixlVReg(src));
622 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
623 GetMasm()->Vmov(VixlReg(dst), VixlRegU(dst), VixlVReg(src).D());
624 GetMasm()->Mov(Convert(Condition::NE), VixlReg(dst), VixlImm(0));
625 GetMasm()->Mov(Convert(Condition::NE), VixlRegU(dst), VixlImm(NAND_HIGH));
626 }
627 }
628
EncodeMoveBitsRaw(Reg dst,Reg src)629 void Aarch32Encoder::EncodeMoveBitsRaw(Reg dst, Reg src)
630 {
631 ASSERT((dst.IsFloat() && src.IsScalar()) || (src.IsFloat() && dst.IsScalar()));
632 if (dst.IsScalar()) {
633 ASSERT(src.GetSize() == dst.GetSize());
634 if (dst.GetSize() == WORD_SIZE) {
635 GetMasm()->Vmov(VixlReg(dst), VixlVReg(src).S());
636 } else {
637 GetMasm()->Vmov(VixlReg(dst), VixlRegU(dst), VixlVReg(src).D());
638 }
639 } else {
640 ASSERT(dst.GetSize() == src.GetSize());
641 if (src.GetSize() == WORD_SIZE) {
642 GetMasm()->Vmov(VixlVReg(dst).S(), VixlReg(src));
643 } else {
644 GetMasm()->Vmov(VixlVReg(dst).D(), VixlReg(src), VixlRegU(src));
645 }
646 }
647 }
EncodeMov(Reg dst,Reg src)648 void Aarch32Encoder::EncodeMov(Reg dst, Reg src)
649 {
650 if (src.GetSize() <= WORD_SIZE && dst.GetSize() == DOUBLE_WORD_SIZE && !src.IsFloat() && !dst.IsFloat()) {
651 SetFalseResult();
652 return;
653 }
654 if (dst == src) {
655 return;
656 }
657 if (dst.IsFloat()) {
658 if (src.GetType().IsScalar()) {
659 if (src.GetSize() == DOUBLE_WORD_SIZE) {
660 GetMasm()->Vmov(VixlVReg(dst).D(), VixlReg(src), VixlRegU(src));
661 return;
662 }
663 GetMasm()->Vmov(VixlVReg(dst).S(), VixlReg(src));
664 return;
665 }
666 if (dst.GetSize() == WORD_SIZE) {
667 GetMasm()->Vmov(VixlVReg(dst).S(), VixlVReg(src));
668 } else {
669 GetMasm()->Vmov(VixlVReg(dst).D(), VixlVReg(src));
670 }
671 return;
672 }
673 ASSERT(dst.IsScalar());
674 if (src.IsFloat()) {
675 if (src.GetSize() == DOUBLE_WORD_SIZE) {
676 GetMasm()->Vmov(VixlReg(dst), VixlRegU(dst), VixlVReg(src).D());
677 return;
678 }
679 GetMasm()->Vmov(VixlReg(dst), VixlVReg(src).S());
680 return;
681 }
682 ASSERT(src.IsScalar());
683 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
684 if (dst.GetSize() > WORD_SIZE) {
685 GetMasm()->Mov(VixlRegU(dst), VixlRegU(src));
686 }
687 }
688
EncodeNeg(Reg dst,Reg src)689 void Aarch32Encoder::EncodeNeg(Reg dst, Reg src)
690 {
691 if (dst.IsFloat()) {
692 GetMasm()->Vneg(VixlVReg(dst), VixlVReg(src));
693 return;
694 }
695
696 if (dst.GetSize() <= WORD_SIZE) {
697 GetMasm()->Rsb(VixlReg(dst), VixlReg(src), VixlImm(0x0));
698 return;
699 }
700
701 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
702 GetMasm()->Rsbs(VixlReg(dst), VixlReg(src), VixlImm(0x0));
703 GetMasm()->Rsc(VixlRegU(dst), VixlRegU(src), VixlImm(0x0));
704 }
705
EncodeAbs(Reg dst,Reg src)706 void Aarch32Encoder::EncodeAbs(Reg dst, Reg src)
707 {
708 if (dst.IsFloat()) {
709 GetMasm()->Vabs(VixlVReg(dst), VixlVReg(src));
710 return;
711 }
712
713 if (dst.GetSize() <= WORD_SIZE) {
714 GetMasm()->Cmp(VixlReg(src), VixlImm(0x0));
715 GetMasm()->Rsb(Convert(Condition::MI), VixlReg(dst), VixlReg(src), VixlImm(0x0));
716 GetMasm()->Mov(Convert(Condition::PL), VixlReg(dst), VixlReg(src));
717 return;
718 }
719
720 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
721 ScopedTmpRegU32 lo_reg(this);
722 ScopedTmpRegU32 hi_reg(this);
723
724 GetMasm()->Rsbs(VixlReg(lo_reg), VixlReg(src), VixlImm(0x0));
725 GetMasm()->Rsc(VixlReg(hi_reg), VixlRegU(src), VixlImm(0x0));
726 GetMasm()->Cmp(VixlRegU(src), VixlImm(0x0));
727 GetMasm()->Mov(Convert(Condition::PL), VixlReg(lo_reg), VixlReg(src));
728 GetMasm()->Mov(Convert(Condition::PL), VixlReg(hi_reg), VixlRegU(src));
729
730 GetMasm()->Mov(VixlReg(dst), VixlReg(lo_reg));
731 GetMasm()->Mov(VixlRegU(dst), VixlReg(hi_reg));
732 }
733
EncodeSqrt(Reg dst,Reg src)734 void Aarch32Encoder::EncodeSqrt(Reg dst, Reg src)
735 {
736 ASSERT(dst.IsFloat());
737 GetMasm()->Vsqrt(VixlVReg(dst), VixlVReg(src));
738 }
739
EncodeNot(Reg dst,Reg src)740 void Aarch32Encoder::EncodeNot([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src)
741 {
742 GetMasm()->Mvn(VixlReg(dst), VixlReg(src));
743 if (dst.GetSize() > WORD_SIZE) {
744 GetMasm()->Mvn(VixlRegU(dst), VixlRegU(src));
745 }
746 }
747
EncodeIsInf(Reg dst,Reg src)748 void Aarch32Encoder::EncodeIsInf(Reg dst, Reg src)
749 {
750 ASSERT(dst.IsScalar() && src.IsFloat());
751
752 if (src.GetSize() == WORD_SIZE) {
753 constexpr uint32_t INF_MASK = 0xff000000;
754
755 ScopedTmpRegU32 tmp(this);
756
757 GetMasm()->Vmov(VixlReg(tmp), VixlVReg(src).S());
758 GetMasm()->Lsl(VixlReg(tmp), VixlReg(tmp), 1); /* 0xff000000 if Infinity */
759 GetMasm()->Mov(VixlReg(dst), INF_MASK);
760 GetMasm()->Cmp(VixlReg(dst), VixlReg(tmp));
761 } else {
762 constexpr uint32_t INF_MASK = 0xffe00000;
763
764 ScopedTmpRegU32 tmp(this);
765 ScopedTmpRegU32 tmp1(this);
766
767 GetMasm()->Vmov(VixlReg(tmp), VixlReg(tmp1), VixlVReg(src).D());
768 GetMasm()->Lsl(VixlReg(tmp1), VixlReg(tmp1), 1); /* 0xffe00000 if Infinity */
769 GetMasm()->Mov(VixlReg(dst), INF_MASK);
770 GetMasm()->Cmp(VixlReg(dst), VixlReg(tmp1));
771 }
772
773 GetMasm()->Mov(VixlReg(dst), VixlImm(0));
774 GetMasm()->Mov(Convert(Condition::EQ), VixlReg(dst), VixlImm(1));
775 }
776
EncodeReverseBytes(Reg dst,Reg src)777 void Aarch32Encoder::EncodeReverseBytes(Reg dst, Reg src)
778 {
779 ASSERT(src.GetSize() > BYTE_SIZE);
780 ASSERT(src.GetSize() == dst.GetSize());
781
782 if (src.GetSize() == HALF_SIZE) {
783 GetMasm()->Rev16(VixlReg(dst), VixlReg(src));
784 GetMasm()->Sxth(VixlReg(dst), VixlReg(dst));
785 } else if (src.GetSize() == WORD_SIZE) {
786 GetMasm()->Rev(VixlReg(dst), VixlReg(src));
787 } else {
788 if (src == dst) {
789 ScopedTmpRegU32 tmp_reg(this);
790 GetMasm()->Mov(VixlReg(tmp_reg), VixlReg(src));
791 GetMasm()->Rev(VixlReg(dst), VixlRegU(src));
792 GetMasm()->Rev(VixlRegU(dst), VixlReg(tmp_reg));
793 } else {
794 GetMasm()->Rev(VixlRegU(dst), VixlReg(src));
795 GetMasm()->Rev(VixlReg(dst), VixlRegU(src));
796 }
797 }
798 }
799
EncodeBitCount(Reg dst,Reg src)800 void Aarch32Encoder::EncodeBitCount([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src)
801 {
802 SetFalseResult();
803 }
804
EncodeCountLeadingZeroBits(Reg dst,Reg src)805 void Aarch32Encoder::EncodeCountLeadingZeroBits(Reg dst, Reg src)
806 {
807 ASSERT(src.GetSize() == WORD_SIZE || src.GetSize() == DOUBLE_WORD_SIZE);
808 if (src.GetSize() == WORD_SIZE) {
809 GetMasm()->Clz(VixlReg(dst), VixlReg(src));
810 return;
811 }
812
813 auto low = CreateLabel();
814 auto end = CreateLabel();
815 auto high_bits = Reg(src.GetId() + 1, INT32_TYPE);
816 EncodeJump(low, high_bits, Condition::EQ);
817 GetMasm()->Clz(VixlReg(dst), VixlReg(high_bits));
818 EncodeJump(end);
819
820 BindLabel(low);
821 GetMasm()->Clz(VixlReg(dst), VixlReg(src));
822 GetMasm()->Adds(VixlReg(dst), VixlReg(dst), VixlImm(WORD_SIZE));
823
824 BindLabel(end);
825 }
826
EncodeCeil(Reg dst,Reg src)827 void Aarch32Encoder::EncodeCeil(Reg dst, Reg src)
828 {
829 GetMasm()->Vrintp(VixlVReg(dst), VixlVReg(src));
830 }
831
EncodeFloor(Reg dst,Reg src)832 void Aarch32Encoder::EncodeFloor(Reg dst, Reg src)
833 {
834 GetMasm()->Vrintm(VixlVReg(dst), VixlVReg(src));
835 }
836
EncodeRint(Reg dst,Reg src)837 void Aarch32Encoder::EncodeRint(Reg dst, Reg src)
838 {
839 GetMasm()->Vrintn(VixlVReg(dst), VixlVReg(src));
840 }
841
EncodeRound(Reg dst,Reg src)842 void Aarch32Encoder::EncodeRound(Reg dst, Reg src)
843 {
844 ScopedTmpRegF64 temp(this);
845 vixl::aarch32::SRegister temp1(temp.GetReg().GetId());
846 vixl::aarch32::SRegister temp2(temp.GetReg().GetId() + 1);
847
848 auto done = CreateLabel();
849 // round to nearest integer, ties away from zero
850 GetMasm()->Vcvta(vixl::aarch32::DataTypeValue::S32, vixl::aarch32::DataTypeValue::F32, temp1, VixlVReg(src).S());
851 GetMasm()->Vmov(VixlReg(dst), temp1);
852 // for positive, zero and NaN inputs, rounding is done
853 GetMasm()->Cmp(VixlReg(dst), 0);
854 GetMasm()->B(vixl::aarch32::ge, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(done));
855 // if input is negative but not a tie, round to nearest is valid.
856 // if input is a negative tie, change rounding direction to positive infinity, dst += 1.
857 GetMasm()->Vrinta(vixl::aarch32::DataTypeValue::F32, temp1, VixlVReg(src).S());
858 // NOLINTNEXTLINE(readability-magic-numbers)
859 const auto HALF = 0.5;
860 GetMasm()->Vmov(temp2, HALF);
861 GetMasm()->Vsub(vixl::aarch32::DataTypeValue::F32, temp1, VixlVReg(src).S(), temp1);
862 GetMasm()->Vcmp(vixl::aarch32::DataTypeValue::F32, temp1, temp2);
863 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
864 GetMasm()->add(vixl::aarch32::eq, VixlReg(dst), VixlReg(dst), 1);
865
866 BindLabel(done);
867 }
868
EncodeReverseBits(Reg dst,Reg src)869 void Aarch32Encoder::EncodeReverseBits(Reg dst, Reg src)
870 {
871 ASSERT(src.GetSize() == WORD_SIZE || src.GetSize() == DOUBLE_WORD_SIZE);
872 ASSERT(src.GetSize() == dst.GetSize());
873
874 if (src.GetSize() == WORD_SIZE) {
875 GetMasm()->Rbit(VixlReg(dst), VixlReg(src));
876 return;
877 }
878
879 if (src == dst) {
880 ScopedTmpRegU32 tmp_reg(this);
881 GetMasm()->Mov(VixlReg(tmp_reg), VixlReg(src));
882 GetMasm()->Rbit(VixlReg(dst), VixlRegU(src));
883 GetMasm()->Rbit(VixlRegU(dst), VixlReg(tmp_reg));
884 return;
885 }
886
887 GetMasm()->Rbit(VixlRegU(dst), VixlReg(src));
888 GetMasm()->Rbit(VixlReg(dst), VixlRegU(src));
889 }
890
EncodeCastToBool(Reg dst,Reg src)891 void Aarch32Encoder::EncodeCastToBool(Reg dst, Reg src)
892 {
893 // In ISA says that we only support casts:
894 // i32tou1, i64tou1, u32tou1, u64tou1
895 ASSERT(src.IsScalar());
896 ASSERT(dst.IsScalar());
897
898 GetMasm()->Cmp(VixlReg(src), VixlImm(0x0));
899 GetMasm()->Mov(Convert(Condition::EQ), VixlReg(dst), VixlImm(0x0));
900 GetMasm()->Mov(Convert(Condition::NE), VixlReg(dst), VixlImm(0x1));
901 if (src.GetSize() == DOUBLE_WORD_SIZE) {
902 GetMasm()->Cmp(VixlRegU(src), VixlImm(0x0));
903 GetMasm()->Mov(Convert(Condition::NE), VixlReg(dst), VixlImm(0x1));
904 }
905 }
906
EncodeCast(Reg dst,bool dst_signed,Reg src,bool src_signed)907 void Aarch32Encoder::EncodeCast(Reg dst, bool dst_signed, Reg src, bool src_signed)
908 {
909 // float/double -> float/double
910 if (dst.IsFloat() && src.IsFloat()) {
911 EncodeCastFloatToFloat(dst, src);
912 return;
913 }
914
915 // uint/int -> float/double
916 if (dst.IsFloat() && src.IsScalar()) {
917 EncodeCastScalarToFloat(dst, src, src_signed);
918 return;
919 }
920
921 // float/double -> uint/int
922 if (dst.IsScalar() && src.IsFloat()) {
923 EncodeCastFloatToScalar(dst, dst_signed, src);
924 return;
925 }
926
927 // uint/int -> uint/int
928 ASSERT(dst.IsScalar() && src.IsScalar());
929 EncodeCastScalar(dst, dst_signed, src, src_signed);
930 }
931
EncodeCastScalar(Reg dst,bool dst_signed,Reg src,bool src_signed)932 void Aarch32Encoder::EncodeCastScalar(Reg dst, bool dst_signed, Reg src, bool src_signed)
933 {
934 size_t src_size = src.GetSize();
935 size_t dst_size = dst.GetSize();
936 // In our ISA minimal type is 32-bit, so type less then 32-bit
937 // we should extend to 32-bit. So we can have 2 cast
938 // (For examble, i8->u16 will work as i8->u16 and u16->u32)
939 if (dst_size < WORD_SIZE) {
940 if (src_size > dst_size) {
941 if (dst_signed) {
942 EncodeCastScalarFromSignedScalar(dst, src);
943 } else {
944 EncodeCastScalarFromUnsignedScalar(dst, src);
945 }
946 return;
947 }
948 if (src_size == dst_size) {
949 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
950 if (src_signed == dst_signed) {
951 return;
952 }
953 if (dst_signed) {
954 EncodeCastScalarFromSignedScalar(Reg(dst.GetId(), INT32_TYPE), dst);
955 } else {
956 EncodeCastScalarFromUnsignedScalar(Reg(dst.GetId(), INT32_TYPE), dst);
957 }
958 return;
959 }
960 if (src_signed) {
961 EncodeCastScalarFromSignedScalar(dst, src);
962 if (!dst_signed) {
963 EncodeCastScalarFromUnsignedScalar(Reg(dst.GetId(), INT32_TYPE), dst);
964 }
965 } else {
966 EncodeCastScalarFromUnsignedScalar(dst, src);
967 if (dst_signed) {
968 EncodeCastScalarFromSignedScalar(Reg(dst.GetId(), INT32_TYPE), dst);
969 }
970 }
971 } else {
972 if (src_size == dst_size) {
973 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
974 if (src_size == DOUBLE_WORD_SIZE) {
975 GetMasm()->Mov(VixlRegU(dst), VixlRegU(src));
976 }
977 return;
978 }
979
980 if (src_signed) {
981 EncodeCastScalarFromSignedScalar(dst, src);
982 } else {
983 EncodeCastScalarFromUnsignedScalar(dst, src);
984 }
985 }
986 }
987
EncodeCastFloatToFloat(Reg dst,Reg src)988 void Aarch32Encoder::EncodeCastFloatToFloat(Reg dst, Reg src)
989 {
990 // float/double -> float/double
991 if (dst.GetSize() == src.GetSize()) {
992 if (dst.GetSize() == WORD_SIZE) {
993 GetMasm()->Vmov(VixlVReg(dst).S(), VixlVReg(src));
994 } else {
995 GetMasm()->Vmov(VixlVReg(dst).D(), VixlVReg(src));
996 }
997 return;
998 }
999
1000 // double -> float
1001 if (dst.GetSize() == WORD_SIZE) {
1002 GetMasm()->Vcvt(vixl::aarch32::DataTypeValue::F32, vixl::aarch32::DataTypeValue::F64, VixlVReg(dst).S(),
1003 VixlVReg(src).D());
1004 } else {
1005 // float -> double
1006 GetMasm()->Vcvt(vixl::aarch32::DataTypeValue::F64, vixl::aarch32::DataTypeValue::F32, VixlVReg(dst).D(),
1007 VixlVReg(src).S());
1008 }
1009 }
1010
EncodeCastScalarToFloat(Reg dst,Reg src,bool src_signed)1011 void Aarch32Encoder::EncodeCastScalarToFloat(Reg dst, Reg src, bool src_signed)
1012 {
1013 // uint/int -> float/double
1014 switch (src.GetSize()) {
1015 case BYTE_SIZE:
1016 case HALF_SIZE:
1017 case WORD_SIZE: {
1018 ScopedTmpRegF32 tmp_reg(this);
1019
1020 GetMasm()->Vmov(VixlVReg(tmp_reg).S(), VixlReg(src));
1021 auto data_type = src_signed ? vixl::aarch32::DataTypeValue::S32 : vixl::aarch32::DataTypeValue::U32;
1022 if (dst.GetSize() == WORD_SIZE) {
1023 GetMasm()->Vcvt(vixl::aarch32::DataTypeValue::F32, data_type, VixlVReg(dst).S(), VixlVReg(tmp_reg).S());
1024 } else {
1025 GetMasm()->Vcvt(vixl::aarch32::DataTypeValue::F64, data_type, VixlVReg(dst).D(), VixlVReg(tmp_reg).S());
1026 }
1027 break;
1028 }
1029 case DOUBLE_WORD_SIZE: {
1030 if (dst.GetSize() == WORD_SIZE) {
1031 if (src_signed) {
1032 // int64 -> float
1033 MakeLibCall(dst, src, reinterpret_cast<void *>(__aeabi_l2f));
1034 } else {
1035 // uint64 -> float
1036 MakeLibCall(dst, src, reinterpret_cast<void *>(__aeabi_ul2f));
1037 }
1038 } else {
1039 if (src_signed) {
1040 // int64 -> double
1041 MakeLibCall(dst, src, reinterpret_cast<void *>(__aeabi_l2d));
1042 } else {
1043 // uint64 -> double
1044 MakeLibCall(dst, src, reinterpret_cast<void *>(__aeabi_ul2d));
1045 }
1046 }
1047 break;
1048 }
1049 default:
1050 SetFalseResult();
1051 break;
1052 }
1053 }
1054
EncodeCastFloatToScalar(Reg dst,bool dst_signed,Reg src)1055 void Aarch32Encoder::EncodeCastFloatToScalar(Reg dst, bool dst_signed, Reg src)
1056 {
1057 // We DON'T support casts from float32/64 to int8/16 and bool, because this caste is not declared anywhere
1058 // in other languages and architecture, we do not know what the behavior should be.
1059 // But there is one implementation in other function: "EncodeCastFloatToScalarWithSmallDst". Call it in the
1060 // "EncodeCast" function instead of "EncodeCastFloat". It works as follows: cast from float32/64 to int32, moving
1061 // sign bit from int32 to dst type, then extend number from dst type to int32 (a necessary condition for an isa).
1062 // All work in dst register.
1063 ASSERT(dst.GetSize() >= WORD_SIZE);
1064
1065 switch (dst.GetSize()) {
1066 case WORD_SIZE: {
1067 ScopedTmpRegF32 tmp_reg(this);
1068
1069 auto data_type = dst_signed ? vixl::aarch32::DataTypeValue::S32 : vixl::aarch32::DataTypeValue::U32;
1070 if (src.GetSize() == WORD_SIZE) {
1071 GetMasm()->Vcvt(data_type, vixl::aarch32::DataTypeValue::F32, VixlVReg(tmp_reg).S(), VixlVReg(src).S());
1072 } else {
1073 GetMasm()->Vcvt(data_type, vixl::aarch32::DataTypeValue::F64, VixlVReg(tmp_reg).S(), VixlVReg(src).D());
1074 }
1075
1076 GetMasm()->Vmov(VixlReg(dst), VixlVReg(tmp_reg).S());
1077 break;
1078 }
1079 case DOUBLE_WORD_SIZE: {
1080 if (src.GetSize() == WORD_SIZE) {
1081 if (dst_signed) {
1082 // float -> int64
1083 EncodeCastFloatToInt64(dst, src);
1084 } else {
1085 // float -> uint64
1086 MakeLibCall(dst, src, reinterpret_cast<void *>(__aeabi_f2ulz));
1087 }
1088 } else {
1089 if (dst_signed) {
1090 // double -> int64
1091 EncodeCastDoubleToInt64(dst, src);
1092 } else {
1093 // double -> uint64
1094 MakeLibCall(dst, src, reinterpret_cast<void *>(__aeabi_d2ulz));
1095 }
1096 }
1097 break;
1098 }
1099 default:
1100 SetFalseResult();
1101 break;
1102 }
1103 }
1104
EncodeCastFloatToScalarWithSmallDst(Reg dst,bool dst_signed,Reg src)1105 void Aarch32Encoder::EncodeCastFloatToScalarWithSmallDst(Reg dst, bool dst_signed, Reg src)
1106 {
1107 switch (dst.GetSize()) {
1108 case BYTE_SIZE:
1109 case HALF_SIZE:
1110 case WORD_SIZE: {
1111 ScopedTmpRegF32 tmp_reg(this);
1112
1113 auto data_type = dst_signed ? vixl::aarch32::DataTypeValue::S32 : vixl::aarch32::DataTypeValue::U32;
1114 if (src.GetSize() == WORD_SIZE) {
1115 GetMasm()->Vcvt(data_type, vixl::aarch32::DataTypeValue::F32, VixlVReg(tmp_reg).S(), VixlVReg(src).S());
1116 } else {
1117 GetMasm()->Vcvt(data_type, vixl::aarch32::DataTypeValue::F64, VixlVReg(tmp_reg).S(), VixlVReg(src).D());
1118 }
1119
1120 GetMasm()->Vmov(VixlReg(dst), VixlVReg(tmp_reg).S());
1121 if (dst.GetSize() < WORD_SIZE) {
1122 EncoderCastExtendFromInt32(dst, dst_signed);
1123 }
1124 break;
1125 }
1126 case DOUBLE_WORD_SIZE: {
1127 if (src.GetSize() == WORD_SIZE) {
1128 if (dst_signed) {
1129 // float -> int64
1130 EncodeCastFloatToInt64(dst, src);
1131 } else {
1132 // float -> uint64
1133 MakeLibCall(dst, src, reinterpret_cast<void *>(__aeabi_f2ulz));
1134 }
1135 } else {
1136 if (dst_signed) {
1137 // double -> int64
1138 EncodeCastDoubleToInt64(dst, src);
1139 } else {
1140 // double -> uint64
1141 MakeLibCall(dst, src, reinterpret_cast<void *>(__aeabi_d2ulz));
1142 }
1143 }
1144 break;
1145 }
1146 default:
1147 SetFalseResult();
1148 break;
1149 }
1150 }
1151
EncoderCastExtendFromInt32(Reg dst,bool dst_signed)1152 void Aarch32Encoder::EncoderCastExtendFromInt32(Reg dst, bool dst_signed)
1153 {
1154 if (dst_signed) {
1155 constexpr uint32_t TEST_BIT = (1U << (static_cast<uint32_t>(WORD_SIZE) - 1));
1156 ScopedTmpReg tmp_reg(this, dst.GetType());
1157
1158 uint32_t set_bit = (dst.GetSize() == BYTE_SIZE) ? (1U << static_cast<uint32_t>(BYTE_SIZE - 1))
1159 : (1U << static_cast<uint32_t>(HALF_SIZE - 1));
1160 uint32_t rem_bit = set_bit - 1;
1161 GetMasm()->And(VixlReg(tmp_reg), VixlReg(dst), TEST_BIT);
1162 auto label_skip = CreateLabel();
1163 auto label_end_if = CreateLabel();
1164 EncodeJump(label_skip, tmp_reg, Condition::EQ);
1165 // Set signed bit in dst reg (accordingly destination type)
1166 // If signed bit == 1
1167 GetMasm()->Orr(VixlReg(dst), VixlReg(dst), set_bit);
1168 EncodeJump(label_end_if);
1169 BindLabel(label_skip);
1170 // If signed bit == 0
1171 GetMasm()->And(VixlReg(dst), VixlReg(dst), rem_bit);
1172 BindLabel(label_end_if);
1173 }
1174 EncodeCastScalar(Reg(dst.GetId(), INT32_TYPE), dst_signed, dst, dst_signed);
1175 }
1176
EncodeCastDoubleToInt64(Reg dst,Reg src)1177 void Aarch32Encoder::EncodeCastDoubleToInt64(Reg dst, Reg src)
1178 {
1179 auto label_check_nan = CreateLabel();
1180 auto label_not_nan = CreateLabel();
1181 auto label_exit = CreateLabel();
1182 ScopedTmpReg tmp_reg1(this, INT32_TYPE);
1183
1184 // Mov double to 2x reg to storage double in hex format and work with them
1185 EncodeMov(dst, src);
1186 // See the exponent of number
1187 GetMasm()->Ubfx(VixlReg(tmp_reg1), VixlRegU(dst), START_EXP_DOUBLE, SIZE_EXP_DOUBLE);
1188 // Max exponent that we can load in int64
1189 // Check that x > MIN_INT64 & x < MAX_INT64, else jump
1190 GetMasm()->Cmp(VixlReg(tmp_reg1), VixlImm(POSSIBLE_EXP_DOUBLE));
1191 // If greater than or equal, branch to "label_not_nan"
1192 GetMasm()->B(vixl::aarch32::hs, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(label_check_nan));
1193 MakeLibCall(dst, src, reinterpret_cast<void *>(__aeabi_d2lz));
1194 EncodeJump(label_exit);
1195
1196 BindLabel(label_check_nan);
1197 // Form of nan number
1198 GetMasm()->Cmp(VixlReg(tmp_reg1), VixlImm(UP_BITS_NAN_DOUBLE));
1199 // If not equal, branch to "label_not_nan"
1200 GetMasm()->B(vixl::aarch32::ne, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(label_not_nan));
1201
1202 GetMasm()->Orrs(VixlReg(tmp_reg1), VixlReg(dst),
1203 vixl::aarch32::Operand(VixlRegU(dst), vixl::aarch32::LSL, SHIFT_BITS_DOUBLE));
1204 auto addr_label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(label_not_nan);
1205 GetMasm()->B(vixl::aarch32::eq, addr_label);
1206 GetMasm()->Mov(VixlReg(dst), VixlImm(0));
1207 GetMasm()->Mov(VixlRegU(dst), VixlImm(0));
1208 EncodeJump(label_exit);
1209
1210 BindLabel(label_not_nan);
1211 GetMasm()->Adds(VixlRegU(dst), VixlRegU(dst), VixlRegU(dst));
1212 GetMasm()->Mov(VixlReg(dst), VixlImm(UINT32_MAX));
1213 GetMasm()->Mov(VixlRegU(dst), VixlImm(INT32_MAX));
1214 GetMasm()->Adc(VixlReg(dst), VixlReg(dst), VixlImm(0));
1215 // If exponent negative, transform maxint64 to minint64
1216 GetMasm()->Adc(VixlRegU(dst), VixlRegU(dst), VixlImm(0));
1217
1218 BindLabel(label_exit);
1219 }
1220
EncodeCastFloatToInt64(Reg dst,Reg src)1221 void Aarch32Encoder::EncodeCastFloatToInt64(Reg dst, Reg src)
1222 {
1223 auto label_check_nan = CreateLabel();
1224 auto label_not_nan = CreateLabel();
1225 auto label_exit = CreateLabel();
1226
1227 ScopedTmpReg tmp_reg(this, INT32_TYPE);
1228 ScopedTmpReg moved_src(this, INT32_TYPE);
1229 EncodeMov(moved_src, src);
1230 // See the exponent of number
1231 GetMasm()->Ubfx(VixlReg(tmp_reg), VixlReg(moved_src), START_EXP_FLOAT, SIZE_EXP_FLOAT);
1232 // Max exponent that we can load in int64
1233 // Check that x > MIN_INT64 & x < MAX_INT64, else jump
1234 GetMasm()->Cmp(VixlReg(tmp_reg), VixlImm(POSSIBLE_EXP_FLOAT));
1235 // If greater than or equal, branch to "label_not_nan"
1236 GetMasm()->B(vixl::aarch32::hs, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(label_check_nan));
1237 MakeLibCall(dst, src, reinterpret_cast<void *>(__aeabi_f2lz));
1238 EncodeJump(label_exit);
1239
1240 BindLabel(label_check_nan);
1241 // Form of nan number
1242 GetMasm()->Cmp(VixlReg(tmp_reg), VixlImm(UP_BITS_NAN_FLOAT));
1243 // If not equal, branch to "label_not_nan"
1244 GetMasm()->B(vixl::aarch32::ne, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(label_not_nan));
1245
1246 GetMasm()->Lsls(VixlReg(tmp_reg), VixlReg(moved_src), vixl::aarch32::Operand(SHIFT_BITS_FLOAT));
1247 // If equal, branch to "label_not_nan"
1248 GetMasm()->B(vixl::aarch32::eq, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(label_not_nan));
1249 GetMasm()->Mov(VixlReg(dst), VixlImm(0));
1250 GetMasm()->Mov(VixlRegU(dst), VixlImm(0));
1251 EncodeJump(label_exit);
1252
1253 BindLabel(label_not_nan);
1254 GetMasm()->Adds(VixlReg(moved_src), VixlReg(moved_src), VixlReg(moved_src));
1255 GetMasm()->Mov(VixlReg(dst), VixlImm(UINT32_MAX));
1256 GetMasm()->Mov(VixlRegU(dst), VixlImm(INT32_MAX));
1257 GetMasm()->Adc(VixlReg(dst), VixlReg(dst), VixlImm(0));
1258 // If exponent negative, transform maxint64 to minint64
1259 GetMasm()->Adc(VixlRegU(dst), VixlRegU(dst), VixlImm(0));
1260
1261 BindLabel(label_exit);
1262 }
1263
EncodeCastScalarFromSignedScalar(Reg dst,Reg src)1264 void Aarch32Encoder::EncodeCastScalarFromSignedScalar(Reg dst, Reg src)
1265 {
1266 size_t src_size = src.GetSize();
1267 size_t dst_size = dst.GetSize();
1268 if (src_size > dst_size) {
1269 src_size = dst_size;
1270 }
1271 switch (src_size) {
1272 case BYTE_SIZE:
1273 GetMasm()->Sxtb(VixlReg(dst), VixlReg(src));
1274 break;
1275 case HALF_SIZE:
1276 GetMasm()->Sxth(VixlReg(dst), VixlReg(src));
1277 break;
1278 case WORD_SIZE:
1279 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1280 break;
1281 case DOUBLE_WORD_SIZE:
1282 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1283 if (dst_size == DOUBLE_WORD_SIZE) {
1284 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1285 return;
1286 }
1287 break;
1288 default:
1289 SetFalseResult();
1290 break;
1291 }
1292 if (dst_size == DOUBLE_WORD_SIZE) {
1293 GetMasm()->Asr(VixlRegU(dst), VixlReg(dst), VixlImm(WORD_SIZE - 1));
1294 }
1295 }
1296
EncodeCastScalarFromUnsignedScalar(Reg dst,Reg src)1297 void Aarch32Encoder::EncodeCastScalarFromUnsignedScalar(Reg dst, Reg src)
1298 {
1299 size_t src_size = src.GetSize();
1300 size_t dst_size = dst.GetSize();
1301 if (src_size > dst_size && dst_size < WORD_SIZE) {
1302 // We need to cut the number, if it is less, than 32-bit. It is by ISA agreement.
1303 int64_t cut_value = (1ULL << dst_size) - 1;
1304 GetMasm()->And(VixlReg(dst), VixlReg(src), VixlImm(cut_value));
1305 return;
1306 }
1307 // Else unsigned extend
1308 switch (src_size) {
1309 case BYTE_SIZE:
1310 GetMasm()->Uxtb(VixlReg(dst), VixlReg(src));
1311 break;
1312 case HALF_SIZE:
1313 GetMasm()->Uxth(VixlReg(dst), VixlReg(src));
1314 break;
1315 case WORD_SIZE:
1316 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1317 break;
1318 case DOUBLE_WORD_SIZE:
1319 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1320 if (dst_size == DOUBLE_WORD_SIZE) {
1321 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1322 return;
1323 }
1324 break;
1325 default:
1326 SetFalseResult();
1327 break;
1328 }
1329 if (dst_size == DOUBLE_WORD_SIZE) {
1330 GetMasm()->Mov(VixlRegU(dst), VixlImm(0x0));
1331 }
1332 }
1333
EncodeAdd(Reg dst,Reg src0,Reg src1)1334 void Aarch32Encoder::EncodeAdd(Reg dst, Reg src0, Reg src1)
1335 {
1336 if (dst.IsFloat()) {
1337 ASSERT(src0.IsFloat() && src1.IsFloat());
1338 GetMasm()->Vadd(VixlVReg(dst), VixlVReg(src0), VixlVReg(src1));
1339 return;
1340 }
1341
1342 if (dst.GetSize() <= WORD_SIZE) {
1343 GetMasm()->Add(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1344 return;
1345 }
1346
1347 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1348 GetMasm()->Adds(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1349 GetMasm()->Adc(VixlRegU(dst), VixlRegU(src0), VixlRegU(src1));
1350 }
1351
EncodeSub(Reg dst,Reg src0,Reg src1)1352 void Aarch32Encoder::EncodeSub(Reg dst, Reg src0, Reg src1)
1353 {
1354 if (dst.IsFloat()) {
1355 GetMasm()->Vsub(VixlVReg(dst), VixlVReg(src0), VixlVReg(src1));
1356 return;
1357 }
1358
1359 if (dst.GetSize() <= WORD_SIZE) {
1360 GetMasm()->Sub(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1361 return;
1362 }
1363
1364 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1365 GetMasm()->Subs(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1366 GetMasm()->Sbc(VixlRegU(dst), VixlRegU(src0), VixlRegU(src1));
1367 }
1368
EncodeMul(Reg dst,Reg src0,Reg src1)1369 void Aarch32Encoder::EncodeMul(Reg dst, Reg src0, Reg src1)
1370 {
1371 if (dst.IsFloat()) {
1372 GetMasm()->Vmul(VixlVReg(dst), VixlVReg(src0), VixlVReg(src1));
1373 return;
1374 }
1375
1376 if (dst.GetSize() <= WORD_SIZE) {
1377 GetMasm()->Mul(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1378 return;
1379 }
1380
1381 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1382 ScopedTmpRegU32 lo_reg(this);
1383 ScopedTmpRegU32 hi_reg(this);
1384
1385 GetMasm()->Umull(VixlReg(lo_reg), VixlReg(hi_reg), VixlReg(src0), VixlReg(src1));
1386 GetMasm()->Mla(VixlReg(hi_reg), VixlRegU(src0), VixlReg(src1), VixlReg(hi_reg));
1387 GetMasm()->Mla(VixlReg(hi_reg), VixlReg(src0), VixlRegU(src1), VixlReg(hi_reg));
1388
1389 GetMasm()->Mov(VixlReg(dst), VixlReg(lo_reg));
1390 GetMasm()->Mov(VixlRegU(dst), VixlReg(hi_reg));
1391 }
1392
MakeLibCall(Reg dst,Reg src0,Reg src1,void * entry_point,bool second_value)1393 void Aarch32Encoder::MakeLibCall(Reg dst, Reg src0, Reg src1, void *entry_point, bool second_value)
1394 {
1395 if (dst.GetType() == FLOAT32_TYPE) {
1396 MakeLibCallWithFloatResult(dst, src0, src1, entry_point, second_value);
1397 return;
1398 }
1399
1400 if (dst.GetType() == FLOAT64_TYPE) {
1401 MakeLibCallWithDoubleResult(dst, src0, src1, entry_point, second_value);
1402 return;
1403 }
1404
1405 if (dst.GetType() == INT64_TYPE) {
1406 MakeLibCallWithInt64Result(dst, src0, src1, entry_point, second_value);
1407 return;
1408 }
1409
1410 ASSERT(dst.GetSize() < DOUBLE_WORD_SIZE);
1411
1412 if (src1.GetId() == vixl::aarch32::r0.GetCode() || src0 == src1) {
1413 ScopedTmpRegU32 tmp(this);
1414 GetMasm()->Mov(VixlReg(tmp), VixlReg(src1));
1415 GetMasm()->Mov(vixl::aarch32::r0, VixlReg(src0));
1416 GetMasm()->Mov(vixl::aarch32::r1, VixlReg(tmp));
1417 } else {
1418 GetMasm()->Mov(vixl::aarch32::r0, VixlReg(src0));
1419 GetMasm()->Mov(vixl::aarch32::r1, VixlReg(src1));
1420 };
1421
1422 // Call lib-method
1423 MakeCall(entry_point);
1424
1425 auto dst_register = second_value ? vixl::aarch32::r1 : vixl::aarch32::r0;
1426
1427 if (dst.GetId() <= vixl::aarch32::r3.GetCode()) {
1428 ScopedTmpRegU32 tmp(this);
1429 GetMasm()->Mov(VixlReg(tmp), dst_register);
1430 GetMasm()->Mov(VixlReg(dst), VixlReg(tmp));
1431 } else {
1432 GetMasm()->Mov(VixlReg(dst), dst_register);
1433 }
1434 }
1435
MakeLibCallWithFloatResult(Reg dst,Reg src0,Reg src1,void * entry_point,bool second_value)1436 void Aarch32Encoder::MakeLibCallWithFloatResult(Reg dst, Reg src0, Reg src1, void *entry_point, bool second_value)
1437 {
1438 #if (PANDA_TARGET_ARM32_ABI_HARD)
1439 // gnueabihf
1440 // use double parameters
1441 if (src1.GetId() == vixl::aarch32::s0.GetCode() || src0 == src1) {
1442 ScopedTmpRegF32 tmp(this);
1443 GetMasm()->Vmov(VixlVReg(tmp), VixlVReg(src1));
1444 GetMasm()->Vmov(vixl::aarch32::s0, VixlVReg(src0));
1445 GetMasm()->Vmov(vixl::aarch32::s1, VixlVReg(tmp));
1446 } else {
1447 GetMasm()->Vmov(vixl::aarch32::s0, VixlVReg(src0));
1448 GetMasm()->Vmov(vixl::aarch32::s1, VixlVReg(src1));
1449 };
1450
1451 MakeCall(entry_point);
1452
1453 auto dst_register = second_value ? vixl::aarch32::s1 : vixl::aarch32::s0;
1454 if (dst.GetId() <= vixl::aarch32::s1.GetCode()) {
1455 ScopedTmpRegF32 tmp(this);
1456 GetMasm()->Vmov(VixlVReg(tmp).S(), dst_register);
1457 GetMasm()->Vmov(VixlVReg(dst).S(), VixlVReg(tmp).S());
1458 } else {
1459 GetMasm()->Vmov(VixlVReg(dst).S(), dst_register);
1460 }
1461 #else
1462 // gnueabi
1463 // use scalar parameters
1464 GetMasm()->Vmov(vixl::aarch32::r0, VixlVReg(src0).S());
1465 GetMasm()->Vmov(vixl::aarch32::r1, VixlVReg(src1).S());
1466
1467 MakeCall(entry_point);
1468
1469 auto dst_register = second_value ? vixl::aarch32::r1 : vixl::aarch32::r0;
1470 GetMasm()->Vmov(VixlVReg(dst).S(), dst_register);
1471 #endif // PANDA_TARGET_ARM32_ABI_HARD
1472 }
1473
MakeLibCallWithDoubleResult(Reg dst,Reg src0,Reg src1,void * entry_point,bool second_value)1474 void Aarch32Encoder::MakeLibCallWithDoubleResult(Reg dst, Reg src0, Reg src1, void *entry_point, bool second_value)
1475 {
1476 #if (PANDA_TARGET_ARM32_ABI_HARD)
1477 // Scope for temp
1478 if (src1.GetId() == vixl::aarch32::d0.GetCode() || src0 == src1) {
1479 ScopedTmpRegF64 tmp(this);
1480 GetMasm()->Vmov(VixlVReg(tmp), VixlVReg(src1));
1481 GetMasm()->Vmov(vixl::aarch32::d0, VixlVReg(src0));
1482 GetMasm()->Vmov(vixl::aarch32::d1, VixlVReg(tmp));
1483 } else {
1484 GetMasm()->Vmov(vixl::aarch32::d0, VixlVReg(src0));
1485 GetMasm()->Vmov(vixl::aarch32::d1, VixlVReg(src1));
1486 };
1487 MakeCall(entry_point);
1488 auto dst_register = second_value ? vixl::aarch32::d1 : vixl::aarch32::d0;
1489
1490 if (dst.GetId() <= vixl::aarch32::d1.GetCode()) {
1491 ScopedTmpRegF64 tmp(this);
1492 GetMasm()->Vmov(VixlVReg(tmp), dst_register);
1493 GetMasm()->Vmov(VixlVReg(dst), VixlVReg(tmp));
1494 } else {
1495 GetMasm()->Vmov(VixlVReg(dst), dst_register);
1496 }
1497
1498 // use double parameters
1499 #else
1500 // use scalar parameters
1501 GetMasm()->Vmov(vixl::aarch32::r0, vixl::aarch32::r1, VixlVReg(src0).D());
1502 GetMasm()->Vmov(vixl::aarch32::r2, vixl::aarch32::r3, VixlVReg(src1).D());
1503
1504 MakeCall(entry_point);
1505
1506 auto dst_register_1 = second_value ? vixl::aarch32::r2 : vixl::aarch32::r0;
1507 auto dst_register_2 = second_value ? vixl::aarch32::r3 : vixl::aarch32::r1;
1508
1509 GetMasm()->Vmov(VixlVReg(dst).D(), dst_register_1, dst_register_2);
1510 #endif // PANDA_TARGET_ARM32_ABI_HARD
1511 }
1512
MakeLibCallWithInt64Result(Reg dst,Reg src0,Reg src1,void * entry_point,bool second_value)1513 void Aarch32Encoder::MakeLibCallWithInt64Result(Reg dst, Reg src0, Reg src1, void *entry_point, bool second_value)
1514 {
1515 // Here I look only for this case, because src - is mapped on two regs.
1516 // (INT64_TYPE), and src0 can't be rewrited
1517 // TODO(igorban) If src0==src1 - the result will be 1 or UB(if src0 = 0)
1518 // It is better to check them in optimizations
1519 // ASSERT(src0 != src1); - do not enable for tests
1520
1521 if (src1.GetId() == vixl::aarch32::r0.GetCode() || src0 == src1) {
1522 ScopedTmpRegU32 tmp1(this);
1523 ScopedTmpRegU32 tmp2(this);
1524 GetMasm()->Mov(VixlReg(tmp1), VixlReg(src1));
1525 GetMasm()->Mov(VixlReg(tmp2), VixlRegU(src1));
1526 GetMasm()->Mov(vixl::aarch32::r0, VixlReg(src0));
1527 GetMasm()->Mov(vixl::aarch32::r1, VixlRegU(src0));
1528 GetMasm()->Mov(vixl::aarch32::r2, VixlReg(tmp1));
1529 GetMasm()->Mov(vixl::aarch32::r3, VixlReg(tmp2));
1530 } else {
1531 GetMasm()->Mov(vixl::aarch32::r0, VixlReg(src0));
1532 GetMasm()->Mov(vixl::aarch32::r1, VixlRegU(src0));
1533 GetMasm()->Mov(vixl::aarch32::r2, VixlReg(src1));
1534 GetMasm()->Mov(vixl::aarch32::r3, VixlRegU(src1));
1535 };
1536
1537 // Call lib-method
1538 MakeCall(entry_point);
1539
1540 auto dst_register_1 = second_value ? vixl::aarch32::r2 : vixl::aarch32::r0;
1541 auto dst_register_2 = second_value ? vixl::aarch32::r3 : vixl::aarch32::r1;
1542
1543 if (dst.GetId() <= vixl::aarch32::r3.GetCode()) {
1544 ScopedTmpRegU32 tmp1(this);
1545 ScopedTmpRegU32 tmp2(this);
1546 GetMasm()->Mov(VixlReg(tmp1), dst_register_1);
1547 GetMasm()->Mov(VixlReg(tmp2), dst_register_2);
1548 GetMasm()->Mov(VixlReg(dst), VixlReg(tmp1));
1549 GetMasm()->Mov(VixlRegU(dst), VixlReg(tmp2));
1550 } else {
1551 GetMasm()->Mov(VixlReg(dst), dst_register_1);
1552 GetMasm()->Mov(VixlRegU(dst), dst_register_2);
1553 }
1554 }
1555
MakeLibCall(Reg dst,Reg src,void * entry_point)1556 void Aarch32Encoder::MakeLibCall(Reg dst, Reg src, void *entry_point)
1557 {
1558 if (dst.IsFloat() && src.IsScalar()) {
1559 if (src.GetSize() != DOUBLE_WORD_SIZE) {
1560 SetFalseResult();
1561 return;
1562 }
1563
1564 bool save_r1 {src.GetId() != vixl::aarch32::r0.GetCode() || dst.GetType() == FLOAT64_TYPE};
1565
1566 GetMasm()->Push(vixl::aarch32::r0);
1567 if (save_r1) {
1568 GetMasm()->Push(vixl::aarch32::r1);
1569 }
1570
1571 if (src.GetId() != vixl::aarch32::r0.GetCode()) {
1572 GetMasm()->Mov(vixl::aarch32::r0, VixlReg(src));
1573 GetMasm()->Mov(vixl::aarch32::r1, VixlRegU(src));
1574 }
1575
1576 MakeCall(entry_point);
1577
1578 if (dst.GetType() == FLOAT64_TYPE) {
1579 GetMasm()->Vmov(VixlVReg(dst).D(), vixl::aarch32::r0, vixl::aarch32::r1);
1580 } else {
1581 GetMasm()->Vmov(VixlVReg(dst).S(), vixl::aarch32::r0);
1582 }
1583
1584 if (save_r1) {
1585 GetMasm()->Pop(vixl::aarch32::r1);
1586 }
1587 GetMasm()->Pop(vixl::aarch32::r0);
1588 } else if (dst.IsScalar() && src.IsFloat()) {
1589 if (dst.GetSize() != DOUBLE_WORD_SIZE) {
1590 SetFalseResult();
1591 return;
1592 }
1593
1594 bool save_r0_r1 {dst.GetId() != vixl::aarch32::r0.GetCode()};
1595
1596 if (save_r0_r1) {
1597 GetMasm()->Push(vixl::aarch32::r0);
1598 GetMasm()->Push(vixl::aarch32::r1);
1599 }
1600
1601 if (src.GetType() == FLOAT64_TYPE) {
1602 GetMasm()->Vmov(vixl::aarch32::r0, vixl::aarch32::r1, VixlVReg(src).D());
1603 } else {
1604 GetMasm()->Vmov(vixl::aarch32::r0, VixlVReg(src).S());
1605 }
1606
1607 MakeCall(entry_point);
1608
1609 if (save_r0_r1) {
1610 GetMasm()->Mov(VixlReg(dst), vixl::aarch32::r0);
1611 GetMasm()->Mov(VixlRegU(dst), vixl::aarch32::r1);
1612
1613 GetMasm()->Pop(vixl::aarch32::r1);
1614 GetMasm()->Pop(vixl::aarch32::r0);
1615 }
1616 } else {
1617 SetFalseResult();
1618 return;
1619 }
1620 }
1621
EncodeDiv(Reg dst,bool dst_signed,Reg src0,Reg src1)1622 void Aarch32Encoder::EncodeDiv(Reg dst, bool dst_signed, Reg src0, Reg src1)
1623 {
1624 if (dst.IsFloat()) {
1625 GetMasm()->Vdiv(VixlVReg(dst), VixlVReg(src0), VixlVReg(src1));
1626 return;
1627 }
1628
1629 if (dst.GetSize() <= WORD_SIZE) {
1630 if (!dst_signed) {
1631 GetMasm()->Udiv(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1632 return;
1633 }
1634
1635 GetMasm()->Sdiv(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1636 return;
1637 }
1638 if (dst_signed) {
1639 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(__aeabi_ldivmod));
1640 } else {
1641 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(__aeabi_uldivmod));
1642 }
1643 }
1644
EncodeMod(Reg dst,bool dst_signed,Reg src0,Reg src1)1645 void Aarch32Encoder::EncodeMod(Reg dst, bool dst_signed, Reg src0, Reg src1)
1646 {
1647 if (dst.GetType() == FLOAT32_TYPE) {
1648 using fp = float (*)(float, float);
1649 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(static_cast<fp>(fmodf)));
1650 return;
1651 }
1652 if (dst.GetType() == FLOAT64_TYPE) {
1653 using fp = double (*)(double, double);
1654 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(static_cast<fp>(fmod)));
1655 return;
1656 }
1657
1658 if (dst.GetSize() <= WORD_SIZE) {
1659 if (dst_signed) {
1660 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(__aeabi_idivmod), true);
1661 } else {
1662 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(__aeabi_uidivmod), true);
1663 }
1664
1665 // dst = -(tmp * src0) + src1
1666 if ((dst.GetSize() == BYTE_SIZE) && dst_signed) {
1667 GetMasm()->Sxtb(VixlReg(dst), VixlReg(dst));
1668 }
1669 if ((dst.GetSize() == HALF_SIZE) && dst_signed) {
1670 GetMasm()->Sxth(VixlReg(dst), VixlReg(dst));
1671 }
1672 return;
1673 }
1674
1675 // Call lib-method
1676 if (dst_signed) {
1677 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(__aeabi_ldivmod), true);
1678 } else {
1679 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(__aeabi_uldivmod), true);
1680 }
1681 }
1682
EncodeMin(Reg dst,bool dst_signed,Reg src0,Reg src1)1683 void Aarch32Encoder::EncodeMin(Reg dst, bool dst_signed, Reg src0, Reg src1)
1684 {
1685 if (dst.IsFloat()) {
1686 EncodeMinMaxFp<false>(dst, src0, src1);
1687 return;
1688 }
1689
1690 if (dst.GetSize() <= WORD_SIZE) {
1691 if (dst_signed) {
1692 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
1693 GetMasm()->Mov(Convert(Condition::LE), VixlReg(dst), VixlReg(src0));
1694 GetMasm()->Mov(Convert(Condition::GT), VixlReg(dst), VixlReg(src1));
1695 return;
1696 }
1697 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
1698 GetMasm()->Mov(Convert(Condition::LS), VixlReg(dst), VixlReg(src0));
1699 GetMasm()->Mov(Convert(Condition::HI), VixlReg(dst), VixlReg(src1));
1700 return;
1701 }
1702
1703 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1704 ScopedTmpRegU32 tmp_reg(this);
1705
1706 GetMasm()->Subs(VixlReg(tmp_reg), VixlReg(src0), VixlReg(src1));
1707 GetMasm()->Sbcs(VixlReg(tmp_reg), VixlRegU(src0), VixlRegU(src1));
1708
1709 auto cc = Convert(dst_signed ? Condition::LT : Condition::LO);
1710 GetMasm()->Mov(cc, VixlReg(dst), VixlReg(src0));
1711 GetMasm()->Mov(cc, VixlRegU(dst), VixlRegU(src0));
1712 GetMasm()->Mov(cc.Negate(), VixlReg(dst), VixlReg(src1));
1713 GetMasm()->Mov(cc.Negate(), VixlRegU(dst), VixlRegU(src1));
1714 }
1715
EncodeMax(Reg dst,bool dst_signed,Reg src0,Reg src1)1716 void Aarch32Encoder::EncodeMax(Reg dst, bool dst_signed, Reg src0, Reg src1)
1717 {
1718 if (dst.IsFloat()) {
1719 EncodeMinMaxFp<true>(dst, src0, src1);
1720 return;
1721 }
1722
1723 if (dst.GetSize() <= WORD_SIZE) {
1724 if (dst_signed) {
1725 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
1726 GetMasm()->Mov(Convert(Condition::GT), VixlReg(dst), VixlReg(src0));
1727 GetMasm()->Mov(Convert(Condition::LE), VixlReg(dst), VixlReg(src1));
1728 return;
1729 }
1730 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
1731 GetMasm()->Mov(Convert(Condition::HI), VixlReg(dst), VixlReg(src0));
1732 GetMasm()->Mov(Convert(Condition::LS), VixlReg(dst), VixlReg(src1));
1733 return;
1734 }
1735
1736 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1737 ScopedTmpRegU32 tmp_reg(this);
1738
1739 GetMasm()->Subs(VixlReg(tmp_reg), VixlReg(src0), VixlReg(src1));
1740 GetMasm()->Sbcs(VixlReg(tmp_reg), VixlRegU(src0), VixlRegU(src1));
1741
1742 auto cc = Convert(dst_signed ? Condition::LT : Condition::LO);
1743 GetMasm()->Mov(cc, VixlReg(dst), VixlReg(src1));
1744 GetMasm()->Mov(cc, VixlRegU(dst), VixlRegU(src1));
1745 GetMasm()->Mov(cc.Negate(), VixlReg(dst), VixlReg(src0));
1746 GetMasm()->Mov(cc.Negate(), VixlRegU(dst), VixlRegU(src0));
1747 }
1748
1749 template <bool is_max>
EncodeMinMaxFp(Reg dst,Reg src0,Reg src1)1750 void Aarch32Encoder::EncodeMinMaxFp(Reg dst, Reg src0, Reg src1)
1751 {
1752 Aarch32LabelHolder::LabelType not_equal(GetAllocator());
1753 Aarch32LabelHolder::LabelType got_nan(GetAllocator());
1754 Aarch32LabelHolder::LabelType end(GetAllocator());
1755 auto &src_a = dst.GetId() != src1.GetId() ? src0 : src1;
1756 auto &src_b = src_a.GetId() == src0.GetId() ? src1 : src0;
1757 GetMasm()->Vmov(VixlVReg(dst), VixlVReg(src_a));
1758 // Vcmp change flags:
1759 // NZCV
1760 // 0011 <- if any operand is NaN
1761 // 0110 <- operands are equals
1762 // 1000 <- operand0 < operand1
1763 // 0010 <- operand0 > operand1
1764 GetMasm()->Vcmp(VixlVReg(src_b), VixlVReg(src_a));
1765 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
1766 GetMasm()->B(Convert(Condition::VS), &got_nan);
1767 GetMasm()->B(Convert(Condition::NE), ¬_equal);
1768
1769 // calculate result for positive/negative zero operands
1770 if (is_max) {
1771 EncodeVand(dst, src_a, src_b);
1772 } else {
1773 EncodeVorr(dst, src_a, src_b);
1774 }
1775 GetMasm()->B(&end);
1776 GetMasm()->Bind(&got_nan);
1777 // if any operand is NaN result is NaN
1778 EncodeVorr(dst, src_a, src_b);
1779 GetMasm()->B(&end);
1780 GetMasm()->bind(¬_equal);
1781 // calculate min/max for other cases
1782 if (is_max) {
1783 GetMasm()->B(Convert(Condition::MI), &end);
1784 } else {
1785 GetMasm()->B(Convert(Condition::HI), &end);
1786 }
1787 GetMasm()->Vmov(VixlVReg(dst), VixlVReg(src_b));
1788 GetMasm()->bind(&end);
1789 }
1790
EncodeVorr(Reg dst,Reg src0,Reg src1)1791 void Aarch32Encoder::EncodeVorr(Reg dst, Reg src0, Reg src1)
1792 {
1793 if (dst.GetType() == FLOAT32_TYPE) {
1794 ScopedTmpRegF64 tmp_reg(this);
1795 GetMasm()->Vmov(vixl::aarch32::SRegister(tmp_reg.GetReg().GetId() + (src0.GetId() & 1U)), VixlVReg(src1).S());
1796 GetMasm()->Vorr(vixl::aarch32::DRegister(tmp_reg.GetReg().GetId() / 2U),
1797 vixl::aarch32::DRegister(src0.GetId() / 2U),
1798 vixl::aarch32::DRegister(tmp_reg.GetReg().GetId() / 2U));
1799 GetMasm()->Vmov(VixlVReg(dst).S(), vixl::aarch32::SRegister(tmp_reg.GetReg().GetId() + (src0.GetId() & 1U)));
1800
1801 } else {
1802 GetMasm()->Vorr(VixlVReg(dst).D(), VixlVReg(src0).D(), VixlVReg(src1).D());
1803 }
1804 }
1805
EncodeVand(Reg dst,Reg src0,Reg src1)1806 void Aarch32Encoder::EncodeVand(Reg dst, Reg src0, Reg src1)
1807 {
1808 if (dst.GetType() == FLOAT32_TYPE) {
1809 ScopedTmpRegF64 tmp_reg(this);
1810 GetMasm()->Vmov(vixl::aarch32::SRegister(tmp_reg.GetReg().GetId() + (src0.GetId() & 1U)), VixlVReg(src1).S());
1811 GetMasm()->Vand(vixl::aarch32::kDataTypeValueNone, vixl::aarch32::DRegister(tmp_reg.GetReg().GetId() / 2U),
1812 vixl::aarch32::DRegister(src0.GetId() / 2U),
1813 vixl::aarch32::DRegister(tmp_reg.GetReg().GetId() / 2U));
1814 GetMasm()->Vmov(VixlVReg(dst).S(), vixl::aarch32::SRegister(tmp_reg.GetReg().GetId() + (src0.GetId() & 1U)));
1815 } else {
1816 GetMasm()->Vand(vixl::aarch32::kDataTypeValueNone, VixlVReg(dst).D(), VixlVReg(src0).D(), VixlVReg(src1).D());
1817 }
1818 }
1819
EncodeShl(Reg dst,Reg src0,Reg src1)1820 void Aarch32Encoder::EncodeShl(Reg dst, Reg src0, Reg src1)
1821 {
1822 if (dst.GetSize() < WORD_SIZE) {
1823 GetMasm()->And(VixlReg(src1), VixlReg(src1), VixlImm(dst.GetSize() - 1));
1824 }
1825
1826 if (dst.GetSize() <= WORD_SIZE) {
1827 GetMasm()->Lsl(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1828 return;
1829 }
1830
1831 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1832 ScopedTmpRegU32 hi_reg(this);
1833 ScopedTmpRegU32 tmp_reg(this);
1834
1835 GetMasm()->Rsb(VixlReg(tmp_reg), VixlReg(src1), VixlImm(WORD_SIZE));
1836 GetMasm()->Lsr(VixlReg(tmp_reg), VixlReg(src0), VixlReg(tmp_reg));
1837 GetMasm()->Orr(VixlReg(hi_reg), VixlReg(tmp_reg),
1838 vixl::aarch32::Operand(VixlRegU(src0), vixl::aarch32::LSL, VixlReg(src1)));
1839 GetMasm()->Subs(VixlReg(tmp_reg), VixlReg(src1), VixlImm(WORD_SIZE));
1840 GetMasm()->Lsl(Convert(Condition::PL), VixlReg(hi_reg), VixlReg(src0), VixlReg(tmp_reg));
1841 GetMasm()->Mov(Convert(Condition::PL), VixlReg(dst), VixlImm(0x0));
1842 GetMasm()->Lsl(Convert(Condition::MI), VixlReg(dst), VixlReg(src0), VixlReg(src1));
1843 GetMasm()->Mov(VixlRegU(dst), VixlReg(hi_reg));
1844 }
1845
EncodeShr(Reg dst,Reg src0,Reg src1)1846 void Aarch32Encoder::EncodeShr(Reg dst, Reg src0, Reg src1)
1847 {
1848 if (dst.GetSize() < WORD_SIZE) {
1849 GetMasm()->And(VixlReg(src1), VixlReg(src1), VixlImm(dst.GetSize() - 1));
1850 }
1851
1852 if (dst.GetSize() <= WORD_SIZE) {
1853 GetMasm()->Lsr(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1854 return;
1855 }
1856
1857 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1858 ScopedTmpRegU32 lo_reg(this);
1859 ScopedTmpRegU32 tmp_reg(this);
1860
1861 GetMasm()->Rsb(VixlReg(tmp_reg), VixlReg(src1), VixlImm(WORD_SIZE));
1862 GetMasm()->Lsr(VixlReg(lo_reg), VixlReg(src0), VixlReg(src1));
1863 GetMasm()->Orr(VixlReg(lo_reg), VixlReg(lo_reg),
1864 vixl::aarch32::Operand(VixlRegU(src0), vixl::aarch32::LSL, VixlReg(tmp_reg)));
1865 GetMasm()->Subs(VixlReg(tmp_reg), VixlReg(src1), VixlImm(WORD_SIZE));
1866 GetMasm()->Lsr(Convert(Condition::PL), VixlReg(lo_reg), VixlRegU(src0), VixlReg(tmp_reg));
1867 GetMasm()->Mov(Convert(Condition::PL), VixlRegU(dst), VixlImm(0x0));
1868 GetMasm()->Lsr(Convert(Condition::MI), VixlRegU(dst), VixlRegU(src0), VixlReg(src1));
1869 GetMasm()->Mov(VixlReg(dst), VixlReg(lo_reg));
1870 }
1871
EncodeAShr(Reg dst,Reg src0,Reg src1)1872 void Aarch32Encoder::EncodeAShr(Reg dst, Reg src0, Reg src1)
1873 {
1874 if (dst.GetSize() < WORD_SIZE) {
1875 GetMasm()->And(VixlReg(src1), VixlReg(src1), VixlImm(dst.GetSize() - 1));
1876 }
1877
1878 if (dst.GetSize() <= WORD_SIZE) {
1879 GetMasm()->Asr(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1880 return;
1881 }
1882
1883 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1884 ScopedTmpRegU32 lo_reg(this);
1885 ScopedTmpRegU32 tmp_reg(this);
1886
1887 GetMasm()->Subs(VixlReg(tmp_reg), VixlReg(src1), VixlImm(WORD_SIZE));
1888 GetMasm()->Lsr(VixlReg(lo_reg), VixlReg(src0), VixlReg(src1));
1889 GetMasm()->Rsb(VixlReg(tmp_reg), VixlReg(src1), VixlImm(WORD_SIZE));
1890 GetMasm()->Orr(VixlReg(lo_reg), VixlReg(lo_reg),
1891 vixl::aarch32::Operand(VixlRegU(src0), vixl::aarch32::LSL, VixlReg(tmp_reg)));
1892 GetMasm()->Rsb(Convert(Condition::PL), VixlReg(tmp_reg), VixlReg(tmp_reg), VixlImm(0x0));
1893 GetMasm()->Asr(Convert(Condition::PL), VixlReg(lo_reg), VixlRegU(src0), VixlReg(tmp_reg));
1894 GetMasm()->Asr(Convert(Condition::PL), VixlRegU(dst), VixlRegU(src0), VixlImm(WORD_SIZE - 1));
1895 GetMasm()->Asr(Convert(Condition::MI), VixlRegU(dst), VixlRegU(src0), VixlReg(src1));
1896 GetMasm()->Mov(VixlReg(dst), VixlReg(lo_reg));
1897 }
1898
EncodeAnd(Reg dst,Reg src0,Reg src1)1899 void Aarch32Encoder::EncodeAnd(Reg dst, Reg src0, Reg src1)
1900 {
1901 GetMasm()->And(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1902 if (dst.GetSize() > WORD_SIZE) {
1903 GetMasm()->And(VixlRegU(dst), VixlRegU(src0), VixlRegU(src1));
1904 }
1905 }
1906
EncodeOr(Reg dst,Reg src0,Reg src1)1907 void Aarch32Encoder::EncodeOr(Reg dst, Reg src0, Reg src1)
1908 {
1909 GetMasm()->Orr(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1910 if (dst.GetSize() > WORD_SIZE) {
1911 GetMasm()->Orr(VixlRegU(dst), VixlRegU(src0), VixlRegU(src1));
1912 }
1913 }
1914
EncodeXor(Reg dst,Reg src0,Reg src1)1915 void Aarch32Encoder::EncodeXor(Reg dst, Reg src0, Reg src1)
1916 {
1917 GetMasm()->Eor(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1918 if (dst.GetSize() > WORD_SIZE) {
1919 GetMasm()->Eor(VixlRegU(dst), VixlRegU(src0), VixlRegU(src1));
1920 }
1921 }
1922
EncodeAdd(Reg dst,Reg src,Imm imm)1923 void Aarch32Encoder::EncodeAdd(Reg dst, Reg src, Imm imm)
1924 {
1925 ASSERT(dst.IsScalar() && "UNIMPLEMENTED");
1926 if (dst.GetSize() <= WORD_SIZE) {
1927 GetMasm()->Add(VixlReg(dst), VixlReg(src), VixlImm(imm));
1928 return;
1929 }
1930
1931 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1932 GetMasm()->Adds(VixlReg(dst), VixlReg(src), VixlImm(imm));
1933 GetMasm()->Adc(VixlRegU(dst), VixlRegU(src), VixlImmU(imm));
1934 }
1935
EncodeSub(Reg dst,Reg src,Imm imm)1936 void Aarch32Encoder::EncodeSub(Reg dst, Reg src, Imm imm)
1937 {
1938 ASSERT(dst.IsScalar() && "UNIMPLEMENTED");
1939 if (dst.GetSize() <= WORD_SIZE) {
1940 GetMasm()->Sub(VixlReg(dst), VixlReg(src), VixlImm(imm));
1941 return;
1942 }
1943
1944 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1945 GetMasm()->Subs(VixlReg(dst), VixlReg(src), VixlImm(imm));
1946 GetMasm()->Sbc(VixlRegU(dst), VixlRegU(src), VixlImmU(imm));
1947 }
1948
EncodeShl(Reg dst,Reg src,Imm imm)1949 void Aarch32Encoder::EncodeShl(Reg dst, Reg src, Imm imm)
1950 {
1951 auto value = static_cast<uint32_t>(GetIntValue(imm));
1952 int32_t imm_value = value & (dst.GetSize() - 1);
1953
1954 ASSERT(dst.IsScalar() && "Invalid operand type");
1955 if (dst.GetSize() <= WORD_SIZE) {
1956 GetMasm()->Lsl(VixlReg(dst), VixlReg(src), VixlImm(imm_value));
1957 return;
1958 }
1959
1960 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1961 ScopedTmpRegU32 hi_reg(this);
1962 ScopedTmpRegU32 tmp_reg(this);
1963
1964 GetMasm()->Lsr(VixlReg(tmp_reg), VixlReg(src), VixlImm(WORD_SIZE - imm_value));
1965 GetMasm()->Mov(VixlReg(hi_reg), VixlImm(imm_value));
1966 GetMasm()->Orr(VixlReg(hi_reg), VixlReg(tmp_reg),
1967 vixl::aarch32::Operand(VixlRegU(src), vixl::aarch32::LSL, VixlReg(hi_reg)));
1968 GetMasm()->Movs(VixlReg(tmp_reg), VixlImm(imm_value - WORD_SIZE));
1969 GetMasm()->Lsl(Convert(Condition::PL), VixlReg(hi_reg), VixlReg(src), VixlReg(tmp_reg));
1970 GetMasm()->Mov(Convert(Condition::PL), VixlReg(dst), VixlImm(0x0));
1971 GetMasm()->Lsl(Convert(Condition::MI), VixlReg(dst), VixlReg(src), VixlImm(imm_value));
1972 GetMasm()->Mov(VixlRegU(dst), VixlReg(hi_reg));
1973 }
1974
EncodeShr(Reg dst,Reg src,Imm imm)1975 void Aarch32Encoder::EncodeShr(Reg dst, Reg src, Imm imm)
1976 {
1977 auto value = static_cast<uint32_t>(GetIntValue(imm));
1978 int32_t imm_value = value & (dst.GetSize() - 1);
1979
1980 ASSERT(dst.IsScalar() && "Invalid operand type");
1981 if (dst.GetSize() <= WORD_SIZE) {
1982 GetMasm()->Lsr(VixlReg(dst), VixlReg(src), imm_value);
1983 return;
1984 }
1985
1986 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1987 ScopedTmpRegU32 lo_reg(this);
1988 ScopedTmpRegU32 tmp_reg(this);
1989
1990 GetMasm()->Mov(VixlReg(tmp_reg), VixlImm(WORD_SIZE - imm_value));
1991 GetMasm()->Lsr(VixlReg(lo_reg), VixlReg(src), VixlImm(imm_value));
1992 GetMasm()->Orr(VixlReg(lo_reg), VixlReg(lo_reg),
1993 vixl::aarch32::Operand(VixlRegU(src), vixl::aarch32::LSL, VixlReg(tmp_reg)));
1994 GetMasm()->Movs(VixlReg(tmp_reg), VixlImm(imm_value - WORD_SIZE));
1995 GetMasm()->Lsr(Convert(Condition::PL), VixlReg(lo_reg), VixlRegU(src), VixlReg(tmp_reg));
1996 GetMasm()->Mov(Convert(Condition::PL), VixlRegU(dst), VixlImm(0x0));
1997 GetMasm()->Lsr(Convert(Condition::MI), VixlRegU(dst), VixlRegU(src), VixlImm(imm_value));
1998 GetMasm()->Mov(VixlReg(dst), VixlReg(lo_reg));
1999 }
2000
EncodeAShr(Reg dst,Reg src,Imm imm)2001 void Aarch32Encoder::EncodeAShr(Reg dst, Reg src, Imm imm)
2002 {
2003 ASSERT(dst.IsScalar() && "Invalid operand type");
2004
2005 auto value = static_cast<uint32_t>(GetIntValue(imm));
2006 int32_t imm_value = value & (dst.GetSize() - 1);
2007
2008 if (dst.GetSize() <= WORD_SIZE) {
2009 GetMasm()->Asr(VixlReg(dst), VixlReg(src), imm_value);
2010 return;
2011 }
2012
2013 ScopedTmpRegU32 lo_reg(this);
2014 ScopedTmpRegU32 tmp_reg(this);
2015 GetMasm()->Movs(VixlReg(tmp_reg), VixlImm(imm_value - WORD_SIZE));
2016 GetMasm()->Lsr(VixlReg(lo_reg), VixlReg(src), VixlImm(imm_value));
2017 GetMasm()->Mov(VixlReg(tmp_reg), VixlImm(WORD_SIZE - imm_value));
2018 GetMasm()->Orr(VixlReg(lo_reg), VixlReg(lo_reg),
2019 vixl::aarch32::Operand(VixlRegU(src), vixl::aarch32::LSL, VixlReg(tmp_reg)));
2020 GetMasm()->Rsb(Convert(Condition::PL), VixlReg(tmp_reg), VixlReg(tmp_reg), VixlImm(0x0));
2021 GetMasm()->Asr(Convert(Condition::PL), VixlReg(lo_reg), VixlRegU(src), VixlReg(tmp_reg));
2022 GetMasm()->Asr(Convert(Condition::PL), VixlRegU(dst), VixlRegU(src), VixlImm(WORD_SIZE - 1));
2023 GetMasm()->Asr(Convert(Condition::MI), VixlRegU(dst), VixlRegU(src), VixlImm(imm_value));
2024 GetMasm()->Mov(VixlReg(dst), VixlReg(lo_reg));
2025 }
2026
EncodeAnd(Reg dst,Reg src,Imm imm)2027 void Aarch32Encoder::EncodeAnd(Reg dst, Reg src, Imm imm)
2028 {
2029 ASSERT(dst.IsScalar() && "Invalid operand type");
2030 GetMasm()->And(VixlReg(dst), VixlReg(src), VixlImm(imm));
2031 if (dst.GetSize() > WORD_SIZE) {
2032 GetMasm()->And(VixlRegU(dst), VixlRegU(src), VixlImmU(imm));
2033 }
2034 }
2035
EncodeOr(Reg dst,Reg src,Imm imm)2036 void Aarch32Encoder::EncodeOr(Reg dst, Reg src, Imm imm)
2037 {
2038 ASSERT(dst.IsScalar() && "Invalid operand type");
2039 GetMasm()->Orr(VixlReg(dst), VixlReg(src), VixlImm(imm));
2040 if (dst.GetSize() > WORD_SIZE) {
2041 GetMasm()->Orr(VixlRegU(dst), VixlRegU(src), VixlImmU(imm));
2042 }
2043 }
2044
EncodeXor(Reg dst,Reg src,Imm imm)2045 void Aarch32Encoder::EncodeXor(Reg dst, Reg src, Imm imm)
2046 {
2047 ASSERT(dst.IsScalar() && "Invalid operand type");
2048 GetMasm()->Eor(VixlReg(dst), VixlReg(src), VixlImm(imm));
2049 if (dst.GetSize() > WORD_SIZE) {
2050 GetMasm()->Eor(VixlRegU(dst), VixlRegU(src), VixlImmU(imm));
2051 }
2052 }
2053
EncodeMov(Reg dst,Imm src)2054 void Aarch32Encoder::EncodeMov(Reg dst, Imm src)
2055 {
2056 if (dst.IsFloat()) {
2057 if (dst.GetSize() == WORD_SIZE) {
2058 GetMasm()->Vmov(Convert(dst.GetType()), VixlVReg(dst).S(), VixlNeonImm(src.GetValue<float>()));
2059 } else {
2060 GetMasm()->Vmov(Convert(dst.GetType()), VixlVReg(dst).D(), VixlNeonImm(src.GetValue<double>()));
2061 }
2062 return;
2063 }
2064
2065 GetMasm()->Mov(VixlReg(dst), VixlImm(src));
2066 if (dst.GetSize() > WORD_SIZE) {
2067 GetMasm()->Mov(VixlRegU(dst), VixlImmU(src));
2068 }
2069 }
2070
EncodeLdr(Reg dst,bool dst_signed,const vixl::aarch32::MemOperand & vixl_mem)2071 void Aarch32Encoder::EncodeLdr(Reg dst, bool dst_signed, const vixl::aarch32::MemOperand &vixl_mem)
2072 {
2073 if (dst.IsFloat()) {
2074 if (dst.GetSize() == WORD_SIZE) {
2075 GetMasm()->Vldr(VixlVReg(dst).S(), vixl_mem);
2076 } else {
2077 GetMasm()->Vldr(VixlVReg(dst).D(), vixl_mem);
2078 }
2079 return;
2080 }
2081 if (dst_signed) {
2082 if (dst.GetSize() == BYTE_SIZE) {
2083 GetMasm()->Ldrsb(VixlReg(dst), vixl_mem);
2084 return;
2085 }
2086 if (dst.GetSize() == HALF_SIZE) {
2087 GetMasm()->Ldrsh(VixlReg(dst), vixl_mem);
2088 return;
2089 }
2090 } else {
2091 if (dst.GetSize() == BYTE_SIZE) {
2092 GetMasm()->Ldrb(VixlReg(dst), vixl_mem);
2093 return;
2094 }
2095 if (dst.GetSize() == HALF_SIZE) {
2096 GetMasm()->Ldrh(VixlReg(dst), vixl_mem);
2097 return;
2098 }
2099 }
2100 if (dst.GetSize() == WORD_SIZE) {
2101 GetMasm()->Ldr(VixlReg(dst), vixl_mem);
2102 return;
2103 }
2104 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
2105 GetMasm()->Ldrd(VixlReg(dst), VixlRegU(dst), vixl_mem);
2106 }
2107
EncodeLdr(Reg dst,bool dst_signed,MemRef mem)2108 void Aarch32Encoder::EncodeLdr(Reg dst, bool dst_signed, MemRef mem)
2109 {
2110 auto type = dst.GetType();
2111 if (IsNeedToPrepareMemLdS(mem, type, dst_signed)) {
2112 ScopedTmpRegU32 tmp_reg(this);
2113 auto tmp = VixlReg(tmp_reg);
2114 auto vixl_mem = PrepareMemLdS(mem, type, tmp, dst_signed);
2115 EncodeLdr(dst, dst_signed, vixl_mem);
2116 } else {
2117 auto vixl_mem = ConvertMem(mem);
2118 EncodeLdr(dst, dst_signed, vixl_mem);
2119 }
2120 }
2121
EncodeLdrAcquire(Reg dst,bool dst_signed,MemRef mem)2122 void Aarch32Encoder::EncodeLdrAcquire(Reg dst, bool dst_signed, MemRef mem)
2123 {
2124 EncodeLdr(dst, dst_signed, mem);
2125 GetMasm()->Dmb(vixl::aarch32::MemoryBarrierType::ISH);
2126 }
2127
EncodeMemoryBarrier(MemoryOrder::Order order)2128 void Aarch32Encoder::EncodeMemoryBarrier(MemoryOrder::Order order)
2129 {
2130 switch (order) {
2131 case MemoryOrder::Acquire:
2132 case MemoryOrder::Release: {
2133 GetMasm()->Dmb(vixl::aarch32::MemoryBarrierType::ISH);
2134 break;
2135 }
2136 case MemoryOrder::Full: {
2137 GetMasm()->Dmb(vixl::aarch32::MemoryBarrierType::ISHST);
2138 break;
2139 }
2140 default:
2141 break;
2142 }
2143 }
2144
EncodeStr(Reg src,const vixl::aarch32::MemOperand & vixl_mem)2145 void Aarch32Encoder::EncodeStr(Reg src, const vixl::aarch32::MemOperand &vixl_mem)
2146 {
2147 if (src.IsFloat()) {
2148 if (src.GetSize() == WORD_SIZE) {
2149 GetMasm()->Vstr(VixlVReg(src).S(), vixl_mem);
2150 } else {
2151 GetMasm()->Vstr(VixlVReg(src).D(), vixl_mem);
2152 }
2153 } else if (src.GetSize() == BYTE_SIZE) {
2154 GetMasm()->Strb(VixlReg(src), vixl_mem);
2155 } else if (src.GetSize() == HALF_SIZE) {
2156 GetMasm()->Strh(VixlReg(src), vixl_mem);
2157 } else if (src.GetSize() == WORD_SIZE) {
2158 GetMasm()->Str(VixlReg(src), vixl_mem);
2159 } else {
2160 ASSERT(src.GetSize() == DOUBLE_WORD_SIZE);
2161 GetMasm()->Strd(VixlReg(src), VixlRegU(src), vixl_mem);
2162 }
2163 }
2164
EncodeStr(Reg src,MemRef mem)2165 void Aarch32Encoder::EncodeStr(Reg src, MemRef mem)
2166 {
2167 auto type = src.GetType();
2168 if (IsNeedToPrepareMemLdS(mem, type, false)) {
2169 ScopedTmpRegU32 tmp_reg(this);
2170 auto tmp = VixlReg(tmp_reg);
2171 auto vixl_mem = PrepareMemLdS(mem, type, tmp, false);
2172 EncodeStr(src, vixl_mem);
2173 } else {
2174 auto vixl_mem = ConvertMem(mem);
2175 EncodeStr(src, vixl_mem);
2176 }
2177 }
2178
EncodeStrRelease(Reg src,MemRef mem)2179 void Aarch32Encoder::EncodeStrRelease(Reg src, MemRef mem)
2180 {
2181 GetMasm()->Dmb(vixl::aarch32::MemoryBarrierType::ISH);
2182 EncodeStr(src, mem);
2183 GetMasm()->Dmb(vixl::aarch32::MemoryBarrierType::ISH);
2184 }
2185
EncodeStrz(Reg src,MemRef mem)2186 void Aarch32Encoder::EncodeStrz(Reg src, MemRef mem)
2187 {
2188 if (src.GetSize() <= WORD_SIZE) {
2189 EncodeSti(Imm(static_cast<int64_t>(0)), mem);
2190 }
2191 EncodeStr(src, mem);
2192 }
2193
EncodeStp(Reg src0,Reg src1,MemRef mem)2194 void Aarch32Encoder::EncodeStp(Reg src0, Reg src1, MemRef mem)
2195 {
2196 ASSERT(src0.IsFloat() == src1.IsFloat());
2197 ASSERT(src0.GetSize() == src1.GetSize());
2198 EncodeStr(src0, mem);
2199 EncodeStr(src1, MemRef(mem.GetBase(), mem.GetIndex(), mem.GetScale(), mem.GetDisp() + WORD_SIZE_BYTE));
2200 }
2201
EncodeLdrExclusive(Reg dst,Reg addr,bool acquire)2202 void Aarch32Encoder::EncodeLdrExclusive(Reg dst, Reg addr, bool acquire)
2203 {
2204 ASSERT(dst.IsScalar());
2205 auto dst_reg = VixlReg(dst);
2206 auto mem_cvt = ConvertMem(MemRef(addr));
2207 if (dst.GetSize() == BYTE_SIZE) {
2208 if (acquire) {
2209 GetMasm()->Ldaexb(dst_reg, mem_cvt);
2210 return;
2211 }
2212 GetMasm()->Ldrexb(dst_reg, mem_cvt);
2213 return;
2214 }
2215 if (dst.GetSize() == HALF_SIZE) {
2216 if (acquire) {
2217 GetMasm()->Ldaexh(dst_reg, mem_cvt);
2218 return;
2219 }
2220 GetMasm()->Ldrexh(dst_reg, mem_cvt);
2221 return;
2222 }
2223 if (dst.GetSize() == DOUBLE_WORD_SIZE) {
2224 auto dst_reg_u = VixlRegU(dst);
2225 if (acquire) {
2226 GetMasm()->Ldaexd(dst_reg, dst_reg_u, mem_cvt);
2227 return;
2228 }
2229 GetMasm()->Ldrexd(dst_reg, dst_reg_u, mem_cvt);
2230 return;
2231 }
2232 if (acquire) {
2233 GetMasm()->Ldaex(dst_reg, mem_cvt);
2234 return;
2235 }
2236 GetMasm()->Ldrex(dst_reg, mem_cvt);
2237 }
2238
EncodeStrExclusive(Reg dst,Reg src,Reg addr,bool release)2239 void Aarch32Encoder::EncodeStrExclusive(Reg dst, Reg src, Reg addr, bool release)
2240 {
2241 ASSERT(dst.IsScalar() && src.IsScalar());
2242 ASSERT(dst.GetSize() != DOUBLE_WORD_SIZE);
2243
2244 bool copy_dst = dst.GetId() == src.GetId() || dst.GetId() == addr.GetId();
2245 ScopedTmpReg tmp(this);
2246 auto dst_reg = copy_dst ? VixlReg(tmp) : VixlReg(dst);
2247 auto src_reg = VixlReg(src);
2248 auto mem_cvt = ConvertMem(MemRef(addr));
2249
2250 if (src.GetSize() == BYTE_SIZE) {
2251 if (release) {
2252 GetMasm()->Stlexb(dst_reg, src_reg, mem_cvt);
2253 } else {
2254 GetMasm()->Strexb(dst_reg, src_reg, mem_cvt);
2255 }
2256 } else if (src.GetSize() == HALF_SIZE) {
2257 if (release) {
2258 GetMasm()->Stlexh(dst_reg, src_reg, mem_cvt);
2259 } else {
2260 GetMasm()->Strexh(dst_reg, src_reg, mem_cvt);
2261 }
2262 } else if (src.GetSize() == DOUBLE_WORD_SIZE) {
2263 auto src_reg_u = VixlRegU(src);
2264 if (release) {
2265 GetMasm()->Stlexd(dst_reg, src_reg, src_reg_u, mem_cvt);
2266 } else {
2267 GetMasm()->Strexd(dst_reg, src_reg, src_reg_u, mem_cvt);
2268 }
2269 } else {
2270 if (release) {
2271 GetMasm()->Stlex(dst_reg, src_reg, mem_cvt);
2272 } else {
2273 GetMasm()->Strex(dst_reg, src_reg, mem_cvt);
2274 }
2275 }
2276
2277 if (copy_dst) {
2278 EncodeMov(dst, tmp);
2279 }
2280 }
2281
FindRegForMem(vixl::aarch32::MemOperand mem)2282 inline static int32_t FindRegForMem(vixl::aarch32::MemOperand mem)
2283 {
2284 int32_t base_reg_id = mem.GetBaseRegister().GetCode();
2285 int32_t index_reg_id = -1;
2286 if (mem.IsShiftedRegister()) {
2287 index_reg_id = mem.GetOffsetRegister().GetCode();
2288 }
2289 // find regs for mem
2290 constexpr int32_t STEP = 2;
2291 for (int32_t i = 0; i < static_cast<int32_t>(BYTE_SIZE); i += STEP) {
2292 if (base_reg_id == i || base_reg_id == i + 1 || index_reg_id == i || index_reg_id == i + 1) {
2293 continue;
2294 }
2295 return i;
2296 }
2297 UNREACHABLE();
2298 return -1;
2299 }
2300
EncodeSti(Imm src,MemRef mem)2301 void Aarch32Encoder::EncodeSti(Imm src, MemRef mem)
2302 {
2303 if (src.GetType().IsFloat()) {
2304 EncodeFloatSti(src, mem);
2305 return;
2306 }
2307 ScopedTmpRegU32 tmp_reg(this);
2308 auto tmp = VixlReg(tmp_reg);
2309 auto type = src.GetType();
2310 if (src.GetSize() <= WORD_SIZE) {
2311 auto vixl_mem = PrepareMemLdS(mem, type, tmp, false);
2312 if (vixl_mem.GetBaseRegister().GetCode() == tmp.GetCode()) {
2313 ScopedTmpRegU32 tmp1_reg(this);
2314 tmp = VixlReg(tmp1_reg);
2315 }
2316 GetMasm()->Mov(tmp, VixlImm(src));
2317 if (src.GetSize() == BYTE_SIZE) {
2318 GetMasm()->Strb(tmp, vixl_mem);
2319 return;
2320 }
2321 if (src.GetSize() == HALF_SIZE) {
2322 GetMasm()->Strh(tmp, vixl_mem);
2323 return;
2324 }
2325 GetMasm()->Str(tmp, vixl_mem);
2326 return;
2327 }
2328
2329 auto vixl_mem = PrepareMemLdS(mem, type, tmp, false, true);
2330 ASSERT(src.GetSize() == DOUBLE_WORD_SIZE);
2331 vixl::aarch32::Register tmp_imm1;
2332 vixl::aarch32::Register tmp_imm2;
2333 // if tmp isn't base reg and tmp is even and tmp+1 isn't SP we can use tmp and tmp + 1
2334 if (vixl_mem.GetBaseRegister().GetCode() != tmp.GetCode() && (tmp.GetCode() % 2U == 0) &&
2335 tmp.GetCode() + 1 != vixl::aarch32::sp.GetCode()) {
2336 tmp_imm1 = tmp;
2337 tmp_imm2 = vixl::aarch32::Register(tmp.GetCode() + 1);
2338 } else {
2339 auto reg_id = FindRegForMem(vixl_mem);
2340 ASSERT(reg_id != -1);
2341 tmp_imm1 = vixl::aarch32::Register(reg_id);
2342 tmp_imm2 = vixl::aarch32::Register(reg_id + 1);
2343 GetMasm()->Push(tmp_imm1);
2344 }
2345
2346 ASSERT(tmp_imm1.IsValid() && tmp_imm2.IsValid());
2347 GetMasm()->Push(tmp_imm2);
2348 GetMasm()->Mov(tmp_imm1, VixlImm(src));
2349 GetMasm()->Mov(tmp_imm2, VixlImmU(src));
2350 GetMasm()->Strd(tmp_imm1, tmp_imm2, vixl_mem);
2351 GetMasm()->Pop(tmp_imm2);
2352 if (tmp_imm1.GetCode() != tmp.GetCode()) {
2353 GetMasm()->Pop(tmp_imm1);
2354 }
2355 }
2356
EncodeFloatSti(Imm src,MemRef mem)2357 void Aarch32Encoder::EncodeFloatSti(Imm src, MemRef mem)
2358 {
2359 ASSERT(src.GetType().IsFloat());
2360 if (src.GetSize() == WORD_SIZE) {
2361 ScopedTmpRegF32 tmp_reg(this);
2362 GetMasm()->Vmov(VixlVReg(tmp_reg).S(), src.GetValue<float>());
2363 EncodeStr(tmp_reg, mem);
2364 } else {
2365 ScopedTmpRegF64 tmp_reg(this);
2366 GetMasm()->Vmov(VixlVReg(tmp_reg).D(), src.GetValue<double>());
2367 EncodeStr(tmp_reg, mem);
2368 }
2369 }
2370
EncodeMemCopy(MemRef mem_from,MemRef mem_to,size_t size)2371 void Aarch32Encoder::EncodeMemCopy(MemRef mem_from, MemRef mem_to, size_t size)
2372 {
2373 if (size == DOUBLE_WORD_SIZE && mem_from.IsOffsetMem() && mem_to.IsOffsetMem()) {
2374 EncodeMemCopy(mem_from, mem_to, WORD_SIZE);
2375 constexpr int32_t STEP = 4;
2376 auto offset_from = mem_from.GetDisp() + STEP;
2377 auto offset_to = mem_to.GetDisp() + STEP;
2378 EncodeMemCopy(MemRef(mem_from.GetBase(), offset_from), MemRef(mem_to.GetBase(), offset_to), WORD_SIZE);
2379
2380 return;
2381 }
2382 ScopedTmpRegU32 tmp_reg(this);
2383 auto tmp = VixlReg(tmp_reg);
2384 ScopedTmpRegU32 tmp_reg1(this);
2385 auto tmp1 = VixlReg(tmp_reg1);
2386 if (size == BYTE_SIZE) {
2387 GetMasm()->Ldrb(tmp, PrepareMemLdS(mem_from, INT8_TYPE, tmp, false));
2388 GetMasm()->Strb(tmp, PrepareMemLdS(mem_to, INT8_TYPE, tmp1, false));
2389 } else if (size == HALF_SIZE) {
2390 GetMasm()->Ldrh(tmp, PrepareMemLdS(mem_from, INT16_TYPE, tmp, false));
2391 GetMasm()->Strh(tmp, PrepareMemLdS(mem_to, INT16_TYPE, tmp1, false));
2392 } else if (size == WORD_SIZE) {
2393 GetMasm()->Ldr(tmp, PrepareMemLdS(mem_from, INT32_TYPE, tmp, false));
2394 GetMasm()->Str(tmp, PrepareMemLdS(mem_to, INT32_TYPE, tmp1, false));
2395 } else {
2396 ASSERT(size == DOUBLE_WORD_SIZE);
2397
2398 auto vixl_mem_from = PrepareMemLdS(mem_from, INT64_TYPE, tmp, false, true);
2399 auto vixl_mem_to = PrepareMemLdS(mem_to, INT64_TYPE, tmp1, false, true);
2400 auto reg_id = FindRegForMem(vixl_mem_to);
2401 ASSERT(reg_id != -1);
2402 [[maybe_unused]] constexpr auto IMM_2 = 2;
2403 ASSERT(reg_id % IMM_2 == 0);
2404 vixl::aarch32::Register tmp_copy1(reg_id);
2405 vixl::aarch32::Register tmp_copy2(reg_id + 1);
2406
2407 GetMasm()->Push(tmp_copy1);
2408 GetMasm()->Push(tmp_copy2);
2409 GetMasm()->Ldrd(tmp_copy1, tmp_copy2, vixl_mem_from);
2410 GetMasm()->Strd(tmp_copy1, tmp_copy2, vixl_mem_to);
2411 GetMasm()->Pop(tmp_copy2);
2412 GetMasm()->Pop(tmp_copy1);
2413 }
2414 }
2415
EncodeMemCopyz(MemRef mem_from,MemRef mem_to,size_t size)2416 void Aarch32Encoder::EncodeMemCopyz(MemRef mem_from, MemRef mem_to, size_t size)
2417 {
2418 ScopedTmpRegU32 tmp_reg(this);
2419 auto tmp = VixlReg(tmp_reg);
2420 ScopedTmpRegU32 tmp_reg1(this);
2421 auto tmp1 = VixlReg(tmp_reg1);
2422
2423 auto type = TypeInfo::GetScalarTypeBySize(size);
2424
2425 auto vixl_mem_from = PrepareMemLdS(mem_from, type, tmp, false, true);
2426 auto vixl_mem_to = PrepareMemLdS(mem_to, INT64_TYPE, tmp1, false, true);
2427 auto reg_id = FindRegForMem(vixl_mem_to);
2428 ASSERT(reg_id != -1);
2429 [[maybe_unused]] constexpr auto IMM_2 = 2;
2430 ASSERT(reg_id % IMM_2 == 0);
2431 vixl::aarch32::Register tmp_copy1(reg_id);
2432 vixl::aarch32::Register tmp_copy2(reg_id + 1);
2433
2434 GetMasm()->Push(tmp_copy1);
2435 GetMasm()->Push(tmp_copy2);
2436 if (size == BYTE_SIZE) {
2437 GetMasm()->Ldrb(tmp_copy1, vixl_mem_from);
2438 GetMasm()->Mov(tmp_copy2, VixlImm(0));
2439 GetMasm()->Strd(tmp_copy1, tmp_copy2, vixl_mem_to);
2440 } else if (size == HALF_SIZE) {
2441 GetMasm()->Ldrh(tmp_copy1, vixl_mem_from);
2442 GetMasm()->Mov(tmp_copy2, VixlImm(0));
2443 GetMasm()->Strd(tmp_copy1, tmp_copy2, vixl_mem_to);
2444 } else if (size == WORD_SIZE) {
2445 GetMasm()->Ldr(tmp_copy1, vixl_mem_from);
2446 GetMasm()->Mov(tmp_copy2, VixlImm(0));
2447 GetMasm()->Strd(tmp_copy1, tmp_copy2, vixl_mem_to);
2448 } else {
2449 ASSERT(size == DOUBLE_WORD_SIZE);
2450 GetMasm()->Ldrd(tmp_copy1, tmp_copy2, vixl_mem_from);
2451 GetMasm()->Strd(tmp_copy1, tmp_copy2, vixl_mem_to);
2452 }
2453 GetMasm()->Pop(tmp_copy2);
2454 GetMasm()->Pop(tmp_copy1);
2455 }
2456
CompareHelper(Reg src0,Reg src1,Condition * cc)2457 void Aarch32Encoder::CompareHelper(Reg src0, Reg src1, Condition *cc)
2458 {
2459 if (src0.IsFloat() && src1.IsFloat()) {
2460 GetMasm()->Vcmp(VixlVReg(src0), VixlVReg(src1));
2461 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
2462 } else if (src0.GetSize() <= WORD_SIZE && src1.GetSize() <= WORD_SIZE) {
2463 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
2464 } else {
2465 if (!IsConditionSigned(*cc)) {
2466 GetMasm()->Cmp(VixlRegU(src0), VixlRegU(src1));
2467 GetMasm()->Cmp(Convert(Condition::EQ), VixlReg(src0), VixlReg(src1));
2468 } else {
2469 bool swap = false;
2470 switch (*cc) {
2471 case Condition::GT:
2472 swap = true;
2473 *cc = Condition::LT;
2474 break;
2475 case Condition::LE:
2476 swap = true;
2477 *cc = Condition::GE;
2478 break;
2479 case Condition::GE:
2480 case Condition::LT:
2481 break;
2482 default:
2483 UNREACHABLE();
2484 }
2485
2486 Reg op0 = swap ? src1 : src0;
2487 Reg op1 = swap ? src0 : src1;
2488 ScopedTmpRegU32 tmp_reg(this);
2489 GetMasm()->Cmp(VixlReg(op0), VixlReg(op1));
2490 GetMasm()->Sbcs(VixlReg(tmp_reg), VixlRegU(op0), VixlRegU(op1));
2491 }
2492 }
2493 }
2494
TestHelper(Reg src0,Reg src1,Condition cc)2495 void Aarch32Encoder::TestHelper(Reg src0, Reg src1, [[maybe_unused]] Condition cc)
2496 {
2497 ASSERT(!src0.IsFloat() && !src1.IsFloat());
2498 ASSERT(cc == Condition::TST_EQ || cc == Condition::TST_NE);
2499
2500 if (src0.GetSize() <= WORD_SIZE && src1.GetSize() <= WORD_SIZE) {
2501 GetMasm()->Tst(VixlReg(src0), VixlReg(src1));
2502 } else {
2503 GetMasm()->Tst(VixlRegU(src0), VixlRegU(src1));
2504 GetMasm()->Tst(Convert(Condition::EQ), VixlReg(src0), VixlReg(src1));
2505 }
2506 }
2507
EncodeCompare(Reg dst,Reg src0,Reg src1,Condition cc)2508 void Aarch32Encoder::EncodeCompare(Reg dst, Reg src0, Reg src1, Condition cc)
2509 {
2510 CompareHelper(src0, src1, &cc);
2511 GetMasm()->Mov(Convert(cc), VixlReg(dst), 0x1);
2512 GetMasm()->Mov(Convert(cc).Negate(), VixlReg(dst), 0x0);
2513 }
2514
EncodeCompareTest(Reg dst,Reg src0,Reg src1,Condition cc)2515 void Aarch32Encoder::EncodeCompareTest(Reg dst, Reg src0, Reg src1, Condition cc)
2516 {
2517 TestHelper(src0, src1, cc);
2518 GetMasm()->Mov(ConvertTest(cc), VixlReg(dst), 0x1);
2519 GetMasm()->Mov(ConvertTest(cc).Negate(), VixlReg(dst), 0x0);
2520 }
2521
EncodeCmp(Reg dst,Reg src0,Reg src1,Condition cc)2522 void Aarch32Encoder::EncodeCmp(Reg dst, Reg src0, Reg src1, Condition cc)
2523 {
2524 if (src0.IsFloat()) {
2525 ASSERT(src1.IsFloat());
2526 ASSERT(cc == Condition::MI || cc == Condition::LT);
2527 GetMasm()->Vcmp(VixlVReg(src0), VixlVReg(src1));
2528 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
2529 } else {
2530 ASSERT(src0.IsScalar() && src1.IsScalar());
2531 ASSERT(cc == Condition::LO || cc == Condition::LT);
2532 if (src0.GetSize() <= WORD_SIZE && src1.GetSize() <= WORD_SIZE) {
2533 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
2534 } else {
2535 if (cc == Condition::LO) {
2536 GetMasm()->Cmp(VixlRegU(src0), VixlRegU(src1));
2537 GetMasm()->Cmp(Convert(Condition::EQ), VixlReg(src0), VixlReg(src1));
2538 } else if (cc == Condition::LT) {
2539 auto label_holder = static_cast<Aarch32LabelHolder *>(GetLabels());
2540 auto end_label = label_holder->CreateLabel();
2541 ScopedTmpRegU32 tmp_reg(this);
2542
2543 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
2544 GetMasm()->Sbcs(VixlReg(tmp_reg), VixlRegU(src0), VixlRegU(src1));
2545 GetMasm()->Mov(Convert(Condition::LT), VixlReg(dst), VixlImm(-1));
2546 GetMasm()->B(Convert(Condition::LT), label_holder->GetLabel(end_label));
2547
2548 GetMasm()->Cmp(VixlReg(src1), VixlReg(src0));
2549 GetMasm()->Sbcs(VixlReg(tmp_reg), VixlRegU(src1), VixlRegU(src0));
2550 GetMasm()->Mov(Convert(Condition::LT), VixlReg(dst), VixlImm(1));
2551 GetMasm()->Mov(Convert(Condition::EQ), VixlReg(dst), VixlImm(0));
2552
2553 label_holder->BindLabel(end_label);
2554 return;
2555 } else {
2556 UNREACHABLE();
2557 }
2558 }
2559 }
2560
2561 GetMasm()->Mov(Convert(Condition::EQ), VixlReg(dst), VixlImm(0x0));
2562 GetMasm()->Mov(Convert(Condition::NE), VixlReg(dst), VixlImm(0x1));
2563
2564 GetMasm()->Rsb(Convert(cc), VixlReg(dst), VixlReg(dst), VixlImm(0x0));
2565 }
2566
EncodeStackOverflowCheck(ssize_t offset)2567 void Aarch32Encoder::EncodeStackOverflowCheck(ssize_t offset)
2568 {
2569 ScopedTmpReg tmp(this);
2570 EncodeAdd(tmp, GetTarget().GetStackReg(), Imm(offset));
2571 EncodeLdr(tmp, false, MemRef(tmp));
2572 }
2573
EncodeSelect(Reg dst,Reg src0,Reg src1,Reg src2,Reg src3,Condition cc)2574 void Aarch32Encoder::EncodeSelect(Reg dst, Reg src0, Reg src1, Reg src2, Reg src3, Condition cc)
2575 {
2576 ASSERT(!src0.IsFloat() && !src1.IsFloat());
2577
2578 CompareHelper(src2, src3, &cc);
2579
2580 GetMasm()->Mov(Convert(cc), VixlReg(dst), VixlReg(src0));
2581 GetMasm()->Mov(Convert(cc).Negate(), VixlReg(dst), VixlReg(src1));
2582
2583 if (src0.GetSize() > WORD_SIZE || src1.GetSize() > WORD_SIZE) {
2584 GetMasm()->Mov(Convert(cc), VixlRegU(dst), VixlRegU(src0));
2585 GetMasm()->Mov(Convert(cc).Negate(), VixlRegU(dst), VixlRegU(src1));
2586 }
2587 }
2588
EncodeSelect(Reg dst,Reg src0,Reg src1,Reg src2,Imm imm,Condition cc)2589 void Aarch32Encoder::EncodeSelect(Reg dst, Reg src0, Reg src1, Reg src2, Imm imm, Condition cc)
2590 {
2591 ASSERT(!src0.IsFloat() && !src1.IsFloat() && !src2.IsFloat());
2592 auto value = GetIntValue(imm);
2593 if (value == 0) {
2594 switch (cc) {
2595 case Condition::LO:
2596 // LO is always false, select src1
2597 GetMasm()->Mov(VixlReg(dst), VixlReg(src1));
2598 if (src0.GetSize() > WORD_SIZE || src1.GetSize() > WORD_SIZE) {
2599 GetMasm()->Mov(VixlRegU(dst), VixlRegU(src1));
2600 }
2601 return;
2602 case Condition::HS:
2603 // HS is always true, select src0
2604 GetMasm()->Mov(VixlReg(dst), VixlReg(src0));
2605 if (src0.GetSize() > WORD_SIZE || src1.GetSize() > WORD_SIZE) {
2606 GetMasm()->Mov(VixlRegU(dst), VixlRegU(src0));
2607 }
2608 return;
2609 case Condition::LS:
2610 // LS is same as EQ
2611 cc = Condition::EQ;
2612 break;
2613 case Condition::HI:
2614 // HI is same as NE
2615 cc = Condition::NE;
2616 break;
2617 default:
2618 break;
2619 }
2620 CompareZeroHelper(src2, &cc);
2621 } else { // value != 0
2622 if (!CompareImmHelper(src2, imm, &cc)) {
2623 return;
2624 }
2625 }
2626
2627 GetMasm()->Mov(Convert(cc), VixlReg(dst), VixlReg(src0));
2628 GetMasm()->Mov(Convert(cc).Negate(), VixlReg(dst), VixlReg(src1));
2629
2630 if (src0.GetSize() > WORD_SIZE || src1.GetSize() > WORD_SIZE) {
2631 GetMasm()->Mov(Convert(cc), VixlRegU(dst), VixlRegU(src0));
2632 GetMasm()->Mov(Convert(cc).Negate(), VixlRegU(dst), VixlRegU(src1));
2633 }
2634 }
2635
EncodeSelectTest(Reg dst,Reg src0,Reg src1,Reg src2,Reg src3,Condition cc)2636 void Aarch32Encoder::EncodeSelectTest(Reg dst, Reg src0, Reg src1, Reg src2, Reg src3, Condition cc)
2637 {
2638 ASSERT(!src0.IsFloat() && !src1.IsFloat() && !src2.IsFloat());
2639
2640 TestHelper(src2, src3, cc);
2641
2642 GetMasm()->Mov(ConvertTest(cc), VixlReg(dst), VixlReg(src0));
2643 GetMasm()->Mov(ConvertTest(cc).Negate(), VixlReg(dst), VixlReg(src1));
2644
2645 if (src0.GetSize() > WORD_SIZE || src1.GetSize() > WORD_SIZE) {
2646 GetMasm()->Mov(ConvertTest(cc), VixlRegU(dst), VixlRegU(src0));
2647 GetMasm()->Mov(ConvertTest(cc).Negate(), VixlRegU(dst), VixlRegU(src1));
2648 }
2649 }
2650
EncodeSelectTest(Reg dst,Reg src0,Reg src1,Reg src2,Imm imm,Condition cc)2651 void Aarch32Encoder::EncodeSelectTest(Reg dst, Reg src0, Reg src1, Reg src2, Imm imm, Condition cc)
2652 {
2653 ASSERT(!src0.IsFloat() && !src1.IsFloat() && !src2.IsFloat());
2654
2655 TestImmHelper(src2, imm, cc);
2656 GetMasm()->Mov(ConvertTest(cc), VixlReg(dst), VixlReg(src0));
2657 GetMasm()->Mov(ConvertTest(cc).Negate(), VixlReg(dst), VixlReg(src1));
2658
2659 if (src0.GetSize() > WORD_SIZE || src1.GetSize() > WORD_SIZE) {
2660 GetMasm()->Mov(ConvertTest(cc), VixlRegU(dst), VixlRegU(src0));
2661 GetMasm()->Mov(ConvertTest(cc).Negate(), VixlRegU(dst), VixlRegU(src1));
2662 }
2663 }
2664
CanEncodeImmAddSubCmp(int64_t imm,uint32_t size,bool signed_compare)2665 bool Aarch32Encoder::CanEncodeImmAddSubCmp(int64_t imm, uint32_t size, bool signed_compare)
2666 {
2667 if (imm == INT64_MIN) {
2668 return false;
2669 }
2670 if (imm < 0) {
2671 imm = -imm;
2672 if (size > WORD_SIZE && signed_compare) {
2673 return false;
2674 }
2675 }
2676 // We don't support 64-bit immediate, even when both higher and lower parts are legal immediates
2677 if (imm > UINT32_MAX) {
2678 return false;
2679 }
2680 return vixl::aarch32::ImmediateA32::IsImmediateA32(imm);
2681 }
2682
CanEncodeImmLogical(uint64_t imm,uint32_t size)2683 bool Aarch32Encoder::CanEncodeImmLogical(uint64_t imm, uint32_t size)
2684 {
2685 // NOLINTNEXTLINE(hicpp-signed-bitwise)
2686 uint64_t high = imm >> WORD_SIZE;
2687 if (size == DOUBLE_WORD_SIZE) {
2688 if (high != 0U && high != UINT32_MAX) {
2689 return false;
2690 }
2691 }
2692 return vixl::aarch32::ImmediateA32::IsImmediateA32(imm);
2693 }
2694
2695 using vixl::aarch32::MemOperand;
2696 using vixl::aarch32::Register;
2697 using vixl::aarch32::SRegister;
2698
2699 template <bool is_store>
LoadStoreRegisters(RegMask registers,bool is_fp,int32_t slot,Reg base,RegMask mask)2700 void Aarch32Encoder::LoadStoreRegisters(RegMask registers, bool is_fp, int32_t slot, Reg base, RegMask mask)
2701 {
2702 if (registers.none()) {
2703 return;
2704 }
2705
2706 vixl::aarch32::Register base_reg = VixlReg(base);
2707 int32_t max_offset = (slot + helpers::ToSigned(registers.GetMaxRegister())) * WORD_SIZE_BYTE;
2708
2709 ScopedTmpRegU32 tmp_reg(this);
2710 auto tmp = VixlReg(tmp_reg);
2711 // Construct single add for big offset
2712 if (is_fp) {
2713 if ((max_offset < -VMEM_OFFSET) || (max_offset > VMEM_OFFSET)) {
2714 GetMasm()->Add(tmp, base_reg, VixlImm(slot * WORD_SIZE_BYTE));
2715 slot = 0;
2716 base_reg = tmp;
2717 }
2718 } else {
2719 if ((max_offset < -MEM_BIG_OFFSET) || (max_offset > MEM_BIG_OFFSET)) {
2720 GetMasm()->Add(tmp, base_reg, VixlImm(slot * WORD_SIZE_BYTE));
2721 slot = 0;
2722 base_reg = tmp;
2723 }
2724 }
2725 bool has_mask = mask.any();
2726 int32_t index = has_mask ? static_cast<int32_t>(mask.GetMinRegister()) : 0;
2727 slot -= index;
2728 for (size_t i = index; i < registers.size(); i++) {
2729 if (has_mask) {
2730 if (!mask.test(i)) {
2731 continue;
2732 }
2733 index++;
2734 }
2735 if (!registers.test(i)) {
2736 continue;
2737 }
2738
2739 if (!has_mask) {
2740 index++;
2741 }
2742 auto mem = MemOperand(base_reg, (slot + index - 1) * WORD_SIZE_BYTE);
2743 if (is_fp) {
2744 auto reg = vixl::aarch32::SRegister(i);
2745 if constexpr (is_store) { // NOLINT
2746 GetMasm()->Vstr(reg, mem);
2747 } else { // NOLINT
2748 GetMasm()->Vldr(reg, mem);
2749 }
2750 } else {
2751 auto reg = vixl::aarch32::Register(i);
2752 if constexpr (is_store) { // NOLINT
2753 GetMasm()->Str(reg, mem);
2754 } else { // NOLINT
2755 GetMasm()->Ldr(reg, mem);
2756 }
2757 }
2758 }
2759 }
2760
2761 template <bool is_store>
LoadStoreRegisters(RegMask registers,ssize_t slot,size_t start_reg,bool is_fp)2762 void Aarch32Encoder::LoadStoreRegisters(RegMask registers, ssize_t slot, size_t start_reg, bool is_fp)
2763 {
2764 if (registers.none()) {
2765 return;
2766 }
2767 int32_t last_reg = registers.size() - 1;
2768 for (; last_reg >= 0; --last_reg) {
2769 if (registers.test(last_reg)) {
2770 break;
2771 }
2772 }
2773 vixl::aarch32::Register base_reg = vixl::aarch32::sp;
2774 auto max_offset = (slot + last_reg) * WORD_SIZE_BYTE;
2775 ScopedTmpRegU32 tmp_reg(this);
2776 auto tmp = VixlReg(tmp_reg);
2777 // Construct single add for big offset
2778 if (is_fp) {
2779 if ((max_offset < -VMEM_OFFSET) || (max_offset > VMEM_OFFSET)) {
2780 GetMasm()->Add(tmp, base_reg, VixlImm(slot * WORD_SIZE_BYTE));
2781 slot = 0;
2782 base_reg = tmp;
2783 }
2784 } else {
2785 if ((max_offset < -MEM_BIG_OFFSET) || (max_offset > MEM_BIG_OFFSET)) {
2786 GetMasm()->Add(tmp, base_reg, VixlImm(slot * WORD_SIZE_BYTE));
2787 slot = 0;
2788 base_reg = tmp;
2789 }
2790 }
2791 for (auto i = start_reg; i < registers.size(); i++) {
2792 if (!registers.test(i)) {
2793 continue;
2794 }
2795 auto mem = MemOperand(base_reg, (slot + i - start_reg) * WORD_SIZE_BYTE);
2796 if (is_fp) {
2797 auto reg = vixl::aarch32::SRegister(i);
2798 if constexpr (is_store) { // NOLINT
2799 GetMasm()->Vstr(reg, mem);
2800 } else { // NOLINT
2801 GetMasm()->Vldr(reg, mem);
2802 }
2803 } else {
2804 auto reg = vixl::aarch32::Register(i);
2805 if constexpr (is_store) { // NOLINT
2806 GetMasm()->Str(reg, mem);
2807 } else { // NOLINT
2808 GetMasm()->Ldr(reg, mem);
2809 }
2810 }
2811 }
2812 }
2813
PushRegisters(RegMask registers,bool is_fp,bool align)2814 void Aarch32Encoder::PushRegisters(RegMask registers, bool is_fp, bool align)
2815 {
2816 (void)registers;
2817 (void)is_fp;
2818 (void)align;
2819 // TODO(msherstennikov): Implement
2820 }
2821
PopRegisters(RegMask registers,bool is_fp,bool align)2822 void Aarch32Encoder::PopRegisters(RegMask registers, bool is_fp, bool align)
2823 {
2824 (void)registers;
2825 (void)is_fp;
2826 (void)align;
2827 // TODO(msherstennikov): Implement
2828 }
2829
DisasmInstr(std::ostream & stream,size_t pc,ssize_t code_offset) const2830 size_t Aarch32Encoder::DisasmInstr(std::ostream &stream, size_t pc, ssize_t code_offset) const
2831 {
2832 auto addr = GetMasm()->GetBuffer()->GetOffsetAddress<const uint32_t *>(pc);
2833 // Display pc is seted, because disassembler use pc
2834 // for upper bits (e.g. 0x40000000), when print one instruction.
2835 if (code_offset < 0) {
2836 vixl::aarch32::PrintDisassembler disasm(GetAllocator(), stream);
2837 disasm.DisassembleA32Buffer(addr, vixl::aarch32::k32BitT32InstructionSizeInBytes);
2838 } else {
2839 const uint64_t DISPLAY_PC = 0x10000000;
2840 vixl::aarch32::PrintDisassembler disasm(GetAllocator(), stream, DISPLAY_PC + pc + code_offset);
2841 disasm.DisassembleA32Buffer(addr, vixl::aarch32::k32BitT32InstructionSizeInBytes);
2842
2843 stream << std::setfill(' ');
2844 }
2845 return pc + vixl::aarch32::k32BitT32InstructionSizeInBytes;
2846 }
2847 } // namespace panda::compiler::aarch32
2848