1 /*
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 /*
16 Encoder (implementation of math and mem Low-level emitters)
17 */
18
19 #include <aarch32/disasm-aarch32.h>
20 #include "compiler/optimizer/code_generator/relocations.h"
21 #include "encode.h"
22 #include "scoped_tmp_reg.h"
23 #include "operands.h"
24 #include "target/aarch32/target.h"
25 #include "lib_helpers.inl"
26
27 #ifndef PANDA_TARGET_MACOS
28 #include "elf.h"
29 #endif // PANDA_TARGET_MACOS
30
31 #include <iomanip>
32
33 namespace ark::compiler::aarch32 {
34
35 /// Converters
Convert(const Condition cc)36 static vixl::aarch32::Condition Convert(const Condition cc)
37 {
38 switch (cc) {
39 case Condition::EQ:
40 return vixl::aarch32::eq;
41 case Condition::NE:
42 return vixl::aarch32::ne;
43 case Condition::LT:
44 return vixl::aarch32::lt;
45 case Condition::GT:
46 return vixl::aarch32::gt;
47 case Condition::LE:
48 return vixl::aarch32::le;
49 case Condition::GE:
50 return vixl::aarch32::ge;
51 case Condition::LO:
52 return vixl::aarch32::lo;
53 case Condition::LS:
54 return vixl::aarch32::ls;
55 case Condition::HI:
56 return vixl::aarch32::hi;
57 case Condition::HS:
58 return vixl::aarch32::hs;
59 // NOTE(igorban) : Remove them
60 case Condition::MI:
61 return vixl::aarch32::mi;
62 case Condition::PL:
63 return vixl::aarch32::pl;
64 case Condition::VS:
65 return vixl::aarch32::vs;
66 case Condition::VC:
67 return vixl::aarch32::vc;
68 case Condition::AL:
69 return vixl::aarch32::al;
70 default:
71 UNREACHABLE();
72 return vixl::aarch32::eq;
73 }
74 }
75
76 /// Converters
ConvertTest(const Condition cc)77 static vixl::aarch32::Condition ConvertTest(const Condition cc)
78 {
79 ASSERT(cc == Condition::TST_EQ || cc == Condition::TST_NE);
80 return cc == Condition::TST_EQ ? vixl::aarch32::eq : vixl::aarch32::ne;
81 }
82
IsConditionSigned(Condition cc)83 static bool IsConditionSigned(Condition cc)
84 {
85 switch (cc) {
86 case Condition::LT:
87 case Condition::LE:
88 case Condition::GT:
89 case Condition::GE:
90 return true;
91
92 default:
93 return false;
94 }
95 }
96
Convert(const TypeInfo info,const bool isSigned=false)97 static vixl::aarch32::DataType Convert(const TypeInfo info, const bool isSigned = false)
98 {
99 if (!isSigned) {
100 if (info.IsFloat()) {
101 if (info.GetSize() == WORD_SIZE) {
102 return vixl::aarch32::DataType(vixl::aarch32::DataTypeValue::F32);
103 }
104 ASSERT(info.GetSize() == DOUBLE_WORD_SIZE);
105 return vixl::aarch32::DataType(vixl::aarch32::DataTypeValue::F64);
106 }
107 switch (info.GetSize()) {
108 case BYTE_SIZE:
109 return vixl::aarch32::DataType(vixl::aarch32::DataTypeValue::I8);
110 case HALF_SIZE:
111 return vixl::aarch32::DataType(vixl::aarch32::DataTypeValue::I16);
112 case WORD_SIZE:
113 return vixl::aarch32::DataType(vixl::aarch32::DataTypeValue::I32);
114 case DOUBLE_WORD_SIZE:
115 return vixl::aarch32::DataType(vixl::aarch32::DataTypeValue::I64);
116 default:
117 break;
118 }
119 }
120 ASSERT(!info.IsFloat());
121
122 switch (info.GetSize()) {
123 case BYTE_SIZE:
124 return vixl::aarch32::DataType(vixl::aarch32::DataTypeValue::S8);
125 case HALF_SIZE:
126 return vixl::aarch32::DataType(vixl::aarch32::DataTypeValue::S16);
127 case WORD_SIZE:
128 return vixl::aarch32::DataType(vixl::aarch32::DataTypeValue::S32);
129 case DOUBLE_WORD_SIZE:
130 return vixl::aarch32::DataType(vixl::aarch32::DataTypeValue::S64);
131 default:
132 break;
133 }
134 return vixl::aarch32::DataType(vixl::aarch32::DataTypeValue::kDataTypeValueInvalid);
135 }
136
AcquireScratchRegister(TypeInfo type)137 Reg Aarch32Encoder::AcquireScratchRegister(TypeInfo type)
138 {
139 ASSERT(GetMasm()->GetCurrentScratchRegisterScope() == nullptr);
140 if (type.IsFloat()) {
141 if (type == FLOAT32_TYPE) {
142 auto reg = GetMasm()->GetScratchVRegisterList()->GetFirstAvailableSRegister();
143 ASSERT(reg.IsValid());
144 GetMasm()->GetScratchVRegisterList()->Remove(reg);
145 return Reg(reg.GetCode(), type);
146 }
147 // Get 2 float registers instead one double - to save agreement about hi-regs in VixlVReg
148 auto reg1 = GetMasm()->GetScratchVRegisterList()->GetFirstAvailableSRegister();
149 auto reg2 = GetMasm()->GetScratchVRegisterList()->GetFirstAvailableSRegister();
150 ASSERT(reg1.IsValid());
151 ASSERT(reg2.IsValid());
152 GetMasm()->GetScratchVRegisterList()->Remove(reg1);
153 GetMasm()->GetScratchVRegisterList()->Remove(reg2);
154 return Reg(reg1.GetCode(), type);
155 }
156 auto reg = GetMasm()->GetScratchRegisterList()->GetFirstAvailableRegister();
157 ASSERT(reg.IsValid());
158 GetMasm()->GetScratchRegisterList()->Remove(reg);
159 return Reg(reg.GetCode(), type);
160 }
161
GetLabels() const162 LabelHolder *Aarch32Encoder::GetLabels() const
163 {
164 return labels_;
165 }
166
GetTarget()167 constexpr auto Aarch32Encoder::GetTarget()
168 {
169 return ark::compiler::Target(Arch::AARCH32);
170 }
171
AcquireScratchRegister(Reg reg)172 void Aarch32Encoder::AcquireScratchRegister(Reg reg)
173 {
174 if (reg == GetTarget().GetLinkReg()) {
175 ASSERT_PRINT(!lrAcquired_, "Trying to acquire LR, which hasn't been released before");
176 lrAcquired_ = true;
177 } else if (reg.IsFloat()) {
178 ASSERT(GetMasm()->GetScratchVRegisterList()->IncludesAliasOf(vixl::aarch32::SRegister(reg.GetId())));
179 GetMasm()->GetScratchVRegisterList()->Remove(vixl::aarch32::SRegister(reg.GetId()));
180 } else {
181 ASSERT(GetMasm()->GetScratchRegisterList()->Includes(vixl::aarch32::Register(reg.GetId())));
182 GetMasm()->GetScratchRegisterList()->Remove(vixl::aarch32::Register(reg.GetId()));
183 }
184 }
185
ReleaseScratchRegister(Reg reg)186 void Aarch32Encoder::ReleaseScratchRegister(Reg reg)
187 {
188 if (reg == GetTarget().GetLinkReg()) {
189 ASSERT_PRINT(lrAcquired_, "Trying to release LR, which hasn't been acquired before");
190 lrAcquired_ = false;
191 } else if (reg.IsFloat()) {
192 GetMasm()->GetScratchVRegisterList()->Combine(vixl::aarch32::SRegister(reg.GetId()));
193 } else {
194 GetMasm()->GetScratchRegisterList()->Combine(vixl::aarch32::Register(reg.GetId()));
195 }
196 }
197
IsScratchRegisterReleased(Reg reg) const198 bool Aarch32Encoder::IsScratchRegisterReleased(Reg reg) const
199 {
200 if (reg == GetTarget().GetLinkReg()) {
201 return !lrAcquired_;
202 }
203 if (reg.IsFloat()) {
204 return GetMasm()->GetScratchVRegisterList()->IncludesAliasOf(vixl::aarch32::SRegister(reg.GetId()));
205 }
206 return GetMasm()->GetScratchRegisterList()->Includes(vixl::aarch32::Register(reg.GetId()));
207 }
208
SetRegister(RegMask * mask,VRegMask * vmask,Reg reg,bool val) const209 void Aarch32Encoder::SetRegister(RegMask *mask, VRegMask *vmask, Reg reg, bool val) const
210 {
211 if (!reg.IsValid()) {
212 return;
213 }
214 if (reg.IsScalar()) {
215 ASSERT(mask != nullptr);
216 mask->set(reg.GetId(), val);
217 if (reg.GetSize() > WORD_SIZE) {
218 mask->set(reg.GetId() + 1, val);
219 }
220 } else {
221 ASSERT(vmask != nullptr);
222 ASSERT(reg.IsFloat());
223 vmask->set(reg.GetId(), val);
224 if (reg.GetSize() > WORD_SIZE) {
225 vmask->set(reg.GetId() + 1, val);
226 }
227 }
228 }
ConvertMem(MemRef mem)229 vixl::aarch32::MemOperand Aarch32Encoder::ConvertMem(MemRef mem)
230 {
231 bool hasIndex = mem.HasIndex();
232 bool hasShift = mem.HasScale();
233 bool hasOffset = mem.HasDisp();
234 auto baseReg = VixlReg(mem.GetBase());
235 if (hasIndex) {
236 // MemRef with index and offser isn't supported
237 ASSERT(!hasOffset);
238 auto indexReg = mem.GetIndex();
239 if (hasShift) {
240 auto shift = mem.GetScale();
241 return vixl::aarch32::MemOperand(baseReg, VixlReg(indexReg), vixl::aarch32::LSL, shift);
242 }
243 return vixl::aarch32::MemOperand(baseReg, VixlReg(indexReg));
244 }
245 if (hasOffset) {
246 auto offset = mem.GetDisp();
247 return vixl::aarch32::MemOperand(baseReg, offset);
248 }
249 return vixl::aarch32::MemOperand(baseReg);
250 }
251
GetCursorOffset() const252 size_t Aarch32Encoder::GetCursorOffset() const
253 {
254 return GetMasm()->GetBuffer()->GetCursorOffset();
255 }
256
SetCursorOffset(size_t offset)257 void Aarch32Encoder::SetCursorOffset(size_t offset)
258 {
259 GetMasm()->GetBuffer()->Rewind(offset);
260 }
261
GetScratchRegistersMask() const262 RegMask Aarch32Encoder::GetScratchRegistersMask() const
263 {
264 return RegMask(GetMasm()->GetScratchRegisterList()->GetList());
265 }
266
GetScratchFpRegistersMask() const267 RegMask Aarch32Encoder::GetScratchFpRegistersMask() const
268 {
269 return RegMask(GetMasm()->GetScratchVRegisterList()->GetList());
270 }
271
GetAvailableScratchRegisters() const272 RegMask Aarch32Encoder::GetAvailableScratchRegisters() const
273 {
274 return RegMask(GetMasm()->GetScratchRegisterList()->GetList());
275 }
276
GetAvailableScratchFpRegisters() const277 VRegMask Aarch32Encoder::GetAvailableScratchFpRegisters() const
278 {
279 return VRegMask(GetMasm()->GetScratchVRegisterList()->GetList());
280 }
281
CreateLabel()282 LabelHolder::LabelId Aarch32LabelHolder::CreateLabel()
283 {
284 ++id_;
285 auto allocator = GetEncoder()->GetAllocator();
286 auto *label = allocator->New<LabelType>(allocator);
287 labels_.push_back(label);
288 ASSERT(labels_.size() == id_);
289 return id_ - 1;
290 }
291
CreateLabels(LabelId size)292 void Aarch32LabelHolder::CreateLabels(LabelId size)
293 {
294 for (LabelId i = 0; i <= size; ++i) {
295 CreateLabel();
296 }
297 }
298
BindLabel(LabelId id)299 void Aarch32LabelHolder::BindLabel(LabelId id)
300 {
301 static_cast<Aarch32Encoder *>(GetEncoder())->GetMasm()->Bind(labels_[id]);
302 }
303
GetLabel(LabelId id)304 Aarch32LabelHolder::LabelType *Aarch32LabelHolder::GetLabel(LabelId id)
305 {
306 ASSERT(labels_.size() > id);
307 return labels_[id];
308 }
309
Size()310 LabelHolder::LabelId Aarch32LabelHolder::Size()
311 {
312 return labels_.size();
313 }
314
GetRefType()315 TypeInfo Aarch32Encoder::GetRefType()
316 {
317 return INT32_TYPE;
318 }
319
BufferData() const320 void *Aarch32Encoder::BufferData() const
321 {
322 return GetMasm()->GetBuffer()->GetStartAddress<void *>();
323 }
324
BufferSize() const325 size_t Aarch32Encoder::BufferSize() const
326 {
327 return GetMasm()->GetBuffer()->GetSizeInBytes();
328 }
329
SaveRegisters(RegMask registers,ssize_t slot,size_t startReg,bool isFp)330 void Aarch32Encoder::SaveRegisters(RegMask registers, ssize_t slot, size_t startReg, bool isFp)
331 {
332 LoadStoreRegisters<true>(registers, slot, startReg, isFp);
333 }
LoadRegisters(RegMask registers,ssize_t slot,size_t startReg,bool isFp)334 void Aarch32Encoder::LoadRegisters(RegMask registers, ssize_t slot, size_t startReg, bool isFp)
335 {
336 LoadStoreRegisters<false>(registers, slot, startReg, isFp);
337 }
338
SaveRegisters(RegMask registers,bool isFp,ssize_t slot,Reg base,RegMask mask)339 void Aarch32Encoder::SaveRegisters(RegMask registers, bool isFp, ssize_t slot, Reg base, RegMask mask)
340 {
341 LoadStoreRegisters<true>(registers, isFp, slot, base, mask);
342 }
LoadRegisters(RegMask registers,bool isFp,ssize_t slot,Reg base,RegMask mask)343 void Aarch32Encoder::LoadRegisters(RegMask registers, bool isFp, ssize_t slot, Reg base, RegMask mask)
344 {
345 LoadStoreRegisters<false>(registers, isFp, slot, base, mask);
346 }
347
GetMasm() const348 vixl::aarch32::MacroAssembler *Aarch32Encoder::GetMasm() const
349 {
350 ASSERT(masm_ != nullptr);
351 return masm_;
352 }
353
GetLabelAddress(LabelHolder::LabelId label)354 size_t Aarch32Encoder::GetLabelAddress(LabelHolder::LabelId label)
355 {
356 auto plabel = labels_->GetLabel(label);
357 ASSERT(plabel->IsBound());
358 return GetMasm()->GetBuffer()->GetOffsetAddress<size_t>(plabel->GetLocation());
359 }
360
LabelHasLinks(LabelHolder::LabelId label)361 bool Aarch32Encoder::LabelHasLinks(LabelHolder::LabelId label)
362 {
363 auto plabel = labels_->GetLabel(label);
364 return plabel->IsReferenced();
365 }
366
IsValid() const367 bool Aarch32Encoder::IsValid() const
368 {
369 return true;
370 }
371
SetMaxAllocatedBytes(size_t size)372 void Aarch32Encoder::SetMaxAllocatedBytes(size_t size)
373 {
374 GetMasm()->GetBuffer()->SetMmapMaxBytes(size);
375 }
Aarch32Encoder(ArenaAllocator * allocator)376 Aarch32Encoder::Aarch32Encoder(ArenaAllocator *allocator) : Encoder(allocator, Arch::AARCH32)
377 {
378 labels_ = allocator->New<Aarch32LabelHolder>(this);
379 if (labels_ == nullptr) {
380 SetFalseResult();
381 }
382 EnableLrAsTempReg(true);
383 }
384
~Aarch32Encoder()385 Aarch32Encoder::~Aarch32Encoder()
386 {
387 auto labels = static_cast<Aarch32LabelHolder *>(GetLabels())->labels_;
388 for (auto label : labels) {
389 label->~Label();
390 }
391 if (masm_ != nullptr) {
392 masm_->~MacroAssembler();
393 masm_ = nullptr;
394 }
395 }
396
InitMasm()397 bool Aarch32Encoder::InitMasm()
398 {
399 if (masm_ == nullptr) {
400 auto allocator = GetAllocator();
401
402 // Initialize Masm
403 masm_ = allocator->New<vixl::aarch32::MacroAssembler>(allocator);
404 if (masm_ == nullptr || !masm_->IsValid()) {
405 SetFalseResult();
406 return false;
407 }
408
409 ASSERT(masm_);
410 for (auto regCode : AARCH32_TMP_REG) {
411 masm_->GetScratchRegisterList()->Combine(vixl::aarch32::Register(regCode));
412 }
413 for (auto vregCode : AARCH32_TMP_VREG) {
414 masm_->GetScratchVRegisterList()->Combine(vixl::aarch32::SRegister(vregCode));
415 }
416
417 // Make sure that the compiler uses the same scratch registers as the assembler
418 CHECK_EQ(RegMask(masm_->GetScratchRegisterList()->GetList()), GetTarget().GetTempRegsMask());
419 CHECK_EQ(RegMask(masm_->GetScratchVRegisterList()->GetList()), GetTarget().GetTempVRegsMask());
420 }
421 return true;
422 }
423
Finalize()424 void Aarch32Encoder::Finalize()
425 {
426 GetMasm()->FinalizeCode();
427 }
428
EncodeJump(LabelHolder::LabelId id)429 void Aarch32Encoder::EncodeJump(LabelHolder::LabelId id)
430 {
431 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
432 GetMasm()->B(label);
433 }
434
EncodeJump(LabelHolder::LabelId id,Reg src0,Reg src1,Condition cc)435 void Aarch32Encoder::EncodeJump(LabelHolder::LabelId id, Reg src0, Reg src1, Condition cc)
436 {
437 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
438 CompareHelper(src0, src1, &cc);
439 GetMasm()->B(Convert(cc), label);
440 }
441
EncodeJumpTest(LabelHolder::LabelId id,Reg src0,Reg src1,Condition cc)442 void Aarch32Encoder::EncodeJumpTest(LabelHolder::LabelId id, Reg src0, Reg src1, Condition cc)
443 {
444 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
445 TestHelper(src0, src1, cc);
446 GetMasm()->B(ConvertTest(cc), label);
447 }
448
EncodeBitTestAndBranch(LabelHolder::LabelId id,Reg reg,uint32_t bitPos,bool bitValue)449 void Aarch32Encoder::EncodeBitTestAndBranch(LabelHolder::LabelId id, Reg reg, uint32_t bitPos, bool bitValue)
450 {
451 ASSERT(reg.IsScalar() && reg.GetSize() > bitPos);
452 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
453 if (reg.GetSize() == DOUBLE_WORD_SIZE) {
454 if (bitPos < WORD_SIZE) {
455 GetMasm()->tst(VixlReg(reg), VixlImm(1U << bitPos));
456 } else {
457 GetMasm()->tst(VixlRegU(reg), VixlImm(1U << (bitPos - WORD_SIZE)));
458 }
459 } else {
460 GetMasm()->tst(VixlReg(reg), VixlImm(1U << bitPos));
461 }
462 if (bitValue) {
463 GetMasm()->B(Convert(Condition::NE), label);
464 } else {
465 GetMasm()->B(Convert(Condition::EQ), label);
466 }
467 }
468
CompareImmHelper(Reg src,int64_t imm,Condition * cc)469 bool Aarch32Encoder::CompareImmHelper(Reg src, int64_t imm, Condition *cc)
470 {
471 ASSERT(src.IsScalar());
472 ASSERT(imm != 0);
473 ASSERT(-static_cast<int64_t>(UINT32_MAX) <= imm && imm <= UINT32_MAX);
474 ASSERT(CanEncodeImmAddSubCmp(imm, src.GetSize(), IsConditionSigned(*cc)));
475
476 return imm < 0 ? CompareNegImmHelper(src, imm, cc) : ComparePosImmHelper(src, imm, cc);
477 }
478
TestImmHelper(Reg src,Imm imm,Condition cc)479 void Aarch32Encoder::TestImmHelper(Reg src, Imm imm, [[maybe_unused]] Condition cc)
480 {
481 auto value = imm.GetAsInt();
482 ASSERT(src.IsScalar());
483 ASSERT(cc == Condition::TST_EQ || cc == Condition::TST_NE);
484 ASSERT(CanEncodeImmLogical(value, src.GetSize()));
485
486 if (src.GetSize() <= WORD_SIZE) {
487 GetMasm()->Tst(VixlReg(src), VixlImm(value));
488 } else {
489 GetMasm()->Tst(VixlRegU(src), VixlImm(0x0));
490 GetMasm()->Tst(Convert(Condition::EQ), VixlReg(src), VixlImm(value));
491 }
492 }
493
CompareNegImmHelper(Reg src,int64_t value,const Condition * cc)494 bool Aarch32Encoder::CompareNegImmHelper(Reg src, int64_t value, const Condition *cc)
495 {
496 if (src.GetSize() <= WORD_SIZE) {
497 GetMasm()->Cmn(VixlReg(src), VixlImm(-value));
498 } else {
499 if (!IsConditionSigned(*cc)) {
500 GetMasm()->Cmn(VixlRegU(src), VixlImm(0x1));
501 GetMasm()->Cmn(Convert(Condition::EQ), VixlReg(src), VixlImm(-value));
502 } else {
503 // There are no effective implementation in this case
504 // Can't get here because of logic behind CanEncodeImmAddSubCmp
505 UNREACHABLE();
506 SetFalseResult();
507 return false;
508 }
509 }
510 return true;
511 }
512
ComparePosImmHelper(Reg src,int64_t value,Condition * cc)513 bool Aarch32Encoder::ComparePosImmHelper(Reg src, int64_t value, Condition *cc)
514 {
515 if (src.GetSize() <= WORD_SIZE) {
516 GetMasm()->Cmp(VixlReg(src), VixlImm(value));
517 } else {
518 if (!IsConditionSigned(*cc)) {
519 GetMasm()->Cmp(VixlRegU(src), VixlImm(0x0));
520 GetMasm()->Cmp(Convert(Condition::EQ), VixlReg(src), VixlImm(value));
521 } else {
522 bool swap = false;
523 switch (*cc) {
524 case Condition::GT:
525 swap = true;
526 *cc = Condition::LT;
527 break;
528 case Condition::LE:
529 swap = true;
530 *cc = Condition::GE;
531 break;
532 case Condition::GE:
533 case Condition::LT:
534 break;
535 default:
536 UNREACHABLE();
537 }
538
539 ScopedTmpRegU32 tmpReg(this);
540 if (swap) {
541 GetMasm()->Rsbs(VixlReg(tmpReg), VixlReg(src), VixlImm(value));
542 GetMasm()->Rscs(VixlReg(tmpReg), VixlRegU(src), vixl::aarch32::Operand(0x0));
543 } else {
544 GetMasm()->Cmp(VixlReg(src), VixlImm(value));
545 GetMasm()->Sbcs(VixlReg(tmpReg), VixlRegU(src), vixl::aarch32::Operand(0x0));
546 }
547 }
548 }
549 return true;
550 }
551
EncodeJump(LabelHolder::LabelId id,Reg src,Imm imm,Condition cc)552 void Aarch32Encoder::EncodeJump(LabelHolder::LabelId id, Reg src, Imm imm, Condition cc)
553 {
554 auto value = imm.GetAsInt();
555 if (value == 0) {
556 EncodeJump(id, src, cc);
557 return;
558 }
559
560 if (!CompareImmHelper(src, value, &cc)) {
561 return;
562 }
563
564 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
565 GetMasm()->B(Convert(cc), label);
566 }
567
EncodeJumpTest(LabelHolder::LabelId id,Reg src,Imm imm,Condition cc)568 void Aarch32Encoder::EncodeJumpTest(LabelHolder::LabelId id, Reg src, Imm imm, Condition cc)
569 {
570 ASSERT(src.IsScalar());
571
572 TestImmHelper(src, imm, cc);
573 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
574 GetMasm()->B(ConvertTest(cc), label);
575 }
576
CompareZeroHelper(Reg src,Condition * cc)577 void Aarch32Encoder::CompareZeroHelper(Reg src, Condition *cc)
578 {
579 ASSERT(src.IsScalar());
580 if (src.GetSize() <= WORD_SIZE) {
581 GetMasm()->Cmp(VixlReg(src), vixl::aarch32::Operand(0x0));
582 } else {
583 ScopedTmpRegU32 tmpReg(this);
584 uint32_t imm = 0x0;
585
586 switch (*cc) {
587 case Condition::EQ:
588 case Condition::NE:
589 GetMasm()->Orrs(VixlReg(tmpReg), VixlReg(src), VixlRegU(src));
590 break;
591 case Condition::LE:
592 imm = 0x1;
593 *cc = Condition::LT;
594 /* fallthrough */
595 [[fallthrough]];
596 case Condition::LT:
597 GetMasm()->Cmp(VixlReg(src), vixl::aarch32::Operand(imm));
598 GetMasm()->Sbcs(VixlReg(tmpReg), VixlRegU(src), vixl::aarch32::Operand(0x0));
599 break;
600 case Condition::GT:
601 imm = 0x1;
602 *cc = Condition::GE;
603 /* fallthrough */
604 [[fallthrough]];
605 case Condition::GE:
606 GetMasm()->Cmp(VixlReg(src), vixl::aarch32::Operand(imm));
607 GetMasm()->Sbcs(VixlReg(tmpReg), VixlRegU(src), vixl::aarch32::Operand(0x0));
608 break;
609 default:
610 UNREACHABLE();
611 }
612 }
613 }
614
EncodeJump(LabelHolder::LabelId id,Reg src,Condition cc)615 void Aarch32Encoder::EncodeJump(LabelHolder::LabelId id, Reg src, Condition cc)
616 {
617 auto label = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(id);
618 ASSERT(src.IsScalar());
619
620 switch (cc) {
621 case Condition::LO:
622 // LO is always false
623 return;
624 case Condition::HS:
625 // HS is always true
626 GetMasm()->B(label);
627 return;
628 case Condition::LS:
629 // LS is same as EQ
630 cc = Condition::EQ;
631 break;
632 case Condition::HI:
633 // HI is same as NE
634 cc = Condition::NE;
635 break;
636 default:
637 break;
638 }
639
640 CompareZeroHelper(src, &cc);
641
642 GetMasm()->B(Convert(cc), label);
643 }
644
EncodeJump(Reg dst)645 void Aarch32Encoder::EncodeJump(Reg dst)
646 {
647 GetMasm()->Bx(VixlReg(dst));
648 }
649
EncodeJump(RelocationInfo * relocation)650 void Aarch32Encoder::EncodeJump([[maybe_unused]] RelocationInfo *relocation)
651 {
652 #ifdef PANDA_TARGET_MACOS
653 LOG(FATAL, COMPILER) << "Not supported in Macos build";
654 #else
655 auto buffer = GetMasm()->GetBuffer();
656 relocation->offset = GetCursorOffset();
657 relocation->addend = 0;
658 relocation->type = R_ARM_CALL;
659 static constexpr uint32_t CALL_WITH_ZERO_OFFSET = 0xeafffffe;
660 buffer->Emit32(CALL_WITH_ZERO_OFFSET);
661 #endif
662 }
663
EncodeNop()664 void Aarch32Encoder::EncodeNop()
665 {
666 GetMasm()->Nop();
667 }
668
MakeCall(compiler::RelocationInfo * relocation)669 void Aarch32Encoder::MakeCall([[maybe_unused]] compiler::RelocationInfo *relocation)
670 {
671 #ifdef PANDA_TARGET_MACOS
672 LOG(FATAL, COMPILER) << "Not supported in Macos build";
673 #else
674 auto buffer = GetMasm()->GetBuffer();
675 relocation->offset = GetCursorOffset();
676 relocation->addend = 0;
677 relocation->type = R_ARM_CALL;
678 static constexpr uint32_t CALL_WITH_ZERO_OFFSET = 0xebfffffe;
679 buffer->Emit32(CALL_WITH_ZERO_OFFSET);
680 #endif
681 }
682
MakeCall(const void * entryPoint)683 void Aarch32Encoder::MakeCall(const void *entryPoint)
684 {
685 ScopedTmpRegU32 tmpReg(this);
686
687 auto entry = static_cast<int32_t>(reinterpret_cast<int64_t>(entryPoint));
688 EncodeMov(tmpReg, Imm(entry));
689 GetMasm()->Blx(VixlReg(tmpReg));
690 }
691
MakeCall(MemRef entryPoint)692 void Aarch32Encoder::MakeCall(MemRef entryPoint)
693 {
694 ScopedTmpRegU32 tmpReg(this);
695
696 EncodeLdr(tmpReg, false, entryPoint);
697 GetMasm()->Blx(VixlReg(tmpReg));
698 }
699
MakeCall(Reg reg)700 void Aarch32Encoder::MakeCall(Reg reg)
701 {
702 GetMasm()->Blx(VixlReg(reg));
703 }
704
MakeCallAot(intptr_t offset)705 void Aarch32Encoder::MakeCallAot([[maybe_unused]] intptr_t offset)
706 {
707 // Unimplemented
708 SetFalseResult();
709 }
710
MakeCallByOffset(intptr_t offset)711 void Aarch32Encoder::MakeCallByOffset([[maybe_unused]] intptr_t offset)
712 {
713 // Unimplemented
714 SetFalseResult();
715 }
716
MakeLoadAotTable(intptr_t offset,Reg reg)717 void Aarch32Encoder::MakeLoadAotTable([[maybe_unused]] intptr_t offset, [[maybe_unused]] Reg reg)
718 {
719 // Unimplemented
720 SetFalseResult();
721 }
722
MakeLoadAotTableAddr(intptr_t offset,Reg addr,Reg val)723 void Aarch32Encoder::MakeLoadAotTableAddr([[maybe_unused]] intptr_t offset, [[maybe_unused]] Reg addr,
724 [[maybe_unused]] Reg val)
725 {
726 // Unimplemented
727 SetFalseResult();
728 }
729
EncodeReturn()730 void Aarch32Encoder::EncodeReturn()
731 {
732 GetMasm()->Bx(vixl::aarch32::lr);
733 }
734
EncodeAbort()735 void Aarch32Encoder::EncodeAbort()
736 {
737 GetMasm()->Udf(0);
738 }
739
EncodeMul(Reg dst,Reg src,Imm imm)740 void Aarch32Encoder::EncodeMul([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src, [[maybe_unused]] Imm imm)
741 {
742 SetFalseResult();
743 }
744
745 /**
746 * The function construct additional instruction for encode memory instructions and returns MemOperand for ldr/str
747 * LDR/STR with immediate offset(for A32)
748 * | mem type | offset size |
749 * | ---------- | ----------- |
750 * |word or byte|-4095 to 4095|
751 * | others | -255 to 255 |
752 *
753 * LDR/STR with register offset(for A32)
754 * | mem type | shift |
755 * | ---------- | ----------- |
756 * |word or byte| 0 to 31 |
757 * | others | -- |
758 *
759 * VLDR and VSTR has base and offset. The offset must be a multiple of 4, and lie in the range -1020 to +1020.
760 */
761 /* static */
IsNeedToPrepareMemLdS(MemRef mem,const TypeInfo & memType,bool isSigned)762 bool Aarch32Encoder::IsNeedToPrepareMemLdS(MemRef mem, const TypeInfo &memType, bool isSigned)
763 {
764 bool hasIndex = mem.HasIndex();
765 bool hasShift = mem.HasScale();
766 bool hasOffset = mem.HasDisp();
767 ASSERT(mem.HasBase());
768 // VLDR and VSTR has base and offset. The offset must be a multiple of 4, and lie in the range -1020 to +1020.
769 if (memType.IsFloat()) {
770 if (hasIndex) {
771 return true;
772 }
773 if (hasOffset) {
774 auto offset = mem.GetDisp();
775 constexpr int32_t IMM_4 = 4;
776 if (!((offset >= -VMEM_OFFSET) && (offset <= VMEM_OFFSET) && ((offset % IMM_4) == 0))) {
777 return true;
778 }
779 }
780 return false;
781 }
782 int32_t maxOffset = MEM_SMALL_OFFSET;
783 bool hasLsl = false;
784 if (!isSigned && (memType == INT32_TYPE || memType == INT8_TYPE)) {
785 hasLsl = true;
786 maxOffset = MEM_BIG_OFFSET;
787 }
788 // LDR/STR with register offset(for A32)
789 // | mem type | shift |
790 // | ---------- | ----------- |
791 // |word or byte| 0 to 31 |
792 // | others | -- |
793 //
794 if (hasIndex) {
795 // MemRef with index and offser isn't supported
796 ASSERT(!hasOffset);
797 if (hasShift) {
798 ASSERT(mem.GetScale() != 0);
799 [[maybe_unused]] constexpr int32_t MAX_SHIFT = 3;
800 ASSERT(mem.GetScale() <= MAX_SHIFT);
801 if (!hasLsl) {
802 return true;
803 }
804 }
805 return false;
806 }
807 // LDR/STR with immediate offset(for A32)
808 // | mem type | offset size |
809 // | ---------- | ----------- |
810 // |word or byte|-4095 to 4095|
811 // | others | -255 to 255 |
812 //
813 if (hasOffset) {
814 ASSERT(mem.GetDisp() != 0);
815 [[maybe_unused]] auto offset = mem.GetDisp();
816 if (!((offset >= -maxOffset) && (offset <= maxOffset))) {
817 return true;
818 }
819 }
820 return false;
821 }
822
823 /**
824 * The function construct additional instruction for encode memory instructions and returns MemOperand for ldr/str
825 * LDR/STR with immediate offset(for A32)
826 * | mem type | offset size |
827 * | ---------- | ----------- |
828 * |word or byte|-4095 to 4095|
829 * | others | -255 to 255 |
830 *
831 * LDR/STR with register offset(for A32)
832 * | mem type | shift |
833 * | ---------- | ----------- |
834 * |word or byte| 0 to 31 |
835 * | others | -- |
836 *
837 * VLDR and VSTR has base and offset. The offset must be a multiple of 4, and lie in the range -1020 to +1020.
838 */
PrepareMemLdS(MemRef mem,const TypeInfo & memType,vixl::aarch32::Register tmp,bool isSigned,bool copySp)839 vixl::aarch32::MemOperand Aarch32Encoder::PrepareMemLdS(MemRef mem, const TypeInfo &memType,
840 vixl::aarch32::Register tmp, bool isSigned, bool copySp)
841 {
842 bool hasIndex = mem.HasIndex();
843 bool hasShift = mem.HasScale();
844 bool hasOffset = mem.HasDisp();
845 ASSERT(mem.HasBase());
846 auto baseReg = VixlReg(mem.GetBase());
847 if (copySp) {
848 if (baseReg.IsSP()) {
849 GetMasm()->Mov(tmp, baseReg);
850 baseReg = tmp;
851 }
852 }
853 // VLDR and VSTR has base and offset. The offset must be a multiple of 4, and lie in the range -1020 to +1020.
854 if (memType.IsFloat()) {
855 return PrepareMemLdSForFloat(mem, tmp);
856 }
857 int32_t maxOffset = MEM_SMALL_OFFSET;
858 bool hasLsl = false;
859 if (!isSigned && (memType == INT32_TYPE || memType == INT8_TYPE)) {
860 hasLsl = true;
861 maxOffset = MEM_BIG_OFFSET;
862 }
863 // LDR/STR with register offset(for A32)
864 // | mem type | shift |
865 // | ---------- | ----------- |
866 // |word or byte| 0 to 31 |
867 // | others | -- |
868 //
869 if (hasIndex) {
870 // MemRef with index and offser isn't supported
871 ASSERT(!hasOffset);
872 auto indexReg = mem.GetIndex();
873 if (hasShift) {
874 ASSERT(mem.GetScale() != 0);
875 auto shift = mem.GetScale();
876 [[maybe_unused]] constexpr int32_t MAX_SHIFT = 3;
877 ASSERT(mem.GetScale() <= MAX_SHIFT);
878 if (hasLsl) {
879 return vixl::aarch32::MemOperand(baseReg, VixlReg(indexReg), vixl::aarch32::LSL, shift);
880 }
881 // from:
882 // mem: base, index, scale
883 // to:
884 // add tmp, base, index, scale
885 // mem tmp
886 GetMasm()->Add(tmp, baseReg, vixl::aarch32::Operand(VixlReg(indexReg), vixl::aarch32::LSL, shift));
887 return vixl::aarch32::MemOperand(tmp);
888 }
889 return vixl::aarch32::MemOperand(baseReg, VixlReg(indexReg));
890 }
891 // LDR/STR with immediate offset(for A32):
892 // | mem type | offset size |
893 // | ---------- | ----------- |
894 // |word or byte|-4095 to 4095|
895 // | others | -255 to 255 |
896 //
897 if (hasOffset) {
898 ASSERT(mem.GetDisp() != 0);
899 auto offset = mem.GetDisp();
900 if ((offset >= -maxOffset) && (offset <= maxOffset)) {
901 return vixl::aarch32::MemOperand(baseReg, offset);
902 }
903 // from:
904 // mem: base, offset
905 // to:
906 // add tmp, base, offset
907 // mem tmp
908 GetMasm()->Add(tmp, baseReg, VixlImm(offset));
909 baseReg = tmp;
910 }
911 return vixl::aarch32::MemOperand(baseReg);
912 }
913
PrepareMemLdSForFloat(MemRef mem,vixl::aarch32::Register tmp)914 vixl::aarch32::MemOperand Aarch32Encoder::PrepareMemLdSForFloat(MemRef mem, vixl::aarch32::Register tmp)
915 {
916 bool hasIndex = mem.HasIndex();
917 bool hasShift = mem.HasScale();
918 bool hasOffset = mem.HasDisp();
919 auto baseReg = VixlReg(mem.GetBase());
920 if (hasIndex) {
921 auto indexReg = mem.GetIndex();
922 auto scale = mem.GetScale();
923 // from:
924 // vmem: base, index, scale, offset
925 // to:
926 // add tmp, base, index, scale
927 // vmem tmp, offset
928 if (hasShift) {
929 ASSERT(scale != 0);
930 [[maybe_unused]] constexpr int32_t MAX_SHIFT = 3;
931 ASSERT(scale > 0 && scale <= MAX_SHIFT);
932 GetMasm()->Add(tmp, baseReg, vixl::aarch32::Operand(VixlReg(indexReg), vixl::aarch32::LSL, scale));
933 } else {
934 ASSERT(scale == 0);
935 GetMasm()->Add(tmp, baseReg, VixlReg(indexReg));
936 }
937 baseReg = tmp;
938 }
939 if (hasOffset) {
940 ASSERT(mem.GetDisp() != 0);
941 auto offset = mem.GetDisp();
942 constexpr int32_t IMM_4 = 4;
943 if ((offset >= -VMEM_OFFSET) && (offset <= VMEM_OFFSET) && ((offset % IMM_4) == 0)) {
944 return vixl::aarch32::MemOperand(baseReg, offset);
945 }
946 // from:
947 // vmem: base, offset
948 // to:
949 // add tmp, base, offset
950 // vmem tmp
951 GetMasm()->Add(tmp, baseReg, VixlImm(offset));
952 baseReg = tmp;
953 }
954 return vixl::aarch32::MemOperand(baseReg);
955 }
956
EncodeFpToBits(Reg dst,Reg src)957 void Aarch32Encoder::EncodeFpToBits(Reg dst, Reg src)
958 {
959 ASSERT(dst.IsScalar() && src.IsFloat());
960 if (dst.GetSize() == WORD_SIZE) {
961 ASSERT(src.GetSize() == WORD_SIZE);
962
963 constexpr uint32_t NANF = 0x7fc00000U;
964
965 GetMasm()->Vcmp(VixlVReg(src), VixlVReg(src));
966 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
967 GetMasm()->Vmov(VixlReg(dst), VixlVReg(src).S());
968 GetMasm()->Mov(Convert(Condition::NE), VixlReg(dst), VixlImm(NANF));
969 } else {
970 ASSERT(src.GetSize() == DOUBLE_WORD_SIZE);
971
972 constexpr uint32_t NAND_HIGH = 0x7ff80000U;
973
974 GetMasm()->Vcmp(VixlVReg(src), VixlVReg(src));
975 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
976 GetMasm()->Vmov(VixlReg(dst), VixlRegU(dst), VixlVReg(src).D());
977 GetMasm()->Mov(Convert(Condition::NE), VixlReg(dst), VixlImm(0));
978 GetMasm()->Mov(Convert(Condition::NE), VixlRegU(dst), VixlImm(NAND_HIGH));
979 }
980 }
981
EncodeMoveBitsRaw(Reg dst,Reg src)982 void Aarch32Encoder::EncodeMoveBitsRaw(Reg dst, Reg src)
983 {
984 ASSERT((dst.IsFloat() && src.IsScalar()) || (src.IsFloat() && dst.IsScalar()));
985 if (dst.IsScalar()) {
986 ASSERT(src.GetSize() == dst.GetSize());
987 if (dst.GetSize() == WORD_SIZE) {
988 GetMasm()->Vmov(VixlReg(dst), VixlVReg(src).S());
989 } else {
990 GetMasm()->Vmov(VixlReg(dst), VixlRegU(dst), VixlVReg(src).D());
991 }
992 } else {
993 ASSERT(dst.GetSize() == src.GetSize());
994 if (src.GetSize() == WORD_SIZE) {
995 GetMasm()->Vmov(VixlVReg(dst).S(), VixlReg(src));
996 } else {
997 GetMasm()->Vmov(VixlVReg(dst).D(), VixlReg(src), VixlRegU(src));
998 }
999 }
1000 }
EncodeMov(Reg dst,Reg src)1001 void Aarch32Encoder::EncodeMov(Reg dst, Reg src)
1002 {
1003 if (src.GetSize() <= WORD_SIZE && dst.GetSize() == DOUBLE_WORD_SIZE && !src.IsFloat() && !dst.IsFloat()) {
1004 SetFalseResult();
1005 return;
1006 }
1007 if (dst == src) {
1008 return;
1009 }
1010 if (dst.IsFloat()) {
1011 if (src.GetType().IsScalar()) {
1012 if (src.GetSize() == DOUBLE_WORD_SIZE) {
1013 GetMasm()->Vmov(VixlVReg(dst).D(), VixlReg(src), VixlRegU(src));
1014 return;
1015 }
1016 GetMasm()->Vmov(VixlVReg(dst).S(), VixlReg(src));
1017 return;
1018 }
1019 if (dst.GetSize() == WORD_SIZE) {
1020 GetMasm()->Vmov(VixlVReg(dst).S(), VixlVReg(src));
1021 } else {
1022 GetMasm()->Vmov(VixlVReg(dst).D(), VixlVReg(src));
1023 }
1024 return;
1025 }
1026 ASSERT(dst.IsScalar());
1027 if (src.IsFloat()) {
1028 if (src.GetSize() == DOUBLE_WORD_SIZE) {
1029 GetMasm()->Vmov(VixlReg(dst), VixlRegU(dst), VixlVReg(src).D());
1030 return;
1031 }
1032 GetMasm()->Vmov(VixlReg(dst), VixlVReg(src).S());
1033 return;
1034 }
1035 ASSERT(src.IsScalar());
1036 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1037 if (dst.GetSize() > WORD_SIZE) {
1038 GetMasm()->Mov(VixlRegU(dst), VixlRegU(src));
1039 }
1040 }
1041
EncodeNeg(Reg dst,Reg src)1042 void Aarch32Encoder::EncodeNeg(Reg dst, Reg src)
1043 {
1044 if (dst.IsFloat()) {
1045 GetMasm()->Vneg(VixlVReg(dst), VixlVReg(src));
1046 return;
1047 }
1048
1049 if (dst.GetSize() <= WORD_SIZE) {
1050 GetMasm()->Rsb(VixlReg(dst), VixlReg(src), VixlImm(0x0));
1051 return;
1052 }
1053
1054 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1055 GetMasm()->Rsbs(VixlReg(dst), VixlReg(src), VixlImm(0x0));
1056 GetMasm()->Rsc(VixlRegU(dst), VixlRegU(src), VixlImm(0x0));
1057 }
1058
EncodeAbs(Reg dst,Reg src)1059 void Aarch32Encoder::EncodeAbs(Reg dst, Reg src)
1060 {
1061 if (dst.IsFloat()) {
1062 GetMasm()->Vabs(VixlVReg(dst), VixlVReg(src));
1063 return;
1064 }
1065
1066 if (dst.GetSize() <= WORD_SIZE) {
1067 GetMasm()->Cmp(VixlReg(src), VixlImm(0x0));
1068 GetMasm()->Rsb(Convert(Condition::MI), VixlReg(dst), VixlReg(src), VixlImm(0x0));
1069 GetMasm()->Mov(Convert(Condition::PL), VixlReg(dst), VixlReg(src));
1070 return;
1071 }
1072
1073 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1074 ScopedTmpRegU32 loReg(this);
1075 ScopedTmpRegU32 hiReg(this);
1076
1077 GetMasm()->Rsbs(VixlReg(loReg), VixlReg(src), VixlImm(0x0));
1078 GetMasm()->Rsc(VixlReg(hiReg), VixlRegU(src), VixlImm(0x0));
1079 GetMasm()->Cmp(VixlRegU(src), VixlImm(0x0));
1080 GetMasm()->Mov(Convert(Condition::PL), VixlReg(loReg), VixlReg(src));
1081 GetMasm()->Mov(Convert(Condition::PL), VixlReg(hiReg), VixlRegU(src));
1082
1083 GetMasm()->Mov(VixlReg(dst), VixlReg(loReg));
1084 GetMasm()->Mov(VixlRegU(dst), VixlReg(hiReg));
1085 }
1086
EncodeSqrt(Reg dst,Reg src)1087 void Aarch32Encoder::EncodeSqrt(Reg dst, Reg src)
1088 {
1089 ASSERT(dst.IsFloat());
1090 GetMasm()->Vsqrt(VixlVReg(dst), VixlVReg(src));
1091 }
1092
EncodeNot(Reg dst,Reg src)1093 void Aarch32Encoder::EncodeNot([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src)
1094 {
1095 GetMasm()->Mvn(VixlReg(dst), VixlReg(src));
1096 if (dst.GetSize() > WORD_SIZE) {
1097 GetMasm()->Mvn(VixlRegU(dst), VixlRegU(src));
1098 }
1099 }
1100
EncodeIsInf(Reg dst,Reg src)1101 void Aarch32Encoder::EncodeIsInf(Reg dst, Reg src)
1102 {
1103 ASSERT(dst.IsScalar() && src.IsFloat());
1104
1105 if (src.GetSize() == WORD_SIZE) {
1106 constexpr uint32_t INF_MASK = 0xff000000;
1107
1108 ScopedTmpRegU32 tmp(this);
1109
1110 GetMasm()->Vmov(VixlReg(tmp), VixlVReg(src).S());
1111 GetMasm()->Lsl(VixlReg(tmp), VixlReg(tmp), 1); /* 0xff000000 if Infinity */
1112 GetMasm()->Mov(VixlReg(dst), INF_MASK);
1113 GetMasm()->Cmp(VixlReg(dst), VixlReg(tmp));
1114 } else {
1115 constexpr uint32_t INF_MASK = 0xffe00000;
1116
1117 ScopedTmpRegU32 tmp(this);
1118 ScopedTmpRegU32 tmp1(this);
1119
1120 GetMasm()->Vmov(VixlReg(tmp), VixlReg(tmp1), VixlVReg(src).D());
1121 GetMasm()->Lsl(VixlReg(tmp1), VixlReg(tmp1), 1); /* 0xffe00000 if Infinity */
1122 GetMasm()->Mov(VixlReg(dst), INF_MASK);
1123 GetMasm()->Cmp(VixlReg(dst), VixlReg(tmp1));
1124 }
1125
1126 GetMasm()->Mov(VixlReg(dst), VixlImm(0));
1127 GetMasm()->Mov(Convert(Condition::EQ), VixlReg(dst), VixlImm(1));
1128 }
1129
EncodeCmpFracWithDelta(Reg src)1130 void Aarch32Encoder::EncodeCmpFracWithDelta(Reg src)
1131 {
1132 ASSERT(src.IsFloat());
1133 ASSERT(src.GetSize() == WORD_SIZE || src.GetSize() == DOUBLE_WORD_SIZE);
1134
1135 // Encode (fabs(src - trunc(src)) <= DELTA)
1136 if (src.GetSize() == WORD_SIZE) {
1137 ScopedTmpRegF32 tmp(this);
1138 vixl::aarch32::SRegister tmp1(tmp.GetReg().GetId() + 1);
1139 GetMasm()->Vrintz(VixlVReg(tmp).S(), VixlVReg(src).S());
1140 EncodeSub(tmp, src, tmp);
1141 EncodeAbs(tmp, tmp);
1142 GetMasm()->Vmov(tmp1, std::numeric_limits<float>::epsilon());
1143 GetMasm()->Vcmp(VixlVReg(tmp).S(), tmp1);
1144 } else {
1145 ScopedTmpRegF64 tmp(this);
1146 vixl::aarch32::DRegister tmp1(tmp.GetReg().GetId() + 1);
1147 GetMasm()->Vrintz(VixlVReg(tmp).D(), VixlVReg(src).D());
1148 EncodeSub(tmp, src, tmp);
1149 EncodeAbs(tmp, tmp);
1150 GetMasm()->Vmov(tmp1, std::numeric_limits<double>::epsilon());
1151 GetMasm()->Vcmp(VixlVReg(tmp).D(), tmp1);
1152 }
1153 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
1154 }
1155
EncodeIsInteger(Reg dst,Reg src)1156 void Aarch32Encoder::EncodeIsInteger(Reg dst, Reg src)
1157 {
1158 ASSERT(dst.IsScalar() && src.IsFloat());
1159 ASSERT(src.GetSize() == WORD_SIZE || src.GetSize() == DOUBLE_WORD_SIZE);
1160
1161 auto labelExit = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(CreateLabel());
1162 auto labelInfOrNan = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(CreateLabel());
1163
1164 EncodeCmpFracWithDelta(src);
1165 GetMasm()->B(vixl::aarch32::vs, labelInfOrNan); // Inf or NaN
1166 GetMasm()->Mov(vixl::aarch32::le, VixlReg(dst), 0x1);
1167 GetMasm()->Mov(vixl::aarch32::gt, VixlReg(dst), 0x0);
1168 GetMasm()->B(labelExit);
1169
1170 // IsInteger returns false if src is Inf or NaN
1171 GetMasm()->Bind(labelInfOrNan);
1172 EncodeMov(dst, Imm(false));
1173
1174 GetMasm()->Bind(labelExit);
1175 }
1176
EncodeIsSafeInteger(Reg dst,Reg src)1177 void Aarch32Encoder::EncodeIsSafeInteger(Reg dst, Reg src)
1178 {
1179 ASSERT(dst.IsScalar() && src.IsFloat());
1180 ASSERT(src.GetSize() == WORD_SIZE || src.GetSize() == DOUBLE_WORD_SIZE);
1181
1182 auto labelExit = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(CreateLabel());
1183 auto labelFalse = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(CreateLabel());
1184
1185 // Check if IsInteger
1186 EncodeCmpFracWithDelta(src);
1187 GetMasm()->B(vixl::aarch32::vs, labelFalse); // Inf or NaN
1188 GetMasm()->B(vixl::aarch32::gt, labelFalse);
1189
1190 // Check if it is safe, i.e. src can be represented in float/double without losing precision
1191 if (src.GetSize() == WORD_SIZE) {
1192 ScopedTmpRegF32 tmp1(this);
1193 vixl::aarch32::SRegister tmp2(tmp1.GetReg().GetId() + 1);
1194 EncodeAbs(tmp1, src);
1195 GetMasm()->Vmov(tmp2, MaxIntAsExactFloat());
1196 GetMasm()->Vcmp(VixlVReg(tmp1).S(), tmp2);
1197 } else {
1198 ScopedTmpRegF64 tmp1(this);
1199 vixl::aarch32::DRegister tmp2(tmp1.GetReg().GetId() + 1);
1200 EncodeAbs(tmp1, src);
1201 GetMasm()->Vmov(tmp2, MaxIntAsExactDouble());
1202 GetMasm()->Vcmp(VixlVReg(tmp1).D(), tmp2);
1203 }
1204 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
1205 GetMasm()->Mov(vixl::aarch32::le, VixlReg(dst), 0x1);
1206 GetMasm()->Mov(vixl::aarch32::gt, VixlReg(dst), 0x0);
1207 GetMasm()->B(labelExit);
1208
1209 // Return false if src !IsInteger
1210 GetMasm()->Bind(labelFalse);
1211 EncodeMov(dst, Imm(false));
1212
1213 GetMasm()->Bind(labelExit);
1214 }
1215
EncodeReverseBytes(Reg dst,Reg src)1216 void Aarch32Encoder::EncodeReverseBytes(Reg dst, Reg src)
1217 {
1218 ASSERT(src.GetSize() > BYTE_SIZE);
1219 ASSERT(src.GetSize() == dst.GetSize());
1220
1221 if (src.GetSize() == HALF_SIZE) {
1222 GetMasm()->Rev16(VixlReg(dst), VixlReg(src));
1223 GetMasm()->Sxth(VixlReg(dst), VixlReg(dst));
1224 } else if (src.GetSize() == WORD_SIZE) {
1225 GetMasm()->Rev(VixlReg(dst), VixlReg(src));
1226 } else {
1227 if (src == dst) {
1228 ScopedTmpRegU32 tmpReg(this);
1229 GetMasm()->Mov(VixlReg(tmpReg), VixlReg(src));
1230 GetMasm()->Rev(VixlReg(dst), VixlRegU(src));
1231 GetMasm()->Rev(VixlRegU(dst), VixlReg(tmpReg));
1232 } else {
1233 GetMasm()->Rev(VixlRegU(dst), VixlReg(src));
1234 GetMasm()->Rev(VixlReg(dst), VixlRegU(src));
1235 }
1236 }
1237 }
1238
EncodeBitCount(Reg dst,Reg src)1239 void Aarch32Encoder::EncodeBitCount([[maybe_unused]] Reg dst, [[maybe_unused]] Reg src)
1240 {
1241 SetFalseResult();
1242 }
1243
EncodeCountLeadingZeroBits(Reg dst,Reg src)1244 void Aarch32Encoder::EncodeCountLeadingZeroBits(Reg dst, Reg src)
1245 {
1246 ASSERT(src.GetSize() == WORD_SIZE || src.GetSize() == DOUBLE_WORD_SIZE);
1247 if (src.GetSize() == WORD_SIZE) {
1248 GetMasm()->Clz(VixlReg(dst), VixlReg(src));
1249 return;
1250 }
1251
1252 auto low = CreateLabel();
1253 auto end = CreateLabel();
1254 auto highBits = Reg(src.GetId() + 1, INT32_TYPE);
1255 EncodeJump(low, highBits, Condition::EQ);
1256 GetMasm()->Clz(VixlReg(dst), VixlReg(highBits));
1257 EncodeJump(end);
1258
1259 BindLabel(low);
1260 GetMasm()->Clz(VixlReg(dst), VixlReg(src));
1261 GetMasm()->Adds(VixlReg(dst), VixlReg(dst), VixlImm(WORD_SIZE));
1262
1263 BindLabel(end);
1264 }
1265
EncodeCeil(Reg dst,Reg src)1266 void Aarch32Encoder::EncodeCeil(Reg dst, Reg src)
1267 {
1268 GetMasm()->Vrintp(VixlVReg(dst), VixlVReg(src));
1269 }
1270
EncodeFloor(Reg dst,Reg src)1271 void Aarch32Encoder::EncodeFloor(Reg dst, Reg src)
1272 {
1273 GetMasm()->Vrintm(VixlVReg(dst), VixlVReg(src));
1274 }
1275
EncodeRint(Reg dst,Reg src)1276 void Aarch32Encoder::EncodeRint(Reg dst, Reg src)
1277 {
1278 GetMasm()->Vrintn(VixlVReg(dst), VixlVReg(src));
1279 }
1280
EncodeTrunc(Reg dst,Reg src)1281 void Aarch32Encoder::EncodeTrunc(Reg dst, Reg src)
1282 {
1283 GetMasm()->Vrintz(VixlVReg(dst), VixlVReg(src));
1284 }
1285
EncodeRoundAway(Reg dst,Reg src)1286 void Aarch32Encoder::EncodeRoundAway(Reg dst, Reg src)
1287 {
1288 GetMasm()->Vrinta(VixlVReg(dst), VixlVReg(src));
1289 }
1290
EncodeRoundToPInf(Reg dst,Reg src)1291 void Aarch32Encoder::EncodeRoundToPInf(Reg dst, Reg src)
1292 {
1293 ScopedTmpRegF64 temp(this);
1294 vixl::aarch32::SRegister temp1(temp.GetReg().GetId());
1295 vixl::aarch32::SRegister temp2(temp.GetReg().GetId() + 1);
1296
1297 auto done = CreateLabel();
1298 // round to nearest integer, ties away from zero
1299 GetMasm()->Vcvta(vixl::aarch32::DataTypeValue::S32, vixl::aarch32::DataTypeValue::F32, temp1, VixlVReg(src).S());
1300 GetMasm()->Vmov(VixlReg(dst), temp1);
1301 // for positive, zero and NaN inputs, rounding is done
1302 GetMasm()->Cmp(VixlReg(dst), 0);
1303 GetMasm()->B(vixl::aarch32::ge, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(done));
1304 // if input is negative but not a tie, round to nearest is valid.
1305 // if input is a negative tie, change rounding direction to positive infinity, dst += 1.
1306 GetMasm()->Vrinta(vixl::aarch32::DataTypeValue::F32, temp1, VixlVReg(src).S());
1307 // NOLINTNEXTLINE(readability-magic-numbers)
1308 GetMasm()->Vmov(temp2, 0.5F);
1309 GetMasm()->Vsub(vixl::aarch32::DataTypeValue::F32, temp1, VixlVReg(src).S(), temp1);
1310 GetMasm()->Vcmp(vixl::aarch32::DataTypeValue::F32, temp1, temp2);
1311 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
1312 GetMasm()->add(vixl::aarch32::eq, VixlReg(dst), VixlReg(dst), 1);
1313
1314 BindLabel(done);
1315 }
1316
EncodeReverseBits(Reg dst,Reg src)1317 void Aarch32Encoder::EncodeReverseBits(Reg dst, Reg src)
1318 {
1319 ASSERT(src.GetSize() == WORD_SIZE || src.GetSize() == DOUBLE_WORD_SIZE);
1320 ASSERT(src.GetSize() == dst.GetSize());
1321
1322 if (src.GetSize() == WORD_SIZE) {
1323 GetMasm()->Rbit(VixlReg(dst), VixlReg(src));
1324 return;
1325 }
1326
1327 if (src == dst) {
1328 ScopedTmpRegU32 tmpReg(this);
1329 GetMasm()->Mov(VixlReg(tmpReg), VixlReg(src));
1330 GetMasm()->Rbit(VixlReg(dst), VixlRegU(src));
1331 GetMasm()->Rbit(VixlRegU(dst), VixlReg(tmpReg));
1332 return;
1333 }
1334
1335 GetMasm()->Rbit(VixlRegU(dst), VixlReg(src));
1336 GetMasm()->Rbit(VixlReg(dst), VixlRegU(src));
1337 }
1338
EncodeCastToBool(Reg dst,Reg src)1339 void Aarch32Encoder::EncodeCastToBool(Reg dst, Reg src)
1340 {
1341 // In ISA says that we only support casts:
1342 // i32tou1, i64tou1, u32tou1, u64tou1
1343 ASSERT(src.IsScalar());
1344 ASSERT(dst.IsScalar());
1345
1346 GetMasm()->Cmp(VixlReg(src), VixlImm(0x0));
1347 GetMasm()->Mov(Convert(Condition::EQ), VixlReg(dst), VixlImm(0x0));
1348 GetMasm()->Mov(Convert(Condition::NE), VixlReg(dst), VixlImm(0x1));
1349 if (src.GetSize() == DOUBLE_WORD_SIZE) {
1350 GetMasm()->Cmp(VixlRegU(src), VixlImm(0x0));
1351 GetMasm()->Mov(Convert(Condition::NE), VixlReg(dst), VixlImm(0x1));
1352 }
1353 }
1354
EncodeCast(Reg dst,bool dstSigned,Reg src,bool srcSigned)1355 void Aarch32Encoder::EncodeCast(Reg dst, bool dstSigned, Reg src, bool srcSigned)
1356 {
1357 // float/double -> float/double
1358 if (dst.IsFloat() && src.IsFloat()) {
1359 EncodeCastFloatToFloat(dst, src);
1360 return;
1361 }
1362
1363 // uint/int -> float/double
1364 if (dst.IsFloat() && src.IsScalar()) {
1365 EncodeCastScalarToFloat(dst, src, srcSigned);
1366 return;
1367 }
1368
1369 // float/double -> uint/int
1370 if (dst.IsScalar() && src.IsFloat()) {
1371 EncodeCastFloatToScalar(dst, dstSigned, src);
1372 return;
1373 }
1374
1375 // uint/int -> uint/int
1376 ASSERT(dst.IsScalar() && src.IsScalar());
1377 EncodeCastScalar(dst, dstSigned, src, srcSigned);
1378 }
1379
EncodeCastScalar(Reg dst,bool dstSigned,Reg src,bool srcSigned)1380 void Aarch32Encoder::EncodeCastScalar(Reg dst, bool dstSigned, Reg src, bool srcSigned)
1381 {
1382 size_t srcSize = src.GetSize();
1383 size_t dstSize = dst.GetSize();
1384 // In our ISA minimal type is 32-bit, so type less then 32-bit
1385 // we should extend to 32-bit. So we can have 2 cast
1386 // (For examble, i8->u16 will work as i8->u16 and u16->u32)
1387 if (dstSize < WORD_SIZE) {
1388 if (srcSize > dstSize) {
1389 if (dstSigned) {
1390 EncodeCastScalarFromSignedScalar(dst, src);
1391 } else {
1392 EncodeCastScalarFromUnsignedScalar(dst, src);
1393 }
1394 return;
1395 }
1396 if (srcSize == dstSize) {
1397 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1398 if (srcSigned == dstSigned) {
1399 return;
1400 }
1401 if (dstSigned) {
1402 EncodeCastScalarFromSignedScalar(Reg(dst.GetId(), INT32_TYPE), dst);
1403 } else {
1404 EncodeCastScalarFromUnsignedScalar(Reg(dst.GetId(), INT32_TYPE), dst);
1405 }
1406 return;
1407 }
1408 if (srcSigned) {
1409 EncodeCastScalarFromSignedScalar(dst, src);
1410 if (!dstSigned) {
1411 EncodeCastScalarFromUnsignedScalar(Reg(dst.GetId(), INT32_TYPE), dst);
1412 }
1413 } else {
1414 EncodeCastScalarFromUnsignedScalar(dst, src);
1415 if (dstSigned) {
1416 EncodeCastScalarFromSignedScalar(Reg(dst.GetId(), INT32_TYPE), dst);
1417 }
1418 }
1419 } else {
1420 if (srcSize == dstSize) {
1421 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1422 if (srcSize == DOUBLE_WORD_SIZE) {
1423 GetMasm()->Mov(VixlRegU(dst), VixlRegU(src));
1424 }
1425 return;
1426 }
1427
1428 if (srcSigned) {
1429 EncodeCastScalarFromSignedScalar(dst, src);
1430 } else {
1431 EncodeCastScalarFromUnsignedScalar(dst, src);
1432 }
1433 }
1434 }
1435
EncodeCastFloatToFloat(Reg dst,Reg src)1436 void Aarch32Encoder::EncodeCastFloatToFloat(Reg dst, Reg src)
1437 {
1438 // float/double -> float/double
1439 if (dst.GetSize() == src.GetSize()) {
1440 if (dst.GetSize() == WORD_SIZE) {
1441 GetMasm()->Vmov(VixlVReg(dst).S(), VixlVReg(src));
1442 } else {
1443 GetMasm()->Vmov(VixlVReg(dst).D(), VixlVReg(src));
1444 }
1445 return;
1446 }
1447
1448 // double -> float
1449 if (dst.GetSize() == WORD_SIZE) {
1450 GetMasm()->Vcvt(vixl::aarch32::DataTypeValue::F32, vixl::aarch32::DataTypeValue::F64, VixlVReg(dst).S(),
1451 VixlVReg(src).D());
1452 } else {
1453 // float -> double
1454 GetMasm()->Vcvt(vixl::aarch32::DataTypeValue::F64, vixl::aarch32::DataTypeValue::F32, VixlVReg(dst).D(),
1455 VixlVReg(src).S());
1456 }
1457 }
1458
EncodeCastScalarToFloat(Reg dst,Reg src,bool srcSigned)1459 void Aarch32Encoder::EncodeCastScalarToFloat(Reg dst, Reg src, bool srcSigned)
1460 {
1461 // uint/int -> float/double
1462 switch (src.GetSize()) {
1463 case BYTE_SIZE:
1464 case HALF_SIZE:
1465 case WORD_SIZE: {
1466 ScopedTmpRegF32 tmpReg(this);
1467
1468 GetMasm()->Vmov(VixlVReg(tmpReg).S(), VixlReg(src));
1469 auto dataType = srcSigned ? vixl::aarch32::DataTypeValue::S32 : vixl::aarch32::DataTypeValue::U32;
1470 if (dst.GetSize() == WORD_SIZE) {
1471 GetMasm()->Vcvt(vixl::aarch32::DataTypeValue::F32, dataType, VixlVReg(dst).S(), VixlVReg(tmpReg).S());
1472 } else {
1473 GetMasm()->Vcvt(vixl::aarch32::DataTypeValue::F64, dataType, VixlVReg(dst).D(), VixlVReg(tmpReg).S());
1474 }
1475 break;
1476 }
1477 case DOUBLE_WORD_SIZE: {
1478 if (dst.GetSize() == WORD_SIZE) {
1479 if (srcSigned) {
1480 // int64 -> float
1481 MakeLibCall(dst, src, reinterpret_cast<void *>(AEABIl2f));
1482 } else {
1483 // uint64 -> float
1484 MakeLibCall(dst, src, reinterpret_cast<void *>(AEABIul2f));
1485 }
1486 } else {
1487 if (srcSigned) {
1488 // int64 -> double
1489 MakeLibCall(dst, src, reinterpret_cast<void *>(AEABIl2d));
1490 } else {
1491 // uint64 -> double
1492 MakeLibCall(dst, src, reinterpret_cast<void *>(AEABIul2d));
1493 }
1494 }
1495 break;
1496 }
1497 default:
1498 SetFalseResult();
1499 break;
1500 }
1501 }
1502
EncodeCastFloatToScalar(Reg dst,bool dstSigned,Reg src)1503 void Aarch32Encoder::EncodeCastFloatToScalar(Reg dst, bool dstSigned, Reg src)
1504 {
1505 // We DON'T support casts from float32/64 to int8/16 and bool, because this caste is not declared anywhere
1506 // in other languages and architecture, we do not know what the behavior should be.
1507 // But there is one implementation in other function: "EncodeCastFloatToScalarWithSmallDst". Call it in the
1508 // "EncodeCast" function instead of "EncodeCastFloat". It works as follows: cast from float32/64 to int32, moving
1509 // sign bit from int32 to dst type, then extend number from dst type to int32 (a necessary condition for an isa).
1510 // All work in dst register.
1511 ASSERT(dst.GetSize() >= WORD_SIZE);
1512
1513 switch (dst.GetSize()) {
1514 case WORD_SIZE: {
1515 ScopedTmpRegF32 tmpReg(this);
1516
1517 auto dataType = dstSigned ? vixl::aarch32::DataTypeValue::S32 : vixl::aarch32::DataTypeValue::U32;
1518 if (src.GetSize() == WORD_SIZE) {
1519 GetMasm()->Vcvt(dataType, vixl::aarch32::DataTypeValue::F32, VixlVReg(tmpReg).S(), VixlVReg(src).S());
1520 } else {
1521 GetMasm()->Vcvt(dataType, vixl::aarch32::DataTypeValue::F64, VixlVReg(tmpReg).S(), VixlVReg(src).D());
1522 }
1523
1524 GetMasm()->Vmov(VixlReg(dst), VixlVReg(tmpReg).S());
1525 break;
1526 }
1527 case DOUBLE_WORD_SIZE: {
1528 if (src.GetSize() == WORD_SIZE) {
1529 if (dstSigned) {
1530 // float -> int64
1531 EncodeCastFloatToInt64(dst, src);
1532 } else {
1533 // float -> uint64
1534 MakeLibCall(dst, src, reinterpret_cast<void *>(AEABIf2ulz));
1535 }
1536 } else {
1537 if (dstSigned) {
1538 // double -> int64
1539 EncodeCastDoubleToInt64(dst, src);
1540 } else {
1541 // double -> uint64
1542 MakeLibCall(dst, src, reinterpret_cast<void *>(AEABId2ulz));
1543 }
1544 }
1545 break;
1546 }
1547 default:
1548 SetFalseResult();
1549 break;
1550 }
1551 }
1552
EncodeCastFloatToScalarWithSmallDst(Reg dst,bool dstSigned,Reg src)1553 void Aarch32Encoder::EncodeCastFloatToScalarWithSmallDst(Reg dst, bool dstSigned, Reg src)
1554 {
1555 switch (dst.GetSize()) {
1556 case BYTE_SIZE:
1557 case HALF_SIZE:
1558 case WORD_SIZE: {
1559 ScopedTmpRegF32 tmpReg(this);
1560
1561 auto dataType = dstSigned ? vixl::aarch32::DataTypeValue::S32 : vixl::aarch32::DataTypeValue::U32;
1562 if (src.GetSize() == WORD_SIZE) {
1563 GetMasm()->Vcvt(dataType, vixl::aarch32::DataTypeValue::F32, VixlVReg(tmpReg).S(), VixlVReg(src).S());
1564 } else {
1565 GetMasm()->Vcvt(dataType, vixl::aarch32::DataTypeValue::F64, VixlVReg(tmpReg).S(), VixlVReg(src).D());
1566 }
1567
1568 GetMasm()->Vmov(VixlReg(dst), VixlVReg(tmpReg).S());
1569 if (dst.GetSize() < WORD_SIZE) {
1570 EncoderCastExtendFromInt32(dst, dstSigned);
1571 }
1572 break;
1573 }
1574 case DOUBLE_WORD_SIZE: {
1575 if (src.GetSize() == WORD_SIZE) {
1576 if (dstSigned) {
1577 // float -> int64
1578 EncodeCastFloatToInt64(dst, src);
1579 } else {
1580 // float -> uint64
1581 MakeLibCall(dst, src, reinterpret_cast<void *>(AEABIf2ulz));
1582 }
1583 } else {
1584 if (dstSigned) {
1585 // double -> int64
1586 EncodeCastDoubleToInt64(dst, src);
1587 } else {
1588 // double -> uint64
1589 MakeLibCall(dst, src, reinterpret_cast<void *>(AEABId2ulz));
1590 }
1591 }
1592 break;
1593 }
1594 default:
1595 SetFalseResult();
1596 break;
1597 }
1598 }
1599
EncoderCastExtendFromInt32(Reg dst,bool dstSigned)1600 void Aarch32Encoder::EncoderCastExtendFromInt32(Reg dst, bool dstSigned)
1601 {
1602 if (dstSigned) {
1603 constexpr uint32_t TEST_BIT = (1U << (static_cast<uint32_t>(WORD_SIZE) - 1));
1604 ScopedTmpReg tmpReg(this, dst.GetType());
1605
1606 uint32_t setBit = (dst.GetSize() == BYTE_SIZE) ? (1U << static_cast<uint32_t>(BYTE_SIZE - 1))
1607 : (1U << static_cast<uint32_t>(HALF_SIZE - 1));
1608 uint32_t remBit = setBit - 1;
1609 GetMasm()->And(VixlReg(tmpReg), VixlReg(dst), TEST_BIT);
1610 auto labelSkip = CreateLabel();
1611 auto labelEndIf = CreateLabel();
1612 EncodeJump(labelSkip, tmpReg, Condition::EQ);
1613 // Set signed bit in dst reg (accordingly destination type)
1614 // If signed bit == 1
1615 GetMasm()->Orr(VixlReg(dst), VixlReg(dst), setBit);
1616 EncodeJump(labelEndIf);
1617 BindLabel(labelSkip);
1618 // If signed bit == 0
1619 GetMasm()->And(VixlReg(dst), VixlReg(dst), remBit);
1620 BindLabel(labelEndIf);
1621 }
1622 EncodeCastScalar(Reg(dst.GetId(), INT32_TYPE), dstSigned, dst, dstSigned);
1623 }
1624
EncodeCastDoubleToInt64(Reg dst,Reg src)1625 void Aarch32Encoder::EncodeCastDoubleToInt64(Reg dst, Reg src)
1626 {
1627 auto labelCheckNan = CreateLabel();
1628 auto labelNotNan = CreateLabel();
1629 auto labelExit = CreateLabel();
1630 ScopedTmpReg tmpReg1(this, INT32_TYPE);
1631
1632 // Mov double to 2x reg to storage double in hex format and work with them
1633 EncodeMov(dst, src);
1634 // See the exponent of number
1635 GetMasm()->Ubfx(VixlReg(tmpReg1), VixlRegU(dst), START_EXP_DOUBLE, SIZE_EXP_DOUBLE);
1636 // Max exponent that we can load in int64
1637 // Check that x > MIN_INT64 & x < MAX_INT64, else jump
1638 GetMasm()->Cmp(VixlReg(tmpReg1), VixlImm(POSSIBLE_EXP_DOUBLE));
1639 // If greater than or equal, branch to "label_not_nan"
1640 GetMasm()->B(vixl::aarch32::hs, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(labelCheckNan));
1641 MakeLibCall(dst, src, reinterpret_cast<void *>(AEABId2lz));
1642 EncodeJump(labelExit);
1643
1644 BindLabel(labelCheckNan);
1645 // Form of nan number
1646 GetMasm()->Cmp(VixlReg(tmpReg1), VixlImm(UP_BITS_NAN_DOUBLE));
1647 // If not equal, branch to "label_not_nan"
1648 GetMasm()->B(vixl::aarch32::ne, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(labelNotNan));
1649
1650 GetMasm()->Orrs(VixlReg(tmpReg1), VixlReg(dst),
1651 vixl::aarch32::Operand(VixlRegU(dst), vixl::aarch32::LSL, SHIFT_BITS_DOUBLE));
1652 auto addrLabel = static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(labelNotNan);
1653 GetMasm()->B(vixl::aarch32::eq, addrLabel);
1654 GetMasm()->Mov(VixlReg(dst), VixlImm(0));
1655 GetMasm()->Mov(VixlRegU(dst), VixlImm(0));
1656 EncodeJump(labelExit);
1657
1658 BindLabel(labelNotNan);
1659 GetMasm()->Adds(VixlRegU(dst), VixlRegU(dst), VixlRegU(dst));
1660 GetMasm()->Mov(VixlReg(dst), VixlImm(UINT32_MAX));
1661 GetMasm()->Mov(VixlRegU(dst), VixlImm(INT32_MAX));
1662 GetMasm()->Adc(VixlReg(dst), VixlReg(dst), VixlImm(0));
1663 // If exponent negative, transform maxint64 to minint64
1664 GetMasm()->Adc(VixlRegU(dst), VixlRegU(dst), VixlImm(0));
1665
1666 BindLabel(labelExit);
1667 }
1668
EncodeCastFloatToInt64(Reg dst,Reg src)1669 void Aarch32Encoder::EncodeCastFloatToInt64(Reg dst, Reg src)
1670 {
1671 auto labelCheckNan = CreateLabel();
1672 auto labelNotNan = CreateLabel();
1673 auto labelExit = CreateLabel();
1674
1675 ScopedTmpReg tmpReg(this, INT32_TYPE);
1676 ScopedTmpReg movedSrc(this, INT32_TYPE);
1677 EncodeMov(movedSrc, src);
1678 // See the exponent of number
1679 GetMasm()->Ubfx(VixlReg(tmpReg), VixlReg(movedSrc), START_EXP_FLOAT, SIZE_EXP_FLOAT);
1680 // Max exponent that we can load in int64
1681 // Check that x > MIN_INT64 & x < MAX_INT64, else jump
1682 GetMasm()->Cmp(VixlReg(tmpReg), VixlImm(POSSIBLE_EXP_FLOAT));
1683 // If greater than or equal, branch to "label_not_nan"
1684 GetMasm()->B(vixl::aarch32::hs, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(labelCheckNan));
1685 MakeLibCall(dst, src, reinterpret_cast<void *>(AEABIf2lz));
1686 EncodeJump(labelExit);
1687
1688 BindLabel(labelCheckNan);
1689 // Form of nan number
1690 GetMasm()->Cmp(VixlReg(tmpReg), VixlImm(UP_BITS_NAN_FLOAT));
1691 // If not equal, branch to "label_not_nan"
1692 GetMasm()->B(vixl::aarch32::ne, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(labelNotNan));
1693
1694 GetMasm()->Lsls(VixlReg(tmpReg), VixlReg(movedSrc), vixl::aarch32::Operand(SHIFT_BITS_FLOAT));
1695 // If equal, branch to "label_not_nan"
1696 GetMasm()->B(vixl::aarch32::eq, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(labelNotNan));
1697 GetMasm()->Mov(VixlReg(dst), VixlImm(0));
1698 GetMasm()->Mov(VixlRegU(dst), VixlImm(0));
1699 EncodeJump(labelExit);
1700
1701 BindLabel(labelNotNan);
1702 GetMasm()->Adds(VixlReg(movedSrc), VixlReg(movedSrc), VixlReg(movedSrc));
1703 GetMasm()->Mov(VixlReg(dst), VixlImm(UINT32_MAX));
1704 GetMasm()->Mov(VixlRegU(dst), VixlImm(INT32_MAX));
1705 GetMasm()->Adc(VixlReg(dst), VixlReg(dst), VixlImm(0));
1706 // If exponent negative, transform maxint64 to minint64
1707 GetMasm()->Adc(VixlRegU(dst), VixlRegU(dst), VixlImm(0));
1708
1709 BindLabel(labelExit);
1710 }
1711
EncodeCastScalarFromSignedScalar(Reg dst,Reg src)1712 void Aarch32Encoder::EncodeCastScalarFromSignedScalar(Reg dst, Reg src)
1713 {
1714 size_t srcSize = src.GetSize();
1715 size_t dstSize = dst.GetSize();
1716 if (srcSize > dstSize) {
1717 srcSize = dstSize;
1718 }
1719 switch (srcSize) {
1720 case BYTE_SIZE:
1721 GetMasm()->Sxtb(VixlReg(dst), VixlReg(src));
1722 break;
1723 case HALF_SIZE:
1724 GetMasm()->Sxth(VixlReg(dst), VixlReg(src));
1725 break;
1726 case WORD_SIZE:
1727 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1728 break;
1729 case DOUBLE_WORD_SIZE:
1730 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1731 if (dstSize == DOUBLE_WORD_SIZE) {
1732 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1733 return;
1734 }
1735 break;
1736 default:
1737 SetFalseResult();
1738 break;
1739 }
1740 if (dstSize == DOUBLE_WORD_SIZE) {
1741 GetMasm()->Asr(VixlRegU(dst), VixlReg(dst), VixlImm(WORD_SIZE - 1));
1742 }
1743 }
1744
EncodeCastScalarFromUnsignedScalar(Reg dst,Reg src)1745 void Aarch32Encoder::EncodeCastScalarFromUnsignedScalar(Reg dst, Reg src)
1746 {
1747 size_t srcSize = src.GetSize();
1748 size_t dstSize = dst.GetSize();
1749 if (srcSize > dstSize && dstSize < WORD_SIZE) {
1750 // We need to cut the number, if it is less, than 32-bit. It is by ISA agreement.
1751 int64_t cutValue = (1ULL << dstSize) - 1;
1752 GetMasm()->And(VixlReg(dst), VixlReg(src), VixlImm(cutValue));
1753 return;
1754 }
1755 // Else unsigned extend
1756 switch (srcSize) {
1757 case BYTE_SIZE:
1758 GetMasm()->Uxtb(VixlReg(dst), VixlReg(src));
1759 break;
1760 case HALF_SIZE:
1761 GetMasm()->Uxth(VixlReg(dst), VixlReg(src));
1762 break;
1763 case WORD_SIZE:
1764 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1765 break;
1766 case DOUBLE_WORD_SIZE:
1767 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1768 if (dstSize == DOUBLE_WORD_SIZE) {
1769 GetMasm()->Mov(VixlReg(dst), VixlReg(src));
1770 return;
1771 }
1772 break;
1773 default:
1774 SetFalseResult();
1775 break;
1776 }
1777 if (dstSize == DOUBLE_WORD_SIZE) {
1778 GetMasm()->Mov(VixlRegU(dst), VixlImm(0x0));
1779 }
1780 }
1781
EncodeAdd(Reg dst,Reg src0,Reg src1)1782 void Aarch32Encoder::EncodeAdd(Reg dst, Reg src0, Reg src1)
1783 {
1784 if (dst.IsFloat()) {
1785 ASSERT(src0.IsFloat() && src1.IsFloat());
1786 GetMasm()->Vadd(VixlVReg(dst), VixlVReg(src0), VixlVReg(src1));
1787 return;
1788 }
1789
1790 if (dst.GetSize() <= WORD_SIZE) {
1791 GetMasm()->Add(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1792 return;
1793 }
1794
1795 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1796 GetMasm()->Adds(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1797 GetMasm()->Adc(VixlRegU(dst), VixlRegU(src0), VixlRegU(src1));
1798 }
1799
EncodeSub(Reg dst,Reg src0,Reg src1)1800 void Aarch32Encoder::EncodeSub(Reg dst, Reg src0, Reg src1)
1801 {
1802 if (dst.IsFloat()) {
1803 GetMasm()->Vsub(VixlVReg(dst), VixlVReg(src0), VixlVReg(src1));
1804 return;
1805 }
1806
1807 if (dst.GetSize() <= WORD_SIZE) {
1808 GetMasm()->Sub(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1809 return;
1810 }
1811
1812 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1813 GetMasm()->Subs(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1814 GetMasm()->Sbc(VixlRegU(dst), VixlRegU(src0), VixlRegU(src1));
1815 }
1816
EncodeMul(Reg dst,Reg src0,Reg src1)1817 void Aarch32Encoder::EncodeMul(Reg dst, Reg src0, Reg src1)
1818 {
1819 if (dst.IsFloat()) {
1820 GetMasm()->Vmul(VixlVReg(dst), VixlVReg(src0), VixlVReg(src1));
1821 return;
1822 }
1823
1824 if (dst.GetSize() <= WORD_SIZE) {
1825 GetMasm()->Mul(VixlReg(dst), VixlReg(src0), VixlReg(src1));
1826 return;
1827 }
1828
1829 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
1830 ScopedTmpRegU32 loReg(this);
1831 ScopedTmpRegU32 hiReg(this);
1832
1833 GetMasm()->Umull(VixlReg(loReg), VixlReg(hiReg), VixlReg(src0), VixlReg(src1));
1834 GetMasm()->Mla(VixlReg(hiReg), VixlRegU(src0), VixlReg(src1), VixlReg(hiReg));
1835 GetMasm()->Mla(VixlReg(hiReg), VixlReg(src0), VixlRegU(src1), VixlReg(hiReg));
1836
1837 GetMasm()->Mov(VixlReg(dst), VixlReg(loReg));
1838 GetMasm()->Mov(VixlRegU(dst), VixlReg(hiReg));
1839 }
1840
MakeLibCall(Reg dst,Reg src0,Reg src1,void * entryPoint,bool secondValue)1841 void Aarch32Encoder::MakeLibCall(Reg dst, Reg src0, Reg src1, void *entryPoint, bool secondValue)
1842 {
1843 if (dst.GetType() == FLOAT32_TYPE) {
1844 MakeLibCallWithFloatResult(dst, src0, src1, entryPoint, secondValue);
1845 return;
1846 }
1847
1848 if (dst.GetType() == FLOAT64_TYPE) {
1849 MakeLibCallWithDoubleResult(dst, src0, src1, entryPoint, secondValue);
1850 return;
1851 }
1852
1853 if (dst.GetType() == INT64_TYPE) {
1854 MakeLibCallWithInt64Result(dst, src0, src1, entryPoint, secondValue);
1855 return;
1856 }
1857
1858 ASSERT(dst.GetSize() < DOUBLE_WORD_SIZE);
1859
1860 if (src1.GetId() == vixl::aarch32::r0.GetCode() || src0 == src1) {
1861 ScopedTmpRegU32 tmp(this);
1862 GetMasm()->Mov(VixlReg(tmp), VixlReg(src1));
1863 GetMasm()->Mov(vixl::aarch32::r0, VixlReg(src0));
1864 GetMasm()->Mov(vixl::aarch32::r1, VixlReg(tmp));
1865 } else {
1866 GetMasm()->Mov(vixl::aarch32::r0, VixlReg(src0));
1867 GetMasm()->Mov(vixl::aarch32::r1, VixlReg(src1));
1868 }
1869
1870 // Call lib-method
1871 MakeCall(entryPoint);
1872
1873 auto dstRegister = secondValue ? vixl::aarch32::r1 : vixl::aarch32::r0;
1874
1875 if (dst.GetId() <= vixl::aarch32::r3.GetCode()) {
1876 ScopedTmpRegU32 tmp(this);
1877 GetMasm()->Mov(VixlReg(tmp), dstRegister);
1878 GetMasm()->Mov(VixlReg(dst), VixlReg(tmp));
1879 } else {
1880 GetMasm()->Mov(VixlReg(dst), dstRegister);
1881 }
1882 }
1883
MakeLibCallWithFloatResult(Reg dst,Reg src0,Reg src1,void * entryPoint,bool secondValue)1884 void Aarch32Encoder::MakeLibCallWithFloatResult(Reg dst, Reg src0, Reg src1, void *entryPoint, bool secondValue)
1885 {
1886 #if (PANDA_TARGET_ARM32_ABI_HARD)
1887 // gnueabihf
1888 // use double parameters
1889 if (src1.GetId() == vixl::aarch32::s0.GetCode() || src0 == src1) {
1890 ScopedTmpRegF32 tmp(this);
1891 GetMasm()->Vmov(VixlVReg(tmp), VixlVReg(src1));
1892 GetMasm()->Vmov(vixl::aarch32::s0, VixlVReg(src0));
1893 GetMasm()->Vmov(vixl::aarch32::s1, VixlVReg(tmp));
1894 } else {
1895 GetMasm()->Vmov(vixl::aarch32::s0, VixlVReg(src0));
1896 GetMasm()->Vmov(vixl::aarch32::s1, VixlVReg(src1));
1897 }
1898
1899 MakeCall(entryPoint);
1900
1901 auto dstRegister = secondValue ? vixl::aarch32::s1 : vixl::aarch32::s0;
1902 if (dst.GetId() <= vixl::aarch32::s1.GetCode()) {
1903 ScopedTmpRegF32 tmp(this);
1904 GetMasm()->Vmov(VixlVReg(tmp).S(), dstRegister);
1905 GetMasm()->Vmov(VixlVReg(dst).S(), VixlVReg(tmp).S());
1906 } else {
1907 GetMasm()->Vmov(VixlVReg(dst).S(), dstRegister);
1908 }
1909 #else
1910 // gnueabi
1911 // use scalar parameters
1912 GetMasm()->Vmov(vixl::aarch32::r0, VixlVReg(src0).S());
1913 GetMasm()->Vmov(vixl::aarch32::r1, VixlVReg(src1).S());
1914
1915 MakeCall(entryPoint);
1916
1917 auto dstRegister = secondValue ? vixl::aarch32::r1 : vixl::aarch32::r0;
1918 GetMasm()->Vmov(VixlVReg(dst).S(), dstRegister);
1919 #endif // PANDA_TARGET_ARM32_ABI_HARD
1920 }
1921
MakeLibCallWithDoubleResult(Reg dst,Reg src0,Reg src1,void * entryPoint,bool secondValue)1922 void Aarch32Encoder::MakeLibCallWithDoubleResult(Reg dst, Reg src0, Reg src1, void *entryPoint, bool secondValue)
1923 {
1924 #if (PANDA_TARGET_ARM32_ABI_HARD)
1925 // Scope for temp
1926 if (src1.GetId() == vixl::aarch32::d0.GetCode() || src0 == src1) {
1927 ScopedTmpRegF64 tmp(this);
1928 GetMasm()->Vmov(VixlVReg(tmp), VixlVReg(src1));
1929 GetMasm()->Vmov(vixl::aarch32::d0, VixlVReg(src0));
1930 GetMasm()->Vmov(vixl::aarch32::d1, VixlVReg(tmp));
1931 } else {
1932 GetMasm()->Vmov(vixl::aarch32::d0, VixlVReg(src0));
1933 GetMasm()->Vmov(vixl::aarch32::d1, VixlVReg(src1));
1934 }
1935 MakeCall(entryPoint);
1936 auto dstRegister = secondValue ? vixl::aarch32::d1 : vixl::aarch32::d0;
1937
1938 if (dst.GetId() <= vixl::aarch32::d1.GetCode()) {
1939 ScopedTmpRegF64 tmp(this);
1940 GetMasm()->Vmov(VixlVReg(tmp), dstRegister);
1941 GetMasm()->Vmov(VixlVReg(dst), VixlVReg(tmp));
1942 } else {
1943 GetMasm()->Vmov(VixlVReg(dst), dstRegister);
1944 }
1945
1946 // use double parameters
1947 #else
1948 // use scalar parameters
1949 GetMasm()->Vmov(vixl::aarch32::r0, vixl::aarch32::r1, VixlVReg(src0).D());
1950 GetMasm()->Vmov(vixl::aarch32::r2, vixl::aarch32::r3, VixlVReg(src1).D());
1951
1952 MakeCall(entryPoint);
1953
1954 auto dstRegister1 = secondValue ? vixl::aarch32::r2 : vixl::aarch32::r0;
1955 auto dstRegister2 = secondValue ? vixl::aarch32::r3 : vixl::aarch32::r1;
1956
1957 GetMasm()->Vmov(VixlVReg(dst).D(), dstRegister1, dstRegister2);
1958 #endif // PANDA_TARGET_ARM32_ABI_HARD
1959 }
1960
MakeLibCallWithInt64Result(Reg dst,Reg src0,Reg src1,void * entryPoint,bool secondValue)1961 void Aarch32Encoder::MakeLibCallWithInt64Result(Reg dst, Reg src0, Reg src1, void *entryPoint, bool secondValue)
1962 {
1963 // Here I look only for this case, because src - is mapped on two regs.
1964 // (INT64_TYPE), and src0 can't be rewrited
1965 // NOTE(igorban) If src0==src1 - the result will be 1 or UB(if src0 = 0)
1966 // It is better to check them in optimizations
1967 // ASSERT(src0 != src1); - do not enable for tests
1968
1969 if (src1.GetId() == vixl::aarch32::r0.GetCode() || src0 == src1) {
1970 ScopedTmpRegU32 tmp1(this);
1971 ScopedTmpRegU32 tmp2(this);
1972 GetMasm()->Mov(VixlReg(tmp1), VixlReg(src1));
1973 GetMasm()->Mov(VixlReg(tmp2), VixlRegU(src1));
1974 GetMasm()->Mov(vixl::aarch32::r0, VixlReg(src0));
1975 GetMasm()->Mov(vixl::aarch32::r1, VixlRegU(src0));
1976 GetMasm()->Mov(vixl::aarch32::r2, VixlReg(tmp1));
1977 GetMasm()->Mov(vixl::aarch32::r3, VixlReg(tmp2));
1978 } else {
1979 GetMasm()->Mov(vixl::aarch32::r0, VixlReg(src0));
1980 GetMasm()->Mov(vixl::aarch32::r1, VixlRegU(src0));
1981 GetMasm()->Mov(vixl::aarch32::r2, VixlReg(src1));
1982 GetMasm()->Mov(vixl::aarch32::r3, VixlRegU(src1));
1983 }
1984
1985 // Call lib-method
1986 MakeCall(entryPoint);
1987
1988 auto dstRegister1 = secondValue ? vixl::aarch32::r2 : vixl::aarch32::r0;
1989 auto dstRegister2 = secondValue ? vixl::aarch32::r3 : vixl::aarch32::r1;
1990
1991 if (dst.GetId() <= vixl::aarch32::r3.GetCode()) {
1992 ScopedTmpRegU32 tmp1(this);
1993 ScopedTmpRegU32 tmp2(this);
1994 GetMasm()->Mov(VixlReg(tmp1), dstRegister1);
1995 GetMasm()->Mov(VixlReg(tmp2), dstRegister2);
1996 GetMasm()->Mov(VixlReg(dst), VixlReg(tmp1));
1997 GetMasm()->Mov(VixlRegU(dst), VixlReg(tmp2));
1998 } else {
1999 GetMasm()->Mov(VixlReg(dst), dstRegister1);
2000 GetMasm()->Mov(VixlRegU(dst), dstRegister2);
2001 }
2002 }
2003
MakeLibCallFromScalarToFloat(Reg dst,Reg src,void * entryPoint)2004 void Aarch32Encoder::MakeLibCallFromScalarToFloat(Reg dst, Reg src, void *entryPoint)
2005 {
2006 if (src.GetSize() != DOUBLE_WORD_SIZE) {
2007 SetFalseResult();
2008 return;
2009 }
2010
2011 bool saveR1 {src.GetId() != vixl::aarch32::r0.GetCode() || dst.GetType() == FLOAT64_TYPE};
2012
2013 GetMasm()->Push(vixl::aarch32::r0);
2014 if (saveR1) {
2015 GetMasm()->Push(vixl::aarch32::r1);
2016 }
2017
2018 if (src.GetId() != vixl::aarch32::r0.GetCode()) {
2019 GetMasm()->Mov(vixl::aarch32::r0, VixlReg(src));
2020 GetMasm()->Mov(vixl::aarch32::r1, VixlRegU(src));
2021 }
2022
2023 MakeCall(entryPoint);
2024
2025 if (dst.GetType() == FLOAT64_TYPE) {
2026 GetMasm()->Vmov(VixlVReg(dst).D(), vixl::aarch32::r0, vixl::aarch32::r1);
2027 } else {
2028 GetMasm()->Vmov(VixlVReg(dst).S(), vixl::aarch32::r0);
2029 }
2030
2031 if (saveR1) {
2032 GetMasm()->Pop(vixl::aarch32::r1);
2033 }
2034 GetMasm()->Pop(vixl::aarch32::r0);
2035 }
2036
MakeLibCallFromFloatToScalar(Reg dst,Reg src,void * entryPoint)2037 void Aarch32Encoder::MakeLibCallFromFloatToScalar(Reg dst, Reg src, void *entryPoint)
2038 {
2039 if (dst.GetSize() != DOUBLE_WORD_SIZE) {
2040 SetFalseResult();
2041 return;
2042 }
2043
2044 bool saveR0R1 {dst.GetId() != vixl::aarch32::r0.GetCode()};
2045
2046 if (saveR0R1) {
2047 GetMasm()->Push(vixl::aarch32::r0);
2048 GetMasm()->Push(vixl::aarch32::r1);
2049 }
2050
2051 if (src.GetType() == FLOAT64_TYPE) {
2052 GetMasm()->Vmov(vixl::aarch32::r0, vixl::aarch32::r1, VixlVReg(src).D());
2053 } else {
2054 GetMasm()->Vmov(vixl::aarch32::r0, VixlVReg(src).S());
2055 }
2056
2057 MakeCall(entryPoint);
2058
2059 if (saveR0R1) {
2060 GetMasm()->Mov(VixlReg(dst), vixl::aarch32::r0);
2061 GetMasm()->Mov(VixlRegU(dst), vixl::aarch32::r1);
2062
2063 GetMasm()->Pop(vixl::aarch32::r1);
2064 GetMasm()->Pop(vixl::aarch32::r0);
2065 }
2066 }
2067
MakeLibCall(Reg dst,Reg src,void * entryPoint)2068 void Aarch32Encoder::MakeLibCall(Reg dst, Reg src, void *entryPoint)
2069 {
2070 if (dst.IsFloat() && src.IsScalar()) {
2071 MakeLibCallFromScalarToFloat(dst, src, entryPoint);
2072 } else if (dst.IsScalar() && src.IsFloat()) {
2073 MakeLibCallFromFloatToScalar(dst, src, entryPoint);
2074 } else {
2075 SetFalseResult();
2076 return;
2077 }
2078 }
2079
EncodeDiv(Reg dst,bool dstSigned,Reg src0,Reg src1)2080 void Aarch32Encoder::EncodeDiv(Reg dst, bool dstSigned, Reg src0, Reg src1)
2081 {
2082 if (dst.IsFloat()) {
2083 GetMasm()->Vdiv(VixlVReg(dst), VixlVReg(src0), VixlVReg(src1));
2084 return;
2085 }
2086
2087 if (dst.GetSize() <= WORD_SIZE) {
2088 if (!dstSigned) {
2089 GetMasm()->Udiv(VixlReg(dst), VixlReg(src0), VixlReg(src1));
2090 return;
2091 }
2092
2093 GetMasm()->Sdiv(VixlReg(dst), VixlReg(src0), VixlReg(src1));
2094 return;
2095 }
2096 if (dstSigned) {
2097 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(AEABIldivmod));
2098 } else {
2099 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(AEABIuldivmod));
2100 }
2101 }
2102
EncodeMod(Reg dst,bool dstSigned,Reg src0,Reg src1)2103 void Aarch32Encoder::EncodeMod(Reg dst, bool dstSigned, Reg src0, Reg src1)
2104 {
2105 if (dst.GetType() == FLOAT32_TYPE) {
2106 using Fp = float (*)(float, float);
2107 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(static_cast<Fp>(fmodf)));
2108 return;
2109 }
2110 if (dst.GetType() == FLOAT64_TYPE) {
2111 using Fp = double (*)(double, double);
2112 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(static_cast<Fp>(fmod)));
2113 return;
2114 }
2115
2116 if (dst.GetSize() <= WORD_SIZE) {
2117 if (dstSigned) {
2118 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(AEABIidivmod), true);
2119 } else {
2120 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(AEABIuidivmod), true);
2121 }
2122
2123 // dst = -(tmp * src0) + src1
2124 if ((dst.GetSize() == BYTE_SIZE) && dstSigned) {
2125 GetMasm()->Sxtb(VixlReg(dst), VixlReg(dst));
2126 }
2127 if ((dst.GetSize() == HALF_SIZE) && dstSigned) {
2128 GetMasm()->Sxth(VixlReg(dst), VixlReg(dst));
2129 }
2130 return;
2131 }
2132
2133 // Call lib-method
2134 if (dstSigned) {
2135 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(AEABIldivmod), true);
2136 } else {
2137 MakeLibCall(dst, src0, src1, reinterpret_cast<void *>(AEABIuldivmod), true);
2138 }
2139 }
2140
EncodeMin(Reg dst,bool dstSigned,Reg src0,Reg src1)2141 void Aarch32Encoder::EncodeMin(Reg dst, bool dstSigned, Reg src0, Reg src1)
2142 {
2143 if (dst.IsFloat()) {
2144 EncodeMinMaxFp<false>(dst, src0, src1);
2145 return;
2146 }
2147
2148 if (dst.GetSize() <= WORD_SIZE) {
2149 if (dstSigned) {
2150 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
2151 GetMasm()->Mov(Convert(Condition::LE), VixlReg(dst), VixlReg(src0));
2152 GetMasm()->Mov(Convert(Condition::GT), VixlReg(dst), VixlReg(src1));
2153 return;
2154 }
2155 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
2156 GetMasm()->Mov(Convert(Condition::LS), VixlReg(dst), VixlReg(src0));
2157 GetMasm()->Mov(Convert(Condition::HI), VixlReg(dst), VixlReg(src1));
2158 return;
2159 }
2160
2161 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
2162 ScopedTmpRegU32 tmpReg(this);
2163
2164 GetMasm()->Subs(VixlReg(tmpReg), VixlReg(src0), VixlReg(src1));
2165 GetMasm()->Sbcs(VixlReg(tmpReg), VixlRegU(src0), VixlRegU(src1));
2166
2167 auto cc = Convert(dstSigned ? Condition::LT : Condition::LO);
2168 GetMasm()->Mov(cc, VixlReg(dst), VixlReg(src0));
2169 GetMasm()->Mov(cc, VixlRegU(dst), VixlRegU(src0));
2170 GetMasm()->Mov(cc.Negate(), VixlReg(dst), VixlReg(src1));
2171 GetMasm()->Mov(cc.Negate(), VixlRegU(dst), VixlRegU(src1));
2172 }
2173
EncodeMax(Reg dst,bool dstSigned,Reg src0,Reg src1)2174 void Aarch32Encoder::EncodeMax(Reg dst, bool dstSigned, Reg src0, Reg src1)
2175 {
2176 if (dst.IsFloat()) {
2177 EncodeMinMaxFp<true>(dst, src0, src1);
2178 return;
2179 }
2180
2181 if (dst.GetSize() <= WORD_SIZE) {
2182 if (dstSigned) {
2183 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
2184 GetMasm()->Mov(Convert(Condition::GT), VixlReg(dst), VixlReg(src0));
2185 GetMasm()->Mov(Convert(Condition::LE), VixlReg(dst), VixlReg(src1));
2186 return;
2187 }
2188 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
2189 GetMasm()->Mov(Convert(Condition::HI), VixlReg(dst), VixlReg(src0));
2190 GetMasm()->Mov(Convert(Condition::LS), VixlReg(dst), VixlReg(src1));
2191 return;
2192 }
2193
2194 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
2195 ScopedTmpRegU32 tmpReg(this);
2196
2197 GetMasm()->Subs(VixlReg(tmpReg), VixlReg(src0), VixlReg(src1));
2198 GetMasm()->Sbcs(VixlReg(tmpReg), VixlRegU(src0), VixlRegU(src1));
2199
2200 auto cc = Convert(dstSigned ? Condition::LT : Condition::LO);
2201 GetMasm()->Mov(cc, VixlReg(dst), VixlReg(src1));
2202 GetMasm()->Mov(cc, VixlRegU(dst), VixlRegU(src1));
2203 GetMasm()->Mov(cc.Negate(), VixlReg(dst), VixlReg(src0));
2204 GetMasm()->Mov(cc.Negate(), VixlRegU(dst), VixlRegU(src0));
2205 }
2206
2207 template <bool IS_MAX>
EncodeMinMaxFp(Reg dst,Reg src0,Reg src1)2208 void Aarch32Encoder::EncodeMinMaxFp(Reg dst, Reg src0, Reg src1)
2209 {
2210 Aarch32LabelHolder::LabelType notEqual(GetAllocator());
2211 Aarch32LabelHolder::LabelType gotNan(GetAllocator());
2212 Aarch32LabelHolder::LabelType end(GetAllocator());
2213 auto &srcA = dst.GetId() != src1.GetId() ? src0 : src1;
2214 auto &srcB = srcA.GetId() == src0.GetId() ? src1 : src0;
2215 GetMasm()->Vmov(VixlVReg(dst), VixlVReg(srcA));
2216 // Vcmp change flags:
2217 // NZCV
2218 // 0011 <- if any operand is NaN
2219 // 0110 <- operands are equals
2220 // 1000 <- operand0 < operand1
2221 // 0010 <- operand0 > operand1
2222 GetMasm()->Vcmp(VixlVReg(srcB), VixlVReg(srcA));
2223 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
2224 GetMasm()->B(Convert(Condition::VS), &gotNan);
2225 GetMasm()->B(Convert(Condition::NE), ¬Equal);
2226
2227 // calculate result for positive/negative zero operands
2228 if (IS_MAX) {
2229 EncodeVand(dst, srcA, srcB);
2230 } else {
2231 EncodeVorr(dst, srcA, srcB);
2232 }
2233 GetMasm()->B(&end);
2234 GetMasm()->Bind(&gotNan);
2235 // if any operand is NaN result is NaN
2236 EncodeVorr(dst, srcA, srcB);
2237 GetMasm()->B(&end);
2238 GetMasm()->bind(¬Equal);
2239 // calculate min/max for other cases
2240 if (IS_MAX) {
2241 GetMasm()->B(Convert(Condition::MI), &end);
2242 } else {
2243 GetMasm()->B(Convert(Condition::HI), &end);
2244 }
2245 GetMasm()->Vmov(VixlVReg(dst), VixlVReg(srcB));
2246 GetMasm()->bind(&end);
2247 }
2248
EncodeVorr(Reg dst,Reg src0,Reg src1)2249 void Aarch32Encoder::EncodeVorr(Reg dst, Reg src0, Reg src1)
2250 {
2251 if (dst.GetType() == FLOAT32_TYPE) {
2252 ScopedTmpRegF64 tmpReg(this);
2253 GetMasm()->Vmov(vixl::aarch32::SRegister(tmpReg.GetReg().GetId() + (src0.GetId() & 1U)), VixlVReg(src1).S());
2254 GetMasm()->Vorr(vixl::aarch32::DRegister(tmpReg.GetReg().GetId() / 2U),
2255 vixl::aarch32::DRegister(src0.GetId() / 2U),
2256 vixl::aarch32::DRegister(tmpReg.GetReg().GetId() / 2U));
2257 GetMasm()->Vmov(VixlVReg(dst).S(), vixl::aarch32::SRegister(tmpReg.GetReg().GetId() + (src0.GetId() & 1U)));
2258 } else {
2259 GetMasm()->Vorr(VixlVReg(dst).D(), VixlVReg(src0).D(), VixlVReg(src1).D());
2260 }
2261 }
2262
EncodeVand(Reg dst,Reg src0,Reg src1)2263 void Aarch32Encoder::EncodeVand(Reg dst, Reg src0, Reg src1)
2264 {
2265 if (dst.GetType() == FLOAT32_TYPE) {
2266 ScopedTmpRegF64 tmpReg(this);
2267 GetMasm()->Vmov(vixl::aarch32::SRegister(tmpReg.GetReg().GetId() + (src0.GetId() & 1U)), VixlVReg(src1).S());
2268 GetMasm()->Vand(vixl::aarch32::kDataTypeValueNone, vixl::aarch32::DRegister(tmpReg.GetReg().GetId() / 2U),
2269 vixl::aarch32::DRegister(src0.GetId() / 2U),
2270 vixl::aarch32::DRegister(tmpReg.GetReg().GetId() / 2U));
2271 GetMasm()->Vmov(VixlVReg(dst).S(), vixl::aarch32::SRegister(tmpReg.GetReg().GetId() + (src0.GetId() & 1U)));
2272 } else {
2273 GetMasm()->Vand(vixl::aarch32::kDataTypeValueNone, VixlVReg(dst).D(), VixlVReg(src0).D(), VixlVReg(src1).D());
2274 }
2275 }
2276
EncodeShl(Reg dst,Reg src0,Reg src1)2277 void Aarch32Encoder::EncodeShl(Reg dst, Reg src0, Reg src1)
2278 {
2279 if (dst.GetSize() < WORD_SIZE) {
2280 GetMasm()->And(VixlReg(src1), VixlReg(src1), VixlImm(dst.GetSize() - 1));
2281 }
2282
2283 if (dst.GetSize() <= WORD_SIZE) {
2284 GetMasm()->Lsl(VixlReg(dst), VixlReg(src0), VixlReg(src1));
2285 return;
2286 }
2287
2288 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
2289 ScopedTmpRegU32 hiReg(this);
2290 ScopedTmpRegU32 tmpReg(this);
2291
2292 GetMasm()->Rsb(VixlReg(tmpReg), VixlReg(src1), VixlImm(WORD_SIZE));
2293 GetMasm()->Lsr(VixlReg(tmpReg), VixlReg(src0), VixlReg(tmpReg));
2294 GetMasm()->Orr(VixlReg(hiReg), VixlReg(tmpReg),
2295 vixl::aarch32::Operand(VixlRegU(src0), vixl::aarch32::LSL, VixlReg(src1)));
2296 GetMasm()->Subs(VixlReg(tmpReg), VixlReg(src1), VixlImm(WORD_SIZE));
2297 GetMasm()->Lsl(Convert(Condition::PL), VixlReg(hiReg), VixlReg(src0), VixlReg(tmpReg));
2298 GetMasm()->Mov(Convert(Condition::PL), VixlReg(dst), VixlImm(0x0));
2299 GetMasm()->Lsl(Convert(Condition::MI), VixlReg(dst), VixlReg(src0), VixlReg(src1));
2300 GetMasm()->Mov(VixlRegU(dst), VixlReg(hiReg));
2301 }
2302
EncodeShr(Reg dst,Reg src0,Reg src1)2303 void Aarch32Encoder::EncodeShr(Reg dst, Reg src0, Reg src1)
2304 {
2305 if (dst.GetSize() < WORD_SIZE) {
2306 GetMasm()->And(VixlReg(src1), VixlReg(src1), VixlImm(dst.GetSize() - 1));
2307 }
2308
2309 if (dst.GetSize() <= WORD_SIZE) {
2310 GetMasm()->Lsr(VixlReg(dst), VixlReg(src0), VixlReg(src1));
2311 return;
2312 }
2313
2314 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
2315 ScopedTmpRegU32 loReg(this);
2316 ScopedTmpRegU32 tmpReg(this);
2317
2318 GetMasm()->Rsb(VixlReg(tmpReg), VixlReg(src1), VixlImm(WORD_SIZE));
2319 GetMasm()->Lsr(VixlReg(loReg), VixlReg(src0), VixlReg(src1));
2320 GetMasm()->Orr(VixlReg(loReg), VixlReg(loReg),
2321 vixl::aarch32::Operand(VixlRegU(src0), vixl::aarch32::LSL, VixlReg(tmpReg)));
2322 GetMasm()->Subs(VixlReg(tmpReg), VixlReg(src1), VixlImm(WORD_SIZE));
2323 GetMasm()->Lsr(Convert(Condition::PL), VixlReg(loReg), VixlRegU(src0), VixlReg(tmpReg));
2324 GetMasm()->Mov(Convert(Condition::PL), VixlRegU(dst), VixlImm(0x0));
2325 GetMasm()->Lsr(Convert(Condition::MI), VixlRegU(dst), VixlRegU(src0), VixlReg(src1));
2326 GetMasm()->Mov(VixlReg(dst), VixlReg(loReg));
2327 }
2328
EncodeAShr(Reg dst,Reg src0,Reg src1)2329 void Aarch32Encoder::EncodeAShr(Reg dst, Reg src0, Reg src1)
2330 {
2331 if (dst.GetSize() < WORD_SIZE) {
2332 GetMasm()->And(VixlReg(src1), VixlReg(src1), VixlImm(dst.GetSize() - 1));
2333 }
2334
2335 if (dst.GetSize() <= WORD_SIZE) {
2336 GetMasm()->Asr(VixlReg(dst), VixlReg(src0), VixlReg(src1));
2337 return;
2338 }
2339
2340 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
2341 ScopedTmpRegU32 loReg(this);
2342 ScopedTmpRegU32 tmpReg(this);
2343
2344 GetMasm()->Subs(VixlReg(tmpReg), VixlReg(src1), VixlImm(WORD_SIZE));
2345 GetMasm()->Lsr(VixlReg(loReg), VixlReg(src0), VixlReg(src1));
2346 GetMasm()->Rsb(VixlReg(tmpReg), VixlReg(src1), VixlImm(WORD_SIZE));
2347 GetMasm()->Orr(VixlReg(loReg), VixlReg(loReg),
2348 vixl::aarch32::Operand(VixlRegU(src0), vixl::aarch32::LSL, VixlReg(tmpReg)));
2349 GetMasm()->Rsb(Convert(Condition::PL), VixlReg(tmpReg), VixlReg(tmpReg), VixlImm(0x0));
2350 GetMasm()->Asr(Convert(Condition::PL), VixlReg(loReg), VixlRegU(src0), VixlReg(tmpReg));
2351 GetMasm()->Asr(Convert(Condition::PL), VixlRegU(dst), VixlRegU(src0), VixlImm(WORD_SIZE - 1));
2352 GetMasm()->Asr(Convert(Condition::MI), VixlRegU(dst), VixlRegU(src0), VixlReg(src1));
2353 GetMasm()->Mov(VixlReg(dst), VixlReg(loReg));
2354 }
2355
EncodeAnd(Reg dst,Reg src0,Reg src1)2356 void Aarch32Encoder::EncodeAnd(Reg dst, Reg src0, Reg src1)
2357 {
2358 GetMasm()->And(VixlReg(dst), VixlReg(src0), VixlReg(src1));
2359 if (dst.GetSize() > WORD_SIZE) {
2360 GetMasm()->And(VixlRegU(dst), VixlRegU(src0), VixlRegU(src1));
2361 }
2362 }
2363
EncodeOr(Reg dst,Reg src0,Reg src1)2364 void Aarch32Encoder::EncodeOr(Reg dst, Reg src0, Reg src1)
2365 {
2366 GetMasm()->Orr(VixlReg(dst), VixlReg(src0), VixlReg(src1));
2367 if (dst.GetSize() > WORD_SIZE) {
2368 GetMasm()->Orr(VixlRegU(dst), VixlRegU(src0), VixlRegU(src1));
2369 }
2370 }
2371
EncodeXor(Reg dst,Reg src0,Reg src1)2372 void Aarch32Encoder::EncodeXor(Reg dst, Reg src0, Reg src1)
2373 {
2374 GetMasm()->Eor(VixlReg(dst), VixlReg(src0), VixlReg(src1));
2375 if (dst.GetSize() > WORD_SIZE) {
2376 GetMasm()->Eor(VixlRegU(dst), VixlRegU(src0), VixlRegU(src1));
2377 }
2378 }
2379
EncodeAdd(Reg dst,Reg src,Imm imm)2380 void Aarch32Encoder::EncodeAdd(Reg dst, Reg src, Imm imm)
2381 {
2382 ASSERT(dst.IsScalar() && "UNIMPLEMENTED");
2383 if (dst.GetSize() <= WORD_SIZE) {
2384 GetMasm()->Add(VixlReg(dst), VixlReg(src), VixlImm(imm));
2385 return;
2386 }
2387
2388 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
2389 GetMasm()->Adds(VixlReg(dst), VixlReg(src), VixlImm(imm));
2390 GetMasm()->Adc(VixlRegU(dst), VixlRegU(src), VixlImmU(imm));
2391 }
2392
EncodeSub(Reg dst,Reg src,Imm imm)2393 void Aarch32Encoder::EncodeSub(Reg dst, Reg src, Imm imm)
2394 {
2395 ASSERT(dst.IsScalar() && "UNIMPLEMENTED");
2396 if (dst.GetSize() <= WORD_SIZE) {
2397 GetMasm()->Sub(VixlReg(dst), VixlReg(src), VixlImm(imm));
2398 return;
2399 }
2400
2401 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
2402 GetMasm()->Subs(VixlReg(dst), VixlReg(src), VixlImm(imm));
2403 GetMasm()->Sbc(VixlRegU(dst), VixlRegU(src), VixlImmU(imm));
2404 }
2405
EncodeShl(Reg dst,Reg src,Imm imm)2406 void Aarch32Encoder::EncodeShl(Reg dst, Reg src, Imm imm)
2407 {
2408 auto value = static_cast<uint32_t>(imm.GetAsInt());
2409 int32_t immValue = value & (dst.GetSize() - 1);
2410
2411 ASSERT(dst.IsScalar() && "Invalid operand type");
2412 if (dst.GetSize() <= WORD_SIZE) {
2413 GetMasm()->Lsl(VixlReg(dst), VixlReg(src), VixlImm(immValue));
2414 return;
2415 }
2416
2417 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
2418 ScopedTmpRegU32 hiReg(this);
2419 ScopedTmpRegU32 tmpReg(this);
2420
2421 GetMasm()->Lsr(VixlReg(tmpReg), VixlReg(src), VixlImm(WORD_SIZE - immValue));
2422 GetMasm()->Mov(VixlReg(hiReg), VixlImm(immValue));
2423 GetMasm()->Orr(VixlReg(hiReg), VixlReg(tmpReg),
2424 vixl::aarch32::Operand(VixlRegU(src), vixl::aarch32::LSL, VixlReg(hiReg)));
2425 GetMasm()->Movs(VixlReg(tmpReg), VixlImm(immValue - WORD_SIZE));
2426 GetMasm()->Lsl(Convert(Condition::PL), VixlReg(hiReg), VixlReg(src), VixlReg(tmpReg));
2427 GetMasm()->Mov(Convert(Condition::PL), VixlReg(dst), VixlImm(0x0));
2428 GetMasm()->Lsl(Convert(Condition::MI), VixlReg(dst), VixlReg(src), VixlImm(immValue));
2429 GetMasm()->Mov(VixlRegU(dst), VixlReg(hiReg));
2430 }
2431
EncodeShr(Reg dst,Reg src,Imm imm)2432 void Aarch32Encoder::EncodeShr(Reg dst, Reg src, Imm imm)
2433 {
2434 auto value = static_cast<uint32_t>(imm.GetAsInt());
2435 int32_t immValue = value & (dst.GetSize() - 1);
2436
2437 ASSERT(dst.IsScalar() && "Invalid operand type");
2438 if (dst.GetSize() <= WORD_SIZE) {
2439 GetMasm()->Lsr(VixlReg(dst), VixlReg(src), immValue);
2440 return;
2441 }
2442
2443 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
2444 ScopedTmpRegU32 loReg(this);
2445 ScopedTmpRegU32 tmpReg(this);
2446
2447 GetMasm()->Mov(VixlReg(tmpReg), VixlImm(WORD_SIZE - immValue));
2448 GetMasm()->Lsr(VixlReg(loReg), VixlReg(src), VixlImm(immValue));
2449 GetMasm()->Orr(VixlReg(loReg), VixlReg(loReg),
2450 vixl::aarch32::Operand(VixlRegU(src), vixl::aarch32::LSL, VixlReg(tmpReg)));
2451 GetMasm()->Movs(VixlReg(tmpReg), VixlImm(immValue - WORD_SIZE));
2452 GetMasm()->Lsr(Convert(Condition::PL), VixlReg(loReg), VixlRegU(src), VixlReg(tmpReg));
2453 GetMasm()->Mov(Convert(Condition::PL), VixlRegU(dst), VixlImm(0x0));
2454 GetMasm()->Lsr(Convert(Condition::MI), VixlRegU(dst), VixlRegU(src), VixlImm(immValue));
2455 GetMasm()->Mov(VixlReg(dst), VixlReg(loReg));
2456 }
2457
EncodeAShr(Reg dst,Reg src,Imm imm)2458 void Aarch32Encoder::EncodeAShr(Reg dst, Reg src, Imm imm)
2459 {
2460 ASSERT(dst.IsScalar() && "Invalid operand type");
2461
2462 auto value = static_cast<uint32_t>(imm.GetAsInt());
2463 int32_t immValue = value & (dst.GetSize() - 1);
2464
2465 if (dst.GetSize() <= WORD_SIZE) {
2466 GetMasm()->Asr(VixlReg(dst), VixlReg(src), immValue);
2467 return;
2468 }
2469
2470 ScopedTmpRegU32 loReg(this);
2471 ScopedTmpRegU32 tmpReg(this);
2472 GetMasm()->Movs(VixlReg(tmpReg), VixlImm(immValue - WORD_SIZE));
2473 GetMasm()->Lsr(VixlReg(loReg), VixlReg(src), VixlImm(immValue));
2474 GetMasm()->Mov(VixlReg(tmpReg), VixlImm(WORD_SIZE - immValue));
2475 GetMasm()->Orr(VixlReg(loReg), VixlReg(loReg),
2476 vixl::aarch32::Operand(VixlRegU(src), vixl::aarch32::LSL, VixlReg(tmpReg)));
2477 GetMasm()->Rsb(Convert(Condition::PL), VixlReg(tmpReg), VixlReg(tmpReg), VixlImm(0x0));
2478 GetMasm()->Asr(Convert(Condition::PL), VixlReg(loReg), VixlRegU(src), VixlReg(tmpReg));
2479 GetMasm()->Asr(Convert(Condition::PL), VixlRegU(dst), VixlRegU(src), VixlImm(WORD_SIZE - 1));
2480 GetMasm()->Asr(Convert(Condition::MI), VixlRegU(dst), VixlRegU(src), VixlImm(immValue));
2481 GetMasm()->Mov(VixlReg(dst), VixlReg(loReg));
2482 }
2483
EncodeAnd(Reg dst,Reg src,Imm imm)2484 void Aarch32Encoder::EncodeAnd(Reg dst, Reg src, Imm imm)
2485 {
2486 ASSERT(dst.IsScalar() && "Invalid operand type");
2487 GetMasm()->And(VixlReg(dst), VixlReg(src), VixlImm(imm));
2488 if (dst.GetSize() > WORD_SIZE) {
2489 GetMasm()->And(VixlRegU(dst), VixlRegU(src), VixlImmU(imm));
2490 }
2491 }
2492
EncodeOr(Reg dst,Reg src,Imm imm)2493 void Aarch32Encoder::EncodeOr(Reg dst, Reg src, Imm imm)
2494 {
2495 ASSERT(dst.IsScalar() && "Invalid operand type");
2496 GetMasm()->Orr(VixlReg(dst), VixlReg(src), VixlImm(imm));
2497 if (dst.GetSize() > WORD_SIZE) {
2498 GetMasm()->Orr(VixlRegU(dst), VixlRegU(src), VixlImmU(imm));
2499 }
2500 }
2501
EncodeXor(Reg dst,Reg src,Imm imm)2502 void Aarch32Encoder::EncodeXor(Reg dst, Reg src, Imm imm)
2503 {
2504 ASSERT(dst.IsScalar() && "Invalid operand type");
2505 GetMasm()->Eor(VixlReg(dst), VixlReg(src), VixlImm(imm));
2506 if (dst.GetSize() > WORD_SIZE) {
2507 GetMasm()->Eor(VixlRegU(dst), VixlRegU(src), VixlImmU(imm));
2508 }
2509 }
2510
EncodeMov(Reg dst,Imm src)2511 void Aarch32Encoder::EncodeMov(Reg dst, Imm src)
2512 {
2513 if (dst.IsFloat()) {
2514 if (dst.GetSize() == WORD_SIZE) {
2515 GetMasm()->Vmov(Convert(dst.GetType()), VixlVReg(dst).S(), VixlNeonImm(src.GetAsFloat()));
2516 } else {
2517 GetMasm()->Vmov(Convert(dst.GetType()), VixlVReg(dst).D(), VixlNeonImm(src.GetAsDouble()));
2518 }
2519 return;
2520 }
2521
2522 GetMasm()->Mov(VixlReg(dst), VixlImm(src));
2523 if (dst.GetSize() > WORD_SIZE) {
2524 GetMasm()->Mov(VixlRegU(dst), VixlImmU(src));
2525 }
2526 }
2527
EncodeLdr(Reg dst,bool dstSigned,const vixl::aarch32::MemOperand & vixlMem)2528 void Aarch32Encoder::EncodeLdr(Reg dst, bool dstSigned, const vixl::aarch32::MemOperand &vixlMem)
2529 {
2530 if (dst.IsFloat()) {
2531 if (dst.GetSize() == WORD_SIZE) {
2532 GetMasm()->Vldr(VixlVReg(dst).S(), vixlMem);
2533 } else {
2534 GetMasm()->Vldr(VixlVReg(dst).D(), vixlMem);
2535 }
2536 return;
2537 }
2538 if (dstSigned) {
2539 if (dst.GetSize() == BYTE_SIZE) {
2540 GetMasm()->Ldrsb(VixlReg(dst), vixlMem);
2541 return;
2542 }
2543 if (dst.GetSize() == HALF_SIZE) {
2544 GetMasm()->Ldrsh(VixlReg(dst), vixlMem);
2545 return;
2546 }
2547 } else {
2548 if (dst.GetSize() == BYTE_SIZE) {
2549 GetMasm()->Ldrb(VixlReg(dst), vixlMem);
2550 return;
2551 }
2552 if (dst.GetSize() == HALF_SIZE) {
2553 GetMasm()->Ldrh(VixlReg(dst), vixlMem);
2554 return;
2555 }
2556 }
2557 if (dst.GetSize() == WORD_SIZE) {
2558 GetMasm()->Ldr(VixlReg(dst), vixlMem);
2559 return;
2560 }
2561 ASSERT(dst.GetSize() == DOUBLE_WORD_SIZE);
2562 GetMasm()->Ldrd(VixlReg(dst), VixlRegU(dst), vixlMem);
2563 }
2564
EncodeLdr(Reg dst,bool dstSigned,MemRef mem)2565 void Aarch32Encoder::EncodeLdr(Reg dst, bool dstSigned, MemRef mem)
2566 {
2567 auto type = dst.GetType();
2568 if (IsNeedToPrepareMemLdS(mem, type, dstSigned)) {
2569 ScopedTmpRegU32 tmpReg(this);
2570 auto tmp = VixlReg(tmpReg);
2571 auto vixlMem = PrepareMemLdS(mem, type, tmp, dstSigned);
2572 EncodeLdr(dst, dstSigned, vixlMem);
2573 } else {
2574 auto vixlMem = ConvertMem(mem);
2575 EncodeLdr(dst, dstSigned, vixlMem);
2576 }
2577 }
2578
EncodeLdrAcquire(Reg dst,bool dstSigned,MemRef mem)2579 void Aarch32Encoder::EncodeLdrAcquire(Reg dst, bool dstSigned, MemRef mem)
2580 {
2581 EncodeLdr(dst, dstSigned, mem);
2582 GetMasm()->Dmb(vixl::aarch32::MemoryBarrierType::ISH);
2583 }
2584
EncodeMemoryBarrier(memory_order::Order order)2585 void Aarch32Encoder::EncodeMemoryBarrier(memory_order::Order order)
2586 {
2587 switch (order) {
2588 case memory_order::ACQUIRE:
2589 case memory_order::RELEASE: {
2590 GetMasm()->Dmb(vixl::aarch32::MemoryBarrierType::ISH);
2591 break;
2592 }
2593 case memory_order::FULL: {
2594 GetMasm()->Dmb(vixl::aarch32::MemoryBarrierType::ISHST);
2595 break;
2596 }
2597 default:
2598 break;
2599 }
2600 }
2601
EncodeStr(Reg src,const vixl::aarch32::MemOperand & vixlMem)2602 void Aarch32Encoder::EncodeStr(Reg src, const vixl::aarch32::MemOperand &vixlMem)
2603 {
2604 if (src.IsFloat()) {
2605 if (src.GetSize() == WORD_SIZE) {
2606 GetMasm()->Vstr(VixlVReg(src).S(), vixlMem);
2607 } else {
2608 GetMasm()->Vstr(VixlVReg(src).D(), vixlMem);
2609 }
2610 } else if (src.GetSize() == BYTE_SIZE) {
2611 GetMasm()->Strb(VixlReg(src), vixlMem);
2612 } else if (src.GetSize() == HALF_SIZE) {
2613 GetMasm()->Strh(VixlReg(src), vixlMem);
2614 } else if (src.GetSize() == WORD_SIZE) {
2615 GetMasm()->Str(VixlReg(src), vixlMem);
2616 } else {
2617 ASSERT(src.GetSize() == DOUBLE_WORD_SIZE);
2618 GetMasm()->Strd(VixlReg(src), VixlRegU(src), vixlMem);
2619 }
2620 }
2621
EncodeStr(Reg src,MemRef mem)2622 void Aarch32Encoder::EncodeStr(Reg src, MemRef mem)
2623 {
2624 auto type = src.GetType();
2625 if (IsNeedToPrepareMemLdS(mem, type, false)) {
2626 ScopedTmpRegU32 tmpReg(this);
2627 auto tmp = VixlReg(tmpReg);
2628 auto vixlMem = PrepareMemLdS(mem, type, tmp, false);
2629 EncodeStr(src, vixlMem);
2630 } else {
2631 auto vixlMem = ConvertMem(mem);
2632 EncodeStr(src, vixlMem);
2633 }
2634 }
2635
EncodeStrRelease(Reg src,MemRef mem)2636 void Aarch32Encoder::EncodeStrRelease(Reg src, MemRef mem)
2637 {
2638 GetMasm()->Dmb(vixl::aarch32::MemoryBarrierType::ISH);
2639 EncodeStr(src, mem);
2640 GetMasm()->Dmb(vixl::aarch32::MemoryBarrierType::ISH);
2641 }
2642
EncodeStrz(Reg src,MemRef mem)2643 void Aarch32Encoder::EncodeStrz(Reg src, MemRef mem)
2644 {
2645 if (src.GetSize() <= WORD_SIZE) {
2646 EncodeSti(0, DOUBLE_WORD_SIZE_BYTES, mem);
2647 }
2648 EncodeStr(src, mem);
2649 }
2650
EncodeStp(Reg src0,Reg src1,MemRef mem)2651 void Aarch32Encoder::EncodeStp(Reg src0, Reg src1, MemRef mem)
2652 {
2653 ASSERT(src0.IsFloat() == src1.IsFloat());
2654 ASSERT(src0.GetSize() == src1.GetSize());
2655 EncodeStr(src0, mem);
2656 EncodeStr(src1, MemRef(mem.GetBase(), mem.GetIndex(), mem.GetScale(), mem.GetDisp() + WORD_SIZE_BYTES));
2657 }
2658
EncodeLdrExclusive(Reg dst,Reg addr,bool acquire)2659 void Aarch32Encoder::EncodeLdrExclusive(Reg dst, Reg addr, bool acquire)
2660 {
2661 ASSERT(dst.IsScalar());
2662 auto dstReg = VixlReg(dst);
2663 auto memCvt = ConvertMem(MemRef(addr));
2664 if (dst.GetSize() == BYTE_SIZE) {
2665 if (acquire) {
2666 GetMasm()->Ldaexb(dstReg, memCvt);
2667 return;
2668 }
2669 GetMasm()->Ldrexb(dstReg, memCvt);
2670 return;
2671 }
2672 if (dst.GetSize() == HALF_SIZE) {
2673 if (acquire) {
2674 GetMasm()->Ldaexh(dstReg, memCvt);
2675 return;
2676 }
2677 GetMasm()->Ldrexh(dstReg, memCvt);
2678 return;
2679 }
2680 if (dst.GetSize() == DOUBLE_WORD_SIZE) {
2681 auto dstRegU = VixlRegU(dst);
2682 if (acquire) {
2683 GetMasm()->Ldaexd(dstReg, dstRegU, memCvt);
2684 return;
2685 }
2686 GetMasm()->Ldrexd(dstReg, dstRegU, memCvt);
2687 return;
2688 }
2689 if (acquire) {
2690 GetMasm()->Ldaex(dstReg, memCvt);
2691 return;
2692 }
2693 GetMasm()->Ldrex(dstReg, memCvt);
2694 }
2695
EncodeStrExclusive(Reg dst,Reg src,Reg addr,bool release)2696 void Aarch32Encoder::EncodeStrExclusive(Reg dst, Reg src, Reg addr, bool release)
2697 {
2698 ASSERT(dst.IsScalar() && src.IsScalar());
2699 ASSERT(dst.GetSize() != DOUBLE_WORD_SIZE);
2700
2701 bool copyDst = dst.GetId() == src.GetId() || dst.GetId() == addr.GetId();
2702 ScopedTmpReg tmp(this);
2703 auto dstReg = copyDst ? VixlReg(tmp) : VixlReg(dst);
2704 auto srcReg = VixlReg(src);
2705 auto memCvt = ConvertMem(MemRef(addr));
2706
2707 if (src.GetSize() == BYTE_SIZE) {
2708 if (release) {
2709 GetMasm()->Stlexb(dstReg, srcReg, memCvt);
2710 } else {
2711 GetMasm()->Strexb(dstReg, srcReg, memCvt);
2712 }
2713 } else if (src.GetSize() == HALF_SIZE) {
2714 if (release) {
2715 GetMasm()->Stlexh(dstReg, srcReg, memCvt);
2716 } else {
2717 GetMasm()->Strexh(dstReg, srcReg, memCvt);
2718 }
2719 } else if (src.GetSize() == DOUBLE_WORD_SIZE) {
2720 auto srcRegU = VixlRegU(src);
2721 if (release) {
2722 GetMasm()->Stlexd(dstReg, srcReg, srcRegU, memCvt);
2723 } else {
2724 GetMasm()->Strexd(dstReg, srcReg, srcRegU, memCvt);
2725 }
2726 } else {
2727 if (release) {
2728 GetMasm()->Stlex(dstReg, srcReg, memCvt);
2729 } else {
2730 GetMasm()->Strex(dstReg, srcReg, memCvt);
2731 }
2732 }
2733
2734 if (copyDst) {
2735 EncodeMov(dst, tmp);
2736 }
2737 }
2738
FindRegForMem(vixl::aarch32::MemOperand mem)2739 static int32_t FindRegForMem(vixl::aarch32::MemOperand mem)
2740 {
2741 int32_t baseRegId = mem.GetBaseRegister().GetCode();
2742 int32_t indexRegId = -1;
2743 if (mem.IsShiftedRegister()) {
2744 indexRegId = mem.GetOffsetRegister().GetCode();
2745 }
2746 // find regs for mem
2747 constexpr int32_t STEP = 2;
2748 for (int32_t i = 0; i < static_cast<int32_t>(BYTE_SIZE); i += STEP) {
2749 if (baseRegId == i || baseRegId == i + 1 || indexRegId == i || indexRegId == i + 1) {
2750 continue;
2751 }
2752 return i;
2753 }
2754 UNREACHABLE();
2755 return -1;
2756 }
2757
EncodeSti(int64_t src,uint8_t srcSizeBytes,MemRef mem)2758 void Aarch32Encoder::EncodeSti(int64_t src, uint8_t srcSizeBytes, MemRef mem)
2759 {
2760 ScopedTmpRegU32 tmpReg(this);
2761 auto tmp = VixlReg(tmpReg);
2762 auto type = TypeInfo::GetScalarTypeBySize(srcSizeBytes * BITS_PER_BYTE);
2763 if (srcSizeBytes <= WORD_SIZE_BYTES) {
2764 auto vixlMem = PrepareMemLdS(mem, type, tmp, false);
2765 if (vixlMem.GetBaseRegister().GetCode() == tmp.GetCode()) {
2766 ScopedTmpRegU32 tmp1Reg(this);
2767 tmp = VixlReg(tmp1Reg);
2768 }
2769 GetMasm()->Mov(tmp, VixlImm(src));
2770 if (srcSizeBytes == 1) {
2771 GetMasm()->Strb(tmp, vixlMem);
2772 return;
2773 }
2774 if (srcSizeBytes == HALF_WORD_SIZE_BYTES) {
2775 GetMasm()->Strh(tmp, vixlMem);
2776 return;
2777 }
2778 GetMasm()->Str(tmp, vixlMem);
2779 return;
2780 }
2781
2782 auto vixlMem = PrepareMemLdS(mem, type, tmp, false, true);
2783 ASSERT(srcSizeBytes == DOUBLE_WORD_SIZE_BYTES);
2784 vixl::aarch32::Register tmpImm1;
2785 vixl::aarch32::Register tmpImm2;
2786 // if tmp isn't base reg and tmp is even and tmp+1 isn't SP we can use tmp and tmp + 1
2787 if (vixlMem.GetBaseRegister().GetCode() != tmp.GetCode() && (tmp.GetCode() % 2U == 0) &&
2788 tmp.GetCode() + 1 != vixl::aarch32::sp.GetCode()) {
2789 tmpImm1 = tmp;
2790 tmpImm2 = vixl::aarch32::Register(tmp.GetCode() + 1);
2791 } else {
2792 auto regId = FindRegForMem(vixlMem);
2793 ASSERT(regId != -1);
2794 tmpImm1 = vixl::aarch32::Register(regId);
2795 tmpImm2 = vixl::aarch32::Register(regId + 1);
2796 GetMasm()->Push(tmpImm1);
2797 }
2798
2799 ASSERT(tmpImm1.IsValid() && tmpImm2.IsValid());
2800 GetMasm()->Push(tmpImm2);
2801 GetMasm()->Mov(tmpImm1, VixlImm(Imm(src)));
2802 GetMasm()->Mov(tmpImm2, VixlImmU(Imm(src)));
2803 GetMasm()->Strd(tmpImm1, tmpImm2, vixlMem);
2804 GetMasm()->Pop(tmpImm2);
2805 if (tmpImm1.GetCode() != tmp.GetCode()) {
2806 GetMasm()->Pop(tmpImm1);
2807 }
2808 }
2809
EncodeSti(float src,MemRef mem)2810 void Aarch32Encoder::EncodeSti(float src, MemRef mem)
2811 {
2812 ScopedTmpRegF32 tmpReg(this);
2813 GetMasm()->Vmov(VixlVReg(tmpReg).S(), src);
2814 EncodeStr(tmpReg, mem);
2815 }
2816
EncodeSti(double src,MemRef mem)2817 void Aarch32Encoder::EncodeSti(double src, MemRef mem)
2818 {
2819 ScopedTmpRegF64 tmpReg(this);
2820 GetMasm()->Vmov(VixlVReg(tmpReg).D(), src);
2821 EncodeStr(tmpReg, mem);
2822 }
2823
EncodeMemCopy(MemRef memFrom,MemRef memTo,size_t size)2824 void Aarch32Encoder::EncodeMemCopy(MemRef memFrom, MemRef memTo, size_t size)
2825 {
2826 if (size == DOUBLE_WORD_SIZE && memFrom.IsOffsetMem() && memTo.IsOffsetMem()) {
2827 EncodeMemCopy(memFrom, memTo, WORD_SIZE);
2828 constexpr int32_t STEP = 4;
2829 auto offsetFrom = memFrom.GetDisp() + STEP;
2830 auto offsetTo = memTo.GetDisp() + STEP;
2831 EncodeMemCopy(MemRef(memFrom.GetBase(), offsetFrom), MemRef(memTo.GetBase(), offsetTo), WORD_SIZE);
2832
2833 return;
2834 }
2835 ScopedTmpRegU32 tmpReg(this);
2836 auto tmp = VixlReg(tmpReg);
2837 ScopedTmpRegU32 tmpReg1(this);
2838 auto tmp1 = VixlReg(tmpReg1);
2839 if (size == BYTE_SIZE) {
2840 GetMasm()->Ldrb(tmp, PrepareMemLdS(memFrom, INT8_TYPE, tmp, false));
2841 GetMasm()->Strb(tmp, PrepareMemLdS(memTo, INT8_TYPE, tmp1, false));
2842 } else if (size == HALF_SIZE) {
2843 GetMasm()->Ldrh(tmp, PrepareMemLdS(memFrom, INT16_TYPE, tmp, false));
2844 GetMasm()->Strh(tmp, PrepareMemLdS(memTo, INT16_TYPE, tmp1, false));
2845 } else if (size == WORD_SIZE) {
2846 GetMasm()->Ldr(tmp, PrepareMemLdS(memFrom, INT32_TYPE, tmp, false));
2847 GetMasm()->Str(tmp, PrepareMemLdS(memTo, INT32_TYPE, tmp1, false));
2848 } else {
2849 ASSERT(size == DOUBLE_WORD_SIZE);
2850
2851 auto vixlMemFrom = PrepareMemLdS(memFrom, INT64_TYPE, tmp, false, true);
2852 auto vixlMemTo = PrepareMemLdS(memTo, INT64_TYPE, tmp1, false, true);
2853 auto regId = FindRegForMem(vixlMemTo);
2854 ASSERT(regId != -1);
2855 [[maybe_unused]] constexpr auto IMM_2 = 2;
2856 ASSERT(regId % IMM_2 == 0);
2857 vixl::aarch32::Register tmpCopy1(regId);
2858 vixl::aarch32::Register tmpCopy2(regId + 1);
2859
2860 GetMasm()->Push(tmpCopy1);
2861 GetMasm()->Push(tmpCopy2);
2862 GetMasm()->Ldrd(tmpCopy1, tmpCopy2, vixlMemFrom);
2863 GetMasm()->Strd(tmpCopy1, tmpCopy2, vixlMemTo);
2864 GetMasm()->Pop(tmpCopy2);
2865 GetMasm()->Pop(tmpCopy1);
2866 }
2867 }
2868
EncodeMemCopyz(MemRef memFrom,MemRef memTo,size_t size)2869 void Aarch32Encoder::EncodeMemCopyz(MemRef memFrom, MemRef memTo, size_t size)
2870 {
2871 ScopedTmpRegU32 tmpReg(this);
2872 auto tmp = VixlReg(tmpReg);
2873 ScopedTmpRegU32 tmpReg1(this);
2874 auto tmp1 = VixlReg(tmpReg1);
2875
2876 auto type = TypeInfo::GetScalarTypeBySize(size);
2877
2878 auto vixlMemFrom = PrepareMemLdS(memFrom, type, tmp, false, true);
2879 auto vixlMemTo = PrepareMemLdS(memTo, INT64_TYPE, tmp1, false, true);
2880 auto regId = FindRegForMem(vixlMemTo);
2881 ASSERT(regId != -1);
2882 [[maybe_unused]] constexpr auto IMM_2 = 2;
2883 ASSERT(regId % IMM_2 == 0);
2884 vixl::aarch32::Register tmpCopy1(regId);
2885 vixl::aarch32::Register tmpCopy2(regId + 1);
2886
2887 GetMasm()->Push(tmpCopy1);
2888 GetMasm()->Push(tmpCopy2);
2889 if (size == BYTE_SIZE) {
2890 GetMasm()->Ldrb(tmpCopy1, vixlMemFrom);
2891 GetMasm()->Mov(tmpCopy2, VixlImm(0));
2892 GetMasm()->Strd(tmpCopy1, tmpCopy2, vixlMemTo);
2893 } else if (size == HALF_SIZE) {
2894 GetMasm()->Ldrh(tmpCopy1, vixlMemFrom);
2895 GetMasm()->Mov(tmpCopy2, VixlImm(0));
2896 GetMasm()->Strd(tmpCopy1, tmpCopy2, vixlMemTo);
2897 } else if (size == WORD_SIZE) {
2898 GetMasm()->Ldr(tmpCopy1, vixlMemFrom);
2899 GetMasm()->Mov(tmpCopy2, VixlImm(0));
2900 GetMasm()->Strd(tmpCopy1, tmpCopy2, vixlMemTo);
2901 } else {
2902 ASSERT(size == DOUBLE_WORD_SIZE);
2903 GetMasm()->Ldrd(tmpCopy1, tmpCopy2, vixlMemFrom);
2904 GetMasm()->Strd(tmpCopy1, tmpCopy2, vixlMemTo);
2905 }
2906 GetMasm()->Pop(tmpCopy2);
2907 GetMasm()->Pop(tmpCopy1);
2908 }
2909
CompareHelper(Reg src0,Reg src1,Condition * cc)2910 void Aarch32Encoder::CompareHelper(Reg src0, Reg src1, Condition *cc)
2911 {
2912 if (src0.IsFloat() && src1.IsFloat()) {
2913 GetMasm()->Vcmp(VixlVReg(src0), VixlVReg(src1));
2914 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
2915 } else if (src0.GetSize() <= WORD_SIZE && src1.GetSize() <= WORD_SIZE) {
2916 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
2917 } else {
2918 if (!IsConditionSigned(*cc)) {
2919 GetMasm()->Cmp(VixlRegU(src0), VixlRegU(src1));
2920 GetMasm()->Cmp(Convert(Condition::EQ), VixlReg(src0), VixlReg(src1));
2921 } else {
2922 bool swap = false;
2923 switch (*cc) {
2924 case Condition::GT:
2925 swap = true;
2926 *cc = Condition::LT;
2927 break;
2928 case Condition::LE:
2929 swap = true;
2930 *cc = Condition::GE;
2931 break;
2932 case Condition::GE:
2933 case Condition::LT:
2934 break;
2935 default:
2936 UNREACHABLE();
2937 }
2938
2939 Reg op0 = swap ? src1 : src0;
2940 Reg op1 = swap ? src0 : src1;
2941 ScopedTmpRegU32 tmpReg(this);
2942 GetMasm()->Cmp(VixlReg(op0), VixlReg(op1));
2943 GetMasm()->Sbcs(VixlReg(tmpReg), VixlRegU(op0), VixlRegU(op1));
2944 }
2945 }
2946 }
2947
TestHelper(Reg src0,Reg src1,Condition cc)2948 void Aarch32Encoder::TestHelper(Reg src0, Reg src1, [[maybe_unused]] Condition cc)
2949 {
2950 ASSERT(!src0.IsFloat() && !src1.IsFloat());
2951 ASSERT(cc == Condition::TST_EQ || cc == Condition::TST_NE);
2952
2953 if (src0.GetSize() <= WORD_SIZE && src1.GetSize() <= WORD_SIZE) {
2954 GetMasm()->Tst(VixlReg(src0), VixlReg(src1));
2955 } else {
2956 GetMasm()->Tst(VixlRegU(src0), VixlRegU(src1));
2957 GetMasm()->Tst(Convert(Condition::EQ), VixlReg(src0), VixlReg(src1));
2958 }
2959 }
2960
EncodeCompare(Reg dst,Reg src0,Reg src1,Condition cc)2961 void Aarch32Encoder::EncodeCompare(Reg dst, Reg src0, Reg src1, Condition cc)
2962 {
2963 CompareHelper(src0, src1, &cc);
2964 GetMasm()->Mov(Convert(cc), VixlReg(dst), 0x1);
2965 GetMasm()->Mov(Convert(cc).Negate(), VixlReg(dst), 0x0);
2966 }
2967
EncodeCompareTest(Reg dst,Reg src0,Reg src1,Condition cc)2968 void Aarch32Encoder::EncodeCompareTest(Reg dst, Reg src0, Reg src1, Condition cc)
2969 {
2970 TestHelper(src0, src1, cc);
2971 GetMasm()->Mov(ConvertTest(cc), VixlReg(dst), 0x1);
2972 GetMasm()->Mov(ConvertTest(cc).Negate(), VixlReg(dst), 0x0);
2973 }
2974
EncodeAtomicByteOr(Reg addr,Reg value,bool fastEncoding)2975 void Aarch32Encoder::EncodeAtomicByteOr(Reg addr, Reg value, [[maybe_unused]] bool fastEncoding)
2976 {
2977 /**
2978 * .try:
2979 * ldrexb r2, [r0]
2980 * orr r2, r2, r1
2981 * strexb r3, r2, [r0]
2982 * cmp r3, #0
2983 * bne .try
2984 */
2985
2986 auto labelCasFailed = CreateLabel();
2987 BindLabel(labelCasFailed);
2988
2989 ScopedTmpReg tmpReg(this);
2990 ScopedTmpReg casResult(this);
2991
2992 GetMasm()->Ldrexb(VixlReg(tmpReg), vixl::aarch32::MemOperand(VixlReg(addr)));
2993 GetMasm()->Orr(VixlReg(tmpReg), VixlReg(tmpReg), VixlReg(value));
2994 GetMasm()->Strexb(VixlReg(casResult), VixlReg(tmpReg), vixl::aarch32::MemOperand(VixlReg(addr)));
2995 GetMasm()->Cmp(VixlReg(casResult), VixlImm(0));
2996 GetMasm()->B(vixl::aarch32::ne, static_cast<Aarch32LabelHolder *>(GetLabels())->GetLabel(labelCasFailed));
2997 }
2998
EncodeCmp(Reg dst,Reg src0,Reg src1,Condition cc)2999 void Aarch32Encoder::EncodeCmp(Reg dst, Reg src0, Reg src1, Condition cc)
3000 {
3001 if (src0.IsFloat()) {
3002 ASSERT(src1.IsFloat());
3003 ASSERT(cc == Condition::MI || cc == Condition::LT);
3004 GetMasm()->Vcmp(VixlVReg(src0), VixlVReg(src1));
3005 GetMasm()->Vmrs(vixl::aarch32::RegisterOrAPSR_nzcv(vixl::aarch32::kPcCode), vixl::aarch32::FPSCR);
3006 } else {
3007 ASSERT(src0.IsScalar() && src1.IsScalar());
3008 ASSERT(cc == Condition::LO || cc == Condition::LT);
3009 if (src0.GetSize() <= WORD_SIZE && src1.GetSize() <= WORD_SIZE) {
3010 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
3011 } else {
3012 if (cc == Condition::LO) {
3013 GetMasm()->Cmp(VixlRegU(src0), VixlRegU(src1));
3014 GetMasm()->Cmp(Convert(Condition::EQ), VixlReg(src0), VixlReg(src1));
3015 } else if (cc == Condition::LT) {
3016 auto labelHolder = static_cast<Aarch32LabelHolder *>(GetLabels());
3017 auto endLabel = labelHolder->CreateLabel();
3018 ScopedTmpRegU32 tmpReg(this);
3019
3020 GetMasm()->Cmp(VixlReg(src0), VixlReg(src1));
3021 GetMasm()->Sbcs(VixlReg(tmpReg), VixlRegU(src0), VixlRegU(src1));
3022 GetMasm()->Mov(Convert(Condition::LT), VixlReg(dst), VixlImm(-1));
3023 GetMasm()->B(Convert(Condition::LT), labelHolder->GetLabel(endLabel));
3024
3025 GetMasm()->Cmp(VixlReg(src1), VixlReg(src0));
3026 GetMasm()->Sbcs(VixlReg(tmpReg), VixlRegU(src1), VixlRegU(src0));
3027 GetMasm()->Mov(Convert(Condition::LT), VixlReg(dst), VixlImm(1));
3028 GetMasm()->Mov(Convert(Condition::EQ), VixlReg(dst), VixlImm(0));
3029
3030 labelHolder->BindLabel(endLabel);
3031 return;
3032 } else {
3033 UNREACHABLE();
3034 }
3035 }
3036 }
3037
3038 GetMasm()->Mov(Convert(Condition::EQ), VixlReg(dst), VixlImm(0x0));
3039 GetMasm()->Mov(Convert(Condition::NE), VixlReg(dst), VixlImm(0x1));
3040
3041 GetMasm()->Rsb(Convert(cc), VixlReg(dst), VixlReg(dst), VixlImm(0x0));
3042 }
3043
EncodeStackOverflowCheck(ssize_t offset)3044 void Aarch32Encoder::EncodeStackOverflowCheck(ssize_t offset)
3045 {
3046 ScopedTmpReg tmp(this);
3047 EncodeAdd(tmp, GetTarget().GetStackReg(), Imm(offset));
3048 EncodeLdr(tmp, false, MemRef(tmp));
3049 }
3050
EncodeSelect(ArgsSelect && args)3051 void Aarch32Encoder::EncodeSelect(ArgsSelect &&args)
3052 {
3053 auto [dst, src0, src1, src2, src3, cc] = args;
3054 ASSERT(!src0.IsFloat() && !src1.IsFloat());
3055
3056 CompareHelper(src2, src3, &cc);
3057
3058 GetMasm()->Mov(Convert(cc), VixlReg(dst), VixlReg(src0));
3059 GetMasm()->Mov(Convert(cc).Negate(), VixlReg(dst), VixlReg(src1));
3060
3061 if (src0.GetSize() > WORD_SIZE || src1.GetSize() > WORD_SIZE) {
3062 GetMasm()->Mov(Convert(cc), VixlRegU(dst), VixlRegU(src0));
3063 GetMasm()->Mov(Convert(cc).Negate(), VixlRegU(dst), VixlRegU(src1));
3064 }
3065 }
3066
EncodeSelect(ArgsSelectImm && args)3067 void Aarch32Encoder::EncodeSelect(ArgsSelectImm &&args)
3068 {
3069 auto [dst, src0, src1, src2, imm, cc] = args;
3070 ASSERT(!src0.IsFloat() && !src1.IsFloat() && !src2.IsFloat());
3071 auto value = imm.GetAsInt();
3072 if (value == 0) {
3073 switch (cc) {
3074 case Condition::LO:
3075 // LO is always false, select src1
3076 GetMasm()->Mov(VixlReg(dst), VixlReg(src1));
3077 if (src0.GetSize() > WORD_SIZE || src1.GetSize() > WORD_SIZE) {
3078 GetMasm()->Mov(VixlRegU(dst), VixlRegU(src1));
3079 }
3080 return;
3081 case Condition::HS:
3082 // HS is always true, select src0
3083 GetMasm()->Mov(VixlReg(dst), VixlReg(src0));
3084 if (src0.GetSize() > WORD_SIZE || src1.GetSize() > WORD_SIZE) {
3085 GetMasm()->Mov(VixlRegU(dst), VixlRegU(src0));
3086 }
3087 return;
3088 case Condition::LS:
3089 // LS is same as EQ
3090 cc = Condition::EQ;
3091 break;
3092 case Condition::HI:
3093 // HI is same as NE
3094 cc = Condition::NE;
3095 break;
3096 default:
3097 break;
3098 }
3099 CompareZeroHelper(src2, &cc);
3100 } else { // value != 0
3101 if (!CompareImmHelper(src2, value, &cc)) {
3102 return;
3103 }
3104 }
3105
3106 GetMasm()->Mov(Convert(cc), VixlReg(dst), VixlReg(src0));
3107 GetMasm()->Mov(Convert(cc).Negate(), VixlReg(dst), VixlReg(src1));
3108
3109 if (src0.GetSize() > WORD_SIZE || src1.GetSize() > WORD_SIZE) {
3110 GetMasm()->Mov(Convert(cc), VixlRegU(dst), VixlRegU(src0));
3111 GetMasm()->Mov(Convert(cc).Negate(), VixlRegU(dst), VixlRegU(src1));
3112 }
3113 }
3114
EncodeSelectTest(ArgsSelect && args)3115 void Aarch32Encoder::EncodeSelectTest(ArgsSelect &&args)
3116 {
3117 auto [dst, src0, src1, src2, src3, cc] = args;
3118 ASSERT(!src0.IsFloat() && !src1.IsFloat() && !src2.IsFloat());
3119
3120 TestHelper(src2, src3, cc);
3121
3122 GetMasm()->Mov(ConvertTest(cc), VixlReg(dst), VixlReg(src0));
3123 GetMasm()->Mov(ConvertTest(cc).Negate(), VixlReg(dst), VixlReg(src1));
3124
3125 if (src0.GetSize() > WORD_SIZE || src1.GetSize() > WORD_SIZE) {
3126 GetMasm()->Mov(ConvertTest(cc), VixlRegU(dst), VixlRegU(src0));
3127 GetMasm()->Mov(ConvertTest(cc).Negate(), VixlRegU(dst), VixlRegU(src1));
3128 }
3129 }
3130
EncodeSelectTest(ArgsSelectImm && args)3131 void Aarch32Encoder::EncodeSelectTest(ArgsSelectImm &&args)
3132 {
3133 auto [dst, src0, src1, src2, imm, cc] = args;
3134 ASSERT(!src0.IsFloat() && !src1.IsFloat() && !src2.IsFloat());
3135
3136 TestImmHelper(src2, imm, cc);
3137 GetMasm()->Mov(ConvertTest(cc), VixlReg(dst), VixlReg(src0));
3138 GetMasm()->Mov(ConvertTest(cc).Negate(), VixlReg(dst), VixlReg(src1));
3139
3140 if (src0.GetSize() > WORD_SIZE || src1.GetSize() > WORD_SIZE) {
3141 GetMasm()->Mov(ConvertTest(cc), VixlRegU(dst), VixlRegU(src0));
3142 GetMasm()->Mov(ConvertTest(cc).Negate(), VixlRegU(dst), VixlRegU(src1));
3143 }
3144 }
3145
CanEncodeImmAddSubCmp(int64_t imm,uint32_t size,bool signedCompare)3146 bool Aarch32Encoder::CanEncodeImmAddSubCmp(int64_t imm, uint32_t size, bool signedCompare)
3147 {
3148 if (imm == INT64_MIN) {
3149 return false;
3150 }
3151 if (imm < 0) {
3152 imm = -imm;
3153 if (size > WORD_SIZE && signedCompare) {
3154 return false;
3155 }
3156 }
3157 // We don't support 64-bit immediate, even when both higher and lower parts are legal immediates
3158 if (imm > UINT32_MAX) {
3159 return false;
3160 }
3161 return vixl::aarch32::ImmediateA32::IsImmediateA32(imm);
3162 }
3163
CanEncodeImmLogical(uint64_t imm,uint32_t size)3164 bool Aarch32Encoder::CanEncodeImmLogical(uint64_t imm, uint32_t size)
3165 {
3166 // NOLINTNEXTLINE(hicpp-signed-bitwise)
3167 uint64_t high = imm >> WORD_SIZE;
3168 if (size == DOUBLE_WORD_SIZE) {
3169 if (high != 0U && high != UINT32_MAX) {
3170 return false;
3171 }
3172 }
3173 #ifndef NDEBUG
3174 if (size < DOUBLE_WORD_SIZE) {
3175 // Test if the highest part is consistent:
3176 ASSERT(((imm >> size) == 0) || (((~imm) >> size) == 0));
3177 }
3178 #endif // NDEBUG
3179 return vixl::aarch32::ImmediateA32::IsImmediateA32(imm);
3180 }
3181
3182 using vixl::aarch32::MemOperand;
3183
3184 template <bool IS_STORE>
LoadStoreRegistersMainLoop(RegMask registers,bool isFp,int32_t slot,Reg base,RegMask mask)3185 void Aarch32Encoder::LoadStoreRegistersMainLoop(RegMask registers, bool isFp, int32_t slot, Reg base, RegMask mask)
3186 {
3187 bool hasMask = mask.any();
3188 int32_t index = hasMask ? static_cast<int32_t>(mask.GetMinRegister()) : 0;
3189 slot -= index;
3190 vixl::aarch32::Register baseReg = VixlReg(base);
3191 for (size_t i = index; i < registers.size(); i++) {
3192 if (hasMask) {
3193 if (!mask.test(i)) {
3194 continue;
3195 }
3196 index++;
3197 }
3198 if (!registers.test(i)) {
3199 continue;
3200 }
3201
3202 if (!hasMask) {
3203 index++;
3204 }
3205 auto mem = MemOperand(baseReg, (slot + index - 1) * WORD_SIZE_BYTES);
3206 if (isFp) {
3207 auto reg = vixl::aarch32::SRegister(i);
3208 if constexpr (IS_STORE) { // NOLINT
3209 GetMasm()->Vstr(reg, mem);
3210 } else { // NOLINT
3211 GetMasm()->Vldr(reg, mem);
3212 }
3213 } else {
3214 auto reg = vixl::aarch32::Register(i);
3215 if constexpr (IS_STORE) { // NOLINT
3216 GetMasm()->Str(reg, mem);
3217 } else { // NOLINT
3218 GetMasm()->Ldr(reg, mem);
3219 }
3220 }
3221 }
3222 }
3223
3224 template <bool IS_STORE>
LoadStoreRegisters(RegMask registers,bool isFp,int32_t slot,Reg base,RegMask mask)3225 void Aarch32Encoder::LoadStoreRegisters(RegMask registers, bool isFp, int32_t slot, Reg base, RegMask mask)
3226 {
3227 if (registers.none()) {
3228 return;
3229 }
3230
3231 vixl::aarch32::Register baseReg = VixlReg(base);
3232 int32_t maxOffset = (slot + helpers::ToSigned(registers.GetMaxRegister())) * WORD_SIZE_BYTES;
3233
3234 ScopedTmpRegU32 tmpReg(this);
3235 auto tmp = VixlReg(tmpReg);
3236 // Construct single add for big offset
3237 if (isFp) {
3238 if ((maxOffset < -VMEM_OFFSET) || (maxOffset > VMEM_OFFSET)) {
3239 GetMasm()->Add(tmp, baseReg, VixlImm(slot * WORD_SIZE_BYTES));
3240 slot = 0;
3241 baseReg = tmp;
3242 }
3243 } else {
3244 if ((maxOffset < -MEM_BIG_OFFSET) || (maxOffset > MEM_BIG_OFFSET)) {
3245 GetMasm()->Add(tmp, baseReg, VixlImm(slot * WORD_SIZE_BYTES));
3246 slot = 0;
3247 baseReg = tmp;
3248 }
3249 }
3250 LoadStoreRegistersMainLoop<IS_STORE>(registers, isFp, slot, base, mask);
3251 }
3252
3253 template <bool IS_STORE>
LoadStoreRegisters(RegMask registers,ssize_t slot,size_t startReg,bool isFp)3254 void Aarch32Encoder::LoadStoreRegisters(RegMask registers, ssize_t slot, size_t startReg, bool isFp)
3255 {
3256 if (registers.none()) {
3257 return;
3258 }
3259 int32_t lastReg = registers.size() - 1;
3260 for (; lastReg >= 0; --lastReg) {
3261 if (registers.test(lastReg)) {
3262 break;
3263 }
3264 }
3265 vixl::aarch32::Register baseReg = vixl::aarch32::sp;
3266 auto maxOffset = (slot + lastReg) * WORD_SIZE_BYTES;
3267 ScopedTmpRegU32 tmpReg(this);
3268 auto tmp = VixlReg(tmpReg);
3269 // Construct single add for big offset
3270 if (isFp) {
3271 if ((maxOffset < -VMEM_OFFSET) || (maxOffset > VMEM_OFFSET)) {
3272 GetMasm()->Add(tmp, baseReg, VixlImm(slot * WORD_SIZE_BYTES));
3273 slot = 0;
3274 baseReg = tmp;
3275 }
3276 } else {
3277 if ((maxOffset < -MEM_BIG_OFFSET) || (maxOffset > MEM_BIG_OFFSET)) {
3278 GetMasm()->Add(tmp, baseReg, VixlImm(slot * WORD_SIZE_BYTES));
3279 slot = 0;
3280 baseReg = tmp;
3281 }
3282 }
3283 for (auto i = startReg; i < registers.size(); i++) {
3284 if (!registers.test(i)) {
3285 continue;
3286 }
3287 auto mem = MemOperand(baseReg, (slot + i - startReg) * WORD_SIZE_BYTES);
3288 if (isFp) {
3289 auto reg = vixl::aarch32::SRegister(i);
3290 if constexpr (IS_STORE) { // NOLINT
3291 GetMasm()->Vstr(reg, mem);
3292 } else { // NOLINT
3293 GetMasm()->Vldr(reg, mem);
3294 }
3295 } else {
3296 auto reg = vixl::aarch32::Register(i);
3297 if constexpr (IS_STORE) { // NOLINT
3298 GetMasm()->Str(reg, mem);
3299 } else { // NOLINT
3300 GetMasm()->Ldr(reg, mem);
3301 }
3302 }
3303 }
3304 }
3305
PushRegisters(RegMask registers,bool isFp)3306 void Aarch32Encoder::PushRegisters(RegMask registers, bool isFp)
3307 {
3308 (void)registers;
3309 (void)isFp;
3310 // NOTE(msherstennikov): Implement
3311 }
3312
PopRegisters(RegMask registers,bool isFp)3313 void Aarch32Encoder::PopRegisters(RegMask registers, bool isFp)
3314 {
3315 (void)registers;
3316 (void)isFp;
3317 // NOTE(msherstennikov): Implement
3318 }
3319
DisasmInstr(std::ostream & stream,size_t pc,ssize_t codeOffset) const3320 size_t Aarch32Encoder::DisasmInstr(std::ostream &stream, size_t pc, ssize_t codeOffset) const
3321 {
3322 auto addr = GetMasm()->GetBuffer()->GetOffsetAddress<const uint32_t *>(pc);
3323 // Display pc is seted, because disassembler use pc
3324 // for upper bits (e.g. 0x40000000), when print one instruction.
3325 if (codeOffset < 0) {
3326 vixl::aarch32::PrintDisassembler disasm(GetAllocator(), stream);
3327 disasm.DisassembleA32Buffer(addr, vixl::aarch32::k32BitT32InstructionSizeInBytes);
3328 } else {
3329 const uint64_t displayPc = 0x10000000;
3330 vixl::aarch32::PrintDisassembler disasm(GetAllocator(), stream, displayPc + pc + codeOffset);
3331 disasm.DisassembleA32Buffer(addr, vixl::aarch32::k32BitT32InstructionSizeInBytes);
3332
3333 stream << std::setfill(' ');
3334 }
3335 return pc + vixl::aarch32::k32BitT32InstructionSizeInBytes;
3336 }
3337 } // namespace ark::compiler::aarch32
3338