1 /*
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 /*
17 Codegen Hi-Level implementation
18 */
19 #include "operands.h"
20 #include "codegen.h"
21 #include "compiler_options.h"
22 #include "lib_call_inst.h"
23 #include "relocations.h"
24 #include "include/compiler_interface.h"
25 #include "ir-dyn-base-types.h"
26 #include "runtime/include/coretypes/string.h"
27 #include "compiler/optimizer/ir/analysis.h"
28 #include "compiler/optimizer/ir/locations.h"
29 #include "compiler/optimizer/analysis/liveness_analyzer.h"
30 #include "optimizer/code_generator/method_properties.h"
31 #include "events/events.h"
32 #include "libpandabase/utils/tsan_interface.h"
33 #include "libpandabase/utils/utils.h"
34 #include <iomanip>
35
36 namespace panda::compiler {
37
38 class OsrEntryStub {
FixIntervals(Codegen * codegen,Encoder * encoder)39 void FixIntervals(Codegen *codegen, Encoder *encoder)
40 {
41 auto &la = codegen->GetGraph()->GetAnalysis<LivenessAnalyzer>();
42 la.EnumerateLiveIntervalsForInst(saveState_, [this, codegen, encoder](const auto &li) {
43 auto inst = li->GetInst();
44 auto location = li->GetLocation();
45 // Skip live registers that are already in the input list of the OsrSaveState
46 const auto &ssInputs = saveState_->GetInputs();
47 if (std::find_if(ssInputs.begin(), ssInputs.end(),
48 [inst](auto &input) { return input.GetInst() == inst; }) != ssInputs.end()) {
49 return;
50 }
51 // Only constants allowed
52 switch (inst->GetOpcode()) {
53 case Opcode::LoadAndInitClass: {
54 auto klass = reinterpret_cast<uintptr_t>(inst->CastToLoadAndInitClass()->GetClass());
55 encoder->EncodeMov(codegen->ConvertRegister(inst->GetDstReg(), inst->GetType()),
56 Imm(reinterpret_cast<uintptr_t>(klass)));
57 break;
58 }
59 case Opcode::Constant: {
60 if (location.IsFixedRegister()) {
61 EncodeConstantMove(li, encoder);
62 } else if (location.IsStack()) {
63 auto slot = location.GetValue();
64 encoder->EncodeSti(
65 bit_cast<int64_t>(inst->CastToConstant()->GetRawValue()), DOUBLE_WORD_SIZE_BYTES,
66 MemRef(codegen->SpReg(), codegen->GetFrameLayout().GetSpillOffsetFromSpInBytes(slot)));
67 } else {
68 ASSERT(location.IsConstant());
69 }
70 break;
71 }
72 // NOTE (ekudriashov): UnresolvedLoadAndInitClass
73 default:
74 break;
75 }
76 });
77 }
78
EncodeConstantMove(const LifeIntervals * li,Encoder * encoder)79 static void EncodeConstantMove(const LifeIntervals *li, Encoder *encoder)
80 {
81 auto inst = li->GetInst();
82 switch (li->GetType()) {
83 case DataType::FLOAT64:
84 encoder->EncodeMov(Reg(li->GetReg(), FLOAT64_TYPE),
85 Imm(bit_cast<double>(inst->CastToConstant()->GetDoubleValue())));
86 break;
87 case DataType::FLOAT32:
88 encoder->EncodeMov(Reg(li->GetReg(), FLOAT32_TYPE),
89 Imm(bit_cast<float>(inst->CastToConstant()->GetFloatValue())));
90 break;
91 case DataType::UINT32:
92 encoder->EncodeMov(Reg(li->GetReg(), INT32_TYPE), Imm(inst->CastToConstant()->GetRawValue()));
93 break;
94 case DataType::UINT64:
95 encoder->EncodeMov(Reg(li->GetReg(), INT64_TYPE), Imm(inst->CastToConstant()->GetRawValue()));
96 break;
97 default:
98 UNREACHABLE();
99 }
100 }
101
102 public:
OsrEntryStub(Codegen * codegen,SaveStateInst * inst)103 OsrEntryStub(Codegen *codegen, SaveStateInst *inst) : label_(codegen->GetEncoder()->CreateLabel()), saveState_(inst)
104 {
105 }
106
107 DEFAULT_MOVE_SEMANTIC(OsrEntryStub);
108 DEFAULT_COPY_SEMANTIC(OsrEntryStub);
109 ~OsrEntryStub() = default;
110
Generate(Codegen * codegen)111 void Generate(Codegen *codegen)
112 {
113 auto encoder = codegen->GetEncoder();
114 auto lr = codegen->GetTarget().GetLinkReg();
115 auto fl = codegen->GetFrameLayout();
116 codegen->CreateStackMap(saveState_->CastToSaveStateOsr());
117 ssize_t slot = CFrameLayout::LOCALS_START_SLOT + CFrameLayout::GetLocalsCount() - 1;
118 encoder->EncodeStp(
119 codegen->FpReg(), lr,
120 MemRef(codegen->FpReg(),
121 -fl.GetOffset<CFrameLayout::OffsetOrigin::FP, CFrameLayout::OffsetUnit::BYTES>(slot)));
122
123 FixIntervals(codegen, encoder);
124 encoder->EncodeJump(label_);
125 }
126
GetInst()127 SaveStateInst *GetInst()
128 {
129 return saveState_;
130 }
131
GetLabel()132 auto &GetLabel()
133 {
134 return label_;
135 }
136
137 private:
138 LabelHolder::LabelId label_;
139 SaveStateInst *saveState_ {nullptr};
140 };
141
Codegen(Graph * graph)142 Codegen::Codegen(Graph *graph)
143 : Optimization(graph),
144 allocator_(graph->GetAllocator()),
145 localAllocator_(graph->GetLocalAllocator()),
146 codeBuilder_(allocator_->New<CodeInfoBuilder>(graph->GetArch(), allocator_)),
147 slowPaths_(graph->GetLocalAllocator()->Adapter()),
148 slowPathsMap_(graph->GetLocalAllocator()->Adapter()),
149 frameLayout_(CFrameLayout(graph->GetArch(), graph->GetStackSlotsCount())),
150 osrEntries_(graph->GetLocalAllocator()->Adapter()),
151 vregIndices_(GetAllocator()->Adapter()),
152 runtime_(graph->GetRuntime()),
153 target_(graph->GetArch()),
154 liveOuts_(graph->GetLocalAllocator()->Adapter()),
155 disasm_(this),
156 spillFillsResolver_(graph)
157 {
158 graph->SetCodeBuilder(codeBuilder_);
159 regfile_ = graph->GetRegisters();
160 if (regfile_ != nullptr) {
161 ASSERT(regfile_->IsValid());
162 ArenaVector<Reg> regsUsage(allocator_->Adapter());
163 Convert(®sUsage, graph->GetUsedRegs<DataType::INT64>(), INT64_TYPE);
164 Convert(®sUsage, graph->GetUsedRegs<DataType::FLOAT64>(), FLOAT64_TYPE);
165 regfile_->SetUsedRegs(regsUsage);
166 #ifndef NDEBUG
167 COMPILER_LOG(DEBUG, CODEGEN) << "Regalloc used registers scalar " << graph->GetUsedRegs<DataType::INT64>();
168 COMPILER_LOG(DEBUG, CODEGEN) << "Regalloc used registers vector " << graph->GetUsedRegs<DataType::FLOAT64>();
169 #endif
170 }
171
172 enc_ = graph->GetEncoder();
173 ASSERT(enc_ != nullptr && enc_->IsValid());
174 enc_->SetRegfile(regfile_);
175 if (enc_->InitMasm()) {
176 enc_->SetFrameLayout(GetFrameLayout());
177 }
178
179 callconv_ = graph->GetCallingConvention();
180 if (callconv_ != nullptr) {
181 ASSERT(callconv_->IsValid());
182 if (callconv_->GetEncoder() == nullptr) {
183 callconv_->SetEncoder(enc_);
184 }
185 }
186
187 auto method = graph->GetMethod();
188 // workaround for test
189 if (method != nullptr) {
190 methodId_ = graph->GetRuntime()->GetMethodId(method);
191 }
192 GetDisasm()->Init();
193 GetDisasm()->SetEncoder(GetEncoder());
194 }
195
GetPassName() const196 const char *Codegen::GetPassName() const
197 {
198 return "Codegen";
199 }
200
AbortIfFailed() const201 bool Codegen::AbortIfFailed() const
202 {
203 return true;
204 }
205
CreateFrameInfo()206 void Codegen::CreateFrameInfo()
207 {
208 // Create FrameInfo for CFrame
209 auto &fl = GetFrameLayout();
210 auto frame = GetGraph()->GetLocalAllocator()->New<FrameInfo>(
211 FrameInfo::PositionedCallers::Encode(true) | FrameInfo::PositionedCallees::Encode(true) |
212 FrameInfo::CallersRelativeFp::Encode(false) | FrameInfo::CalleesRelativeFp::Encode(true));
213 frame->SetFrameSize(fl.GetFrameSize<CFrameLayout::OffsetUnit::BYTES>());
214 frame->SetSpillsCount(fl.GetSpillsCount());
215
216 frame->SetCallersOffset(fl.GetOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::SLOTS>(
217 fl.GetStackStartSlot() + fl.GetCallerLastSlot(false)));
218 frame->SetFpCallersOffset(fl.GetOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::SLOTS>(
219 fl.GetStackStartSlot() + fl.GetCallerLastSlot(true)));
220 frame->SetCalleesOffset(-fl.GetOffset<CFrameLayout::OffsetOrigin::FP, CFrameLayout::OffsetUnit::SLOTS>(
221 fl.GetStackStartSlot() + fl.GetCalleeLastSlot(false)));
222 frame->SetFpCalleesOffset(-fl.GetOffset<CFrameLayout::OffsetOrigin::FP, CFrameLayout::OffsetUnit::SLOTS>(
223 fl.GetStackStartSlot() + fl.GetCalleeLastSlot(true)));
224
225 frame->SetSetupFrame(true);
226 frame->SetSaveFrameAndLinkRegs(true);
227 frame->SetSaveUnusedCalleeRegs(!GetGraph()->GetMethodProperties().GetCompactPrologueAllowed());
228 frame->SetAdjustSpReg(true);
229 frame->SetHasFloatRegs(GetGraph()->HasFloatRegs());
230
231 GetCodeBuilder()->SetHasFloatRegs(GetGraph()->HasFloatRegs());
232
233 SetFrameInfo(frame);
234 }
235
FillOnlyParameters(RegMask * liveRegs,uint32_t numParams,bool isFastpath) const236 void Codegen::FillOnlyParameters(RegMask *liveRegs, uint32_t numParams, bool isFastpath) const
237 {
238 ASSERT(numParams <= 6U);
239 if (GetArch() == Arch::AARCH64 && !isFastpath) {
240 numParams = AlignUp(numParams, 2U);
241 }
242 *liveRegs &= GetTarget().GetParamRegsMask(numParams);
243 }
244
Convert(ArenaVector<Reg> * regsUsage,const ArenaVector<bool> * mask,TypeInfo typeInfo)245 void Codegen::Convert(ArenaVector<Reg> *regsUsage, const ArenaVector<bool> *mask, TypeInfo typeInfo)
246 {
247 ASSERT(regsUsage != nullptr);
248 // There are no used registers
249 if (mask == nullptr) {
250 return;
251 }
252 ASSERT(mask->size() == MAX_NUM_REGS);
253 for (uint32_t i = 0; i < MAX_NUM_REGS; ++i) {
254 if ((*mask)[i]) {
255 regsUsage->emplace_back(i, typeInfo);
256 }
257 }
258 }
259
260 #ifdef INTRINSIC_SLOW_PATH_ENTRY_ENABLED
CreateIrtocIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)261 void Codegen::CreateIrtocIntrinsic(IntrinsicInst *inst, [[maybe_unused]] Reg dst, [[maybe_unused]] SRCREGS src)
262 {
263 switch (inst->GetIntrinsicId()) {
264 case RuntimeInterface::IntrinsicId::INTRINSIC_SLOW_PATH_ENTRY:
265 IntrinsicSlowPathEntry(inst);
266 break;
267 case RuntimeInterface::IntrinsicId::INTRINSIC_UNREACHABLE:
268 GetEncoder()->EncodeAbort();
269 break;
270 case RuntimeInterface::IntrinsicId::INTRINSIC_SAVE_REGISTERS_EP:
271 IntrinsicSaveRegisters(inst);
272 break;
273 case RuntimeInterface::IntrinsicId::INTRINSIC_RESTORE_REGISTERS_EP:
274 IntrinsicRestoreRegisters(inst);
275 break;
276 case RuntimeInterface::IntrinsicId::INTRINSIC_TAIL_CALL:
277 IntrinsicTailCall(inst);
278 break;
279 case RuntimeInterface::IntrinsicId::INTRINSIC_INTERPRETER_RETURN:
280 GetCallingConvention()->GenerateNativeEpilogue(*GetFrameInfo(), []() {});
281 break;
282 case RuntimeInterface::IntrinsicId::INTRINSIC_LOAD_ACQUIRE_MARK_WORD_EXCLUSIVE:
283 ASSERT(GetRuntime()->GetObjMarkWordOffset(GetArch()) == 0);
284 GetEncoder()->EncodeLdrExclusive(dst, src[0], true);
285 break;
286 case RuntimeInterface::IntrinsicId::INTRINSIC_STORE_RELEASE_MARK_WORD_EXCLUSIVE:
287 ASSERT(GetRuntime()->GetObjMarkWordOffset(GetArch()) == 0);
288 GetEncoder()->EncodeStrExclusive(dst, src[SECOND_OPERAND], src[0], true);
289 break;
290 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPARE_AND_SET_MARK_WORD:
291 ASSERT(GetRuntime()->GetObjMarkWordOffset(GetArch()) == 0);
292 GetEncoder()->EncodeCompareAndSwap(dst, src[0], src[SECOND_OPERAND], src[THIRD_OPERAND]);
293 break;
294 case RuntimeInterface::IntrinsicId::INTRINSIC_DATA_MEMORY_BARRIER_FULL:
295 GetEncoder()->EncodeMemoryBarrier(memory_order::FULL);
296 break;
297 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPRESS_EIGHT_UTF16_TO_UTF8_CHARS_USING_SIMD:
298 GetEncoder()->EncodeCompressEightUtf16ToUtf8CharsUsingSimd(src[FIRST_OPERAND], src[SECOND_OPERAND]);
299 break;
300 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPRESS_SIXTEEN_UTF16_TO_UTF8_CHARS_USING_SIMD:
301 GetEncoder()->EncodeCompressSixteenUtf16ToUtf8CharsUsingSimd(src[FIRST_OPERAND], src[SECOND_OPERAND]);
302 break;
303 default:
304 UNREACHABLE();
305 break;
306 }
307 }
308 #endif
309
BeginMethod()310 bool Codegen::BeginMethod()
311 {
312 // Do not try to encode too large graph
313 auto instSize = GetGraph()->GetCurrentInstructionId();
314 auto instsPerByte = GetEncoder()->MaxArchInstPerEncoded();
315 auto maxBitsInInst = GetInstructionSizeBits(GetArch());
316 instSize += slowPaths_.size() * INST_IN_SLOW_PATH;
317 if ((instSize * instsPerByte * maxBitsInInst) > g_options.GetCompilerMaxGenCodeSize()) {
318 return false;
319 }
320 // After this - encoder aborted, if allocated too much size.
321 GetEncoder()->SetMaxAllocatedBytes(g_options.GetCompilerMaxGenCodeSize());
322
323 if (GetGraph()->IsAotMode()) {
324 GetEncoder()->SetCodeOffset(GetGraph()->GetAotData()->GetCodeOffset() + CodeInfo::GetCodeOffset(GetArch()));
325 } else {
326 GetEncoder()->SetCodeOffset(0);
327 }
328
329 codeBuilder_->BeginMethod(GetFrameLayout().GetFrameSize<CFrameLayout::OffsetUnit::BYTES>(),
330 GetGraph()->GetVRegsCount() + GetGraph()->GetEnvCount());
331
332 GetEncoder()->BindLabel(GetLabelEntry());
333 SetStartCodeOffset(GetEncoder()->GetCursorOffset());
334
335 GeneratePrologue();
336
337 return GetEncoder()->GetResult();
338 }
339
GeneratePrologue()340 void Codegen::GeneratePrologue()
341 {
342 SCOPED_DISASM_STR(this, "Method Prologue");
343
344 GetCallingConvention()->GeneratePrologue(*frameInfo_);
345
346 if (!GetGraph()->GetMode().IsNative()) {
347 GetEncoder()->EncodeSti(1, 1, MemRef(ThreadReg(), GetRuntime()->GetTlsFrameKindOffset(GetArch())));
348 }
349 if (!GetGraph()->GetMode().IsNative()) {
350 // Create stack overflow check
351 GetEncoder()->EncodeStackOverflowCheck(-GetRuntime()->GetStackOverflowCheckOffset());
352 // Create empty stackmap for the stack overflow check
353 GetCodeBuilder()->BeginStackMap(0, 0, nullptr, 0, false, false);
354 GetCodeBuilder()->EndStackMap();
355 }
356
357 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
358 if (GetGraph()->IsAotMode()) {
359 SCOPED_DISASM_STR(this, "LoadMethod for trace");
360 ScopedTmpReg method_reg(GetEncoder());
361 LoadMethod(method_reg);
362 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_ENTER)), method_reg,
363 Imm(static_cast<size_t>(events::MethodEnterKind::COMPILED)));
364 } else {
365 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
366 Imm(reinterpret_cast<size_t>(GetGraph()->GetMethod())),
367 Imm(static_cast<size_t>(events::MethodEnterKind::COMPILED)));
368 }
369 #endif
370 }
371
GenerateEpilogue()372 void Codegen::GenerateEpilogue()
373 {
374 SCOPED_DISASM_STR(this, "Method Epilogue");
375
376 #if defined(EVENT_METHOD_EXIT_ENABLED) && EVENT_METHOD_EXIT_ENABLED != 0
377 GetCallingConvention()->GenerateEpilogue(*frame_info_, [this]() {
378 if (GetGraph()->IsAotMode()) {
379 ScopedTmpReg method_reg(GetEncoder());
380 LoadMethod(method_reg);
381 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_EXIT)), method_reg,
382 Imm(static_cast<size_t>(events::MethodExitKind::COMPILED)));
383 } else {
384 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_EXIT)),
385 Imm(reinterpret_cast<size_t>(GetGraph()->GetMethod())),
386 Imm(static_cast<size_t>(events::MethodExitKind::COMPILED)));
387 }
388 });
389 #else
390 GetCallingConvention()->GenerateEpilogue(*frameInfo_, []() {});
391 #endif
392 }
393
VisitGraph()394 bool Codegen::VisitGraph()
395 {
396 EncodeVisitor visitor(this);
397 visitor_ = &visitor;
398
399 const auto &blocks = GetGraph()->GetBlocksLinearOrder();
400
401 for (auto bb : blocks) {
402 GetEncoder()->BindLabel(bb->GetId());
403 for (auto inst : bb->AllInsts()) {
404 SCOPED_DISASM_INST(this, inst);
405
406 #ifdef PANDA_COMPILER_DEBUG_INFO
407 auto debugInfo = inst->GetDebugInfo();
408 if (GetGraph()->IsLineDebugInfoEnabled() && debugInfo != nullptr) {
409 debugInfo->SetOffset(GetEncoder()->GetCursorOffset());
410 }
411 #endif
412 visitor.VisitInstruction(inst);
413 if (!visitor.GetResult()) {
414 COMPILER_LOG(DEBUG, CODEGEN)
415 << "Can't encode instruction: " << GetOpcodeString(inst->GetOpcode()) << *inst;
416 break;
417 }
418 }
419
420 if (bb->NeedsJump()) {
421 EmitJump(bb);
422 }
423
424 if (!visitor.GetResult()) {
425 return false;
426 }
427 }
428
429 auto instsPerByte = GetEncoder()->MaxArchInstPerEncoded();
430 auto maxBitsInInst = GetInstructionSizeBits(GetArch());
431 auto instSize = GetGraph()->GetCurrentInstructionId() + slowPaths_.size() * INST_IN_SLOW_PATH;
432 if ((instSize * instsPerByte * maxBitsInInst) > g_options.GetCompilerMaxGenCodeSize()) {
433 return false;
434 }
435
436 EmitSlowPaths();
437 visitor_ = nullptr;
438
439 return true;
440 }
441
EmitJump(const BasicBlock * bb)442 void Codegen::EmitJump(const BasicBlock *bb)
443 {
444 BasicBlock *sucBb = nullptr;
445
446 if (bb->GetLastInst() == nullptr) {
447 ASSERT(bb->IsEmpty());
448 sucBb = bb->GetSuccsBlocks()[0];
449 } else {
450 switch (bb->GetLastInst()->GetOpcode()) {
451 case Opcode::If:
452 case Opcode::IfImm:
453 ASSERT(bb->GetSuccsBlocks().size() == MAX_SUCCS_NUM);
454 sucBb = bb->GetFalseSuccessor();
455 break;
456 default:
457 sucBb = bb->GetSuccsBlocks()[0];
458 break;
459 }
460 }
461 SCOPED_DISASM_STR(this, std::string("Jump from BB_") + std::to_string(bb->GetId()) + " to BB_" +
462 std::to_string(sucBb->GetId()));
463
464 auto label = sucBb->GetId();
465 GetEncoder()->EncodeJump(label);
466 }
467
EndMethod()468 void Codegen::EndMethod()
469 {
470 for (auto &osrStub : osrEntries_) {
471 SCOPED_DISASM_STR(this,
472 std::string("Osr stub for OsrStateStump ") + std::to_string(osrStub->GetInst()->GetId()));
473 osrStub->Generate(this);
474 }
475
476 GetEncoder()->Finalize();
477 }
478
479 // Allocates memory, copies generated code to it, sets the code to the graph's codegen data. Also this function
480 // encodes the code info and sets it to the graph.
CopyToCodeCache()481 bool Codegen::CopyToCodeCache()
482 {
483 auto codeEntry = reinterpret_cast<void *>(GetEncoder()->GetLabelAddress(GetLabelEntry()));
484 auto codeSize = GetEncoder()->GetCursorOffset();
485 bool saveAllCalleeRegisters = !GetGraph()->GetMethodProperties().GetCompactPrologueAllowed();
486
487 auto code = reinterpret_cast<uint8_t *>(GetAllocator()->Alloc(codeSize));
488 if (code == nullptr) {
489 return false;
490 }
491 memcpy_s(code, codeSize, codeEntry, codeSize);
492 GetGraph()->SetCode(EncodeDataType(code, codeSize));
493
494 RegMask calleeRegs;
495 VRegMask calleeVregs;
496 GetRegfile()->FillUsedCalleeSavedRegisters(&calleeRegs, &calleeVregs, saveAllCalleeRegisters);
497 constexpr size_t MAX_NUM_REGISTERS = 32;
498 static_assert(MAX_NUM_REGS <= MAX_NUM_REGISTERS && MAX_NUM_VREGS <= MAX_NUM_REGISTERS);
499 codeBuilder_->SetSavedCalleeRegsMask(static_cast<uint32_t>(calleeRegs.to_ulong()),
500 static_cast<uint32_t>(calleeVregs.to_ulong()));
501
502 ArenaVector<uint8_t> codeInfoData(GetGraph()->GetAllocator()->Adapter());
503 codeBuilder_->Encode(&codeInfoData);
504 GetGraph()->SetCodeInfo(Span(codeInfoData));
505
506 return true;
507 }
508
IssueDisasm()509 void Codegen::IssueDisasm()
510 {
511 if (GetGraph()->GetMode().SupportManagedCode() && g_options.IsCompilerDisasmDumpCodeInfo()) {
512 GetDisasm()->PrintCodeInfo(this);
513 }
514 GetDisasm()->PrintCodeStatistics(this);
515 auto &stream = GetDisasm()->GetStream();
516 for (auto &item : GetDisasm()->GetItems()) {
517 if (std::holds_alternative<CodeItem>(item)) {
518 auto &code = std::get<CodeItem>(item);
519 for (size_t pc = code.GetPosition(); pc < code.GetCursorOffset();) {
520 stream << GetDisasm()->GetIndent(code.GetDepth());
521 auto newPc = GetEncoder()->DisasmInstr(stream, pc, 0);
522 stream << "\n";
523 pc = newPc;
524 }
525 } else if (std::holds_alternative<std::string>(item)) {
526 stream << (std::get<std::string>(item));
527 }
528 }
529 }
530
RunImpl()531 bool Codegen::RunImpl()
532 {
533 Initialize();
534
535 auto encoder = GetEncoder();
536 encoder->GetLabels()->CreateLabels(GetGraph()->GetVectorBlocks().size());
537 labelEntry_ = encoder->CreateLabel();
538 labelExit_ = encoder->CreateLabel();
539
540 #ifndef NDEBUG
541 if (g_options.IsCompilerNonOptimizing()) {
542 // In case of non-optimizing compiler lowering pass is not run but low-level instructions may
543 // also appear on codegen so, to satisfy GraphChecker, the flag should be raised.
544 GetGraph()->SetLowLevelInstructionsEnabled();
545 }
546 #endif // NDEBUG
547
548 if ((GetCallingConvention() == nullptr) || (GetEncoder() == nullptr)) {
549 return false;
550 }
551
552 if (GetCallingConvention()->IsDynCallMode()) {
553 auto numExpectedArgs = GetRuntime()->GetMethodTotalArgumentsCount(GetGraph()->GetMethod());
554 if (numExpectedArgs > GetRuntime()->GetDynamicNumFixedArgs()) {
555 CallConvDynInfo dynInfo(numExpectedArgs,
556 GetRuntime()->GetEntrypointTlsOffset(
557 GetArch(), RuntimeInterface::EntrypointId::EXPAND_COMPILED_CODE_ARGS_DYN));
558 GetCallingConvention()->SetDynInfo(dynInfo);
559 frameInfo_->SetSaveFrameAndLinkRegs(true);
560 } else {
561 GetCallingConvention()->SetDynInfo(CallConvDynInfo());
562 }
563 }
564
565 if (!GetEncoder()->GetResult()) {
566 return false;
567 }
568
569 // Remove registers from the temp registers, if they are in the regalloc mask, i.e. available for regalloc.
570 auto usedRegs = ~GetGraph()->GetArchUsedRegs();
571 auto forbiddenTemps = usedRegs & GetTarget().GetTempRegsMask();
572 if (forbiddenTemps.Any()) {
573 for (size_t i = forbiddenTemps.GetMinRegister(); i <= forbiddenTemps.GetMaxRegister(); i++) {
574 if (forbiddenTemps[i] && encoder->IsScratchRegisterReleased(Reg(i, INT64_TYPE))) {
575 encoder->AcquireScratchRegister(Reg(i, INT64_TYPE));
576 }
577 }
578 }
579
580 return Finalize();
581 }
582
Initialize()583 void Codegen::Initialize()
584 {
585 CreateFrameInfo();
586
587 GetRegfile()->SetCalleeSaved(GetRegfile()->GetCalleeSaved());
588
589 if (!GetGraph()->SupportManagedCode()) {
590 for (auto inst : GetGraph()->GetStartBlock()->AllInsts()) {
591 if (inst->GetOpcode() == Opcode::LiveIn && GetTarget().GetTempRegsMask().Test(inst->GetDstReg())) {
592 GetEncoder()->AcquireScratchRegister(Reg(inst->GetDstReg(), INT64_TYPE));
593 }
594 }
595 }
596
597 bool hasCalls = false;
598
599 for (auto bb : GetGraph()->GetBlocksLinearOrder()) {
600 // Calls may be in the middle of method
601 for (auto inst : bb->Insts()) {
602 // For throw instruction need jump2runtime same way
603 if (inst->IsCall() || inst->GetOpcode() == Opcode::Throw) {
604 hasCalls = true;
605 break;
606 }
607 }
608 if (hasCalls) {
609 break;
610 }
611 }
612
613 /* Convert Graph::GetUsedRegs(), which is std::vector<bool>, to simple
614 * RegMask and save it in the Codegen. These masks are used to determine
615 * which registers we need to save in prologue.
616 *
617 * NB! It's related to IRTOC specific prologue generation (see CodegenFastPath etc.).
618 * Arch specific CallingConvention::GenerateProlog() relies on reg usage information
619 * prepared in the Codegen constructor (before Initialize() is called).
620 */
621 auto fillMask = [](RegMask *mask, auto *vector) {
622 if (vector == nullptr) {
623 return;
624 }
625 ASSERT(mask->size() >= vector->size());
626 mask->reset();
627 for (size_t i = 0; i < mask->size(); i++) {
628 if ((*vector)[i]) {
629 mask->set(i);
630 }
631 }
632 };
633 fillMask(&usedRegs_, GetGraph()->GetUsedRegs<DataType::INT64>());
634 fillMask(&usedVregs_, GetGraph()->GetUsedRegs<DataType::FLOAT64>());
635 usedVregs_ &= GetTarget().GetAvailableVRegsMask();
636 usedRegs_ &= GetTarget().GetAvailableRegsMask();
637 usedRegs_ &= ~GetGraph()->GetArchUsedRegs();
638 usedVregs_ &= ~GetGraph()->GetArchUsedVRegs();
639
640 /* Remove used registers from Encoder's scratch registers */
641 RegMask usedTemps = usedRegs_ & GetTarget().GetTempRegsMask();
642 if (usedTemps.any()) {
643 for (size_t i = 0; i < usedTemps.size(); i++) {
644 if (usedTemps[i]) {
645 GetEncoder()->AcquireScratchRegister(Reg(i, INT64_TYPE));
646 }
647 }
648 }
649 }
650
Finalize()651 bool Codegen::Finalize()
652 {
653 if (GetDisasm()->IsEnabled()) {
654 GetDisasm()->PrintMethodEntry(this);
655 }
656
657 if (!BeginMethod()) {
658 return false;
659 }
660
661 if (!VisitGraph()) {
662 return false;
663 }
664 EndMethod();
665
666 if (!CopyToCodeCache()) {
667 return false;
668 }
669
670 if (GetDisasm()->IsEnabled()) {
671 IssueDisasm();
672 }
673
674 return true;
675 }
676
ConvertRegister(Register r,DataType::Type type)677 Reg Codegen::ConvertRegister(Register r, DataType::Type type)
678 {
679 switch (type) {
680 case DataType::BOOL:
681 case DataType::UINT8:
682 case DataType::INT8: {
683 return Reg(r, INT8_TYPE);
684 }
685 case DataType::UINT16:
686 case DataType::INT16: {
687 return Reg(r, INT16_TYPE);
688 }
689 case DataType::UINT32:
690 case DataType::INT32: {
691 return Reg(r, INT32_TYPE);
692 }
693 case DataType::UINT64:
694 case DataType::INT64:
695 case DataType::ANY: {
696 return Reg(r, INT64_TYPE);
697 }
698 case DataType::FLOAT32: {
699 return Reg(r, FLOAT32_TYPE);
700 }
701 case DataType::FLOAT64: {
702 return Reg(r, FLOAT64_TYPE);
703 }
704 case DataType::REFERENCE: {
705 return ConvertRegister(r, DataType::GetIntTypeForReference(GetArch()));
706 }
707 case DataType::POINTER: {
708 return Reg(r, ConvertDataType(DataType::POINTER, GetArch()));
709 }
710 default:
711 // Invalid converted register
712 return INVALID_REGISTER;
713 }
714 }
715
716 // Panda don't support types less then 32, so we need sign or zero extended to 32
ConvertImmWithExtend(uint64_t imm,DataType::Type type)717 Imm Codegen::ConvertImmWithExtend(uint64_t imm, DataType::Type type)
718 {
719 switch (type) {
720 case DataType::BOOL:
721 case DataType::UINT8:
722 return Imm(static_cast<uint32_t>(static_cast<uint8_t>(imm)));
723 case DataType::INT8:
724 return Imm(static_cast<int32_t>(bit_cast<int8_t, uint8_t>(imm)));
725 case DataType::UINT16:
726 return Imm(static_cast<uint32_t>(static_cast<uint16_t>(imm)));
727 case DataType::INT16:
728 return Imm(static_cast<int32_t>(bit_cast<int16_t, uint16_t>(imm)));
729 // NOLINTNEXTLINE(bugprone-branch-clone)
730 case DataType::UINT32:
731 return Imm(bit_cast<int32_t, uint32_t>(imm));
732 case DataType::INT32:
733 return Imm(bit_cast<int32_t, uint32_t>(imm));
734 // NOLINTNEXTLINE(bugprone-branch-clone)
735 case DataType::UINT64:
736 return Imm(bit_cast<int64_t, uint64_t>(imm));
737 case DataType::INT64:
738 return Imm(bit_cast<int64_t, uint64_t>(imm));
739 case DataType::FLOAT32:
740 return Imm(bit_cast<float, uint32_t>(static_cast<uint32_t>(imm)));
741 case DataType::FLOAT64:
742 return Imm(bit_cast<double, uint64_t>(imm));
743 case DataType::ANY:
744 return Imm(bit_cast<uint64_t, uint64_t>(imm));
745 case DataType::REFERENCE:
746 if (imm == 0) {
747 return Imm(0);
748 }
749 [[fallthrough]]; /* fall-through */
750 default:
751 // Invalid converted immediate
752 UNREACHABLE();
753 }
754 return Imm(0);
755 }
756
ConvertCc(ConditionCode cc)757 Condition Codegen::ConvertCc(ConditionCode cc)
758 {
759 switch (cc) {
760 case CC_EQ:
761 return Condition::EQ;
762 case CC_NE:
763 return Condition::NE;
764 case CC_LT:
765 return Condition::LT;
766 case CC_LE:
767 return Condition::LE;
768 case CC_GT:
769 return Condition::GT;
770 case CC_GE:
771 return Condition::GE;
772 case CC_B:
773 return Condition::LO;
774 case CC_BE:
775 return Condition::LS;
776 case CC_A:
777 return Condition::HI;
778 case CC_AE:
779 return Condition::HS;
780 case CC_TST_EQ:
781 return Condition::TST_EQ;
782 case CC_TST_NE:
783 return Condition::TST_NE;
784 default:
785 UNREACHABLE();
786 return Condition::EQ;
787 }
788 return Condition::EQ;
789 }
790
ConvertCcOverflow(ConditionCode cc)791 Condition Codegen::ConvertCcOverflow(ConditionCode cc)
792 {
793 switch (cc) {
794 case CC_EQ:
795 return Condition::VS;
796 case CC_NE:
797 return Condition::VC;
798 default:
799 UNREACHABLE();
800 return Condition::VS;
801 }
802 return Condition::VS;
803 }
804
EmitSlowPaths()805 void Codegen::EmitSlowPaths()
806 {
807 for (auto slowPath : slowPaths_) {
808 slowPath->Generate(this);
809 }
810 }
811
CreateStackMap(Inst * inst,Inst * user)812 void Codegen::CreateStackMap(Inst *inst, Inst *user)
813 {
814 SaveStateInst *saveState = nullptr;
815 if (inst->IsSaveState()) {
816 saveState = static_cast<SaveStateInst *>(inst);
817 } else {
818 saveState = inst->GetSaveState();
819 }
820 ASSERT(saveState != nullptr);
821
822 bool requireVregMap = inst->RequireRegMap();
823 uint32_t outerBpc = inst->GetPc();
824 for (auto callInst = saveState->GetCallerInst(); callInst != nullptr;
825 callInst = callInst->GetSaveState()->GetCallerInst()) {
826 outerBpc = callInst->GetPc();
827 }
828 codeBuilder_->BeginStackMap(outerBpc, GetEncoder()->GetCursorOffset(), saveState->GetRootsStackMask(),
829 saveState->GetRootsRegsMask().to_ulong(), requireVregMap,
830 saveState->GetOpcode() == Opcode::SaveStateOsr);
831 if (user == nullptr) {
832 user = inst;
833 if (inst == saveState && inst->HasUsers()) {
834 auto users = inst->GetUsers();
835 auto it = std::find_if(users.begin(), users.end(),
836 [](auto &u) { return u.GetInst()->GetOpcode() != Opcode::ReturnInlined; });
837 user = it->GetInst();
838 }
839 }
840 CreateStackMapRec(saveState, requireVregMap, user);
841
842 codeBuilder_->EndStackMap();
843 if (GetDisasm()->IsEnabled()) {
844 GetDisasm()->PrintStackMap(this);
845 }
846 }
847
CreateStackMapRec(SaveStateInst * saveState,bool requireVregMap,Inst * targetSite)848 void Codegen::CreateStackMapRec(SaveStateInst *saveState, bool requireVregMap, Inst *targetSite)
849 {
850 bool hasInlineInfo = saveState->GetCallerInst() != nullptr;
851 size_t vregsCount = 0;
852 if (requireVregMap) {
853 auto runtime = GetRuntime();
854 if (auto caller = saveState->GetCallerInst()) {
855 vregsCount = runtime->GetMethodRegistersCount(caller->GetCallMethod()) +
856 runtime->GetMethodArgumentsCount(caller->GetCallMethod());
857 } else {
858 vregsCount = runtime->GetMethodRegistersCount(saveState->GetMethod()) +
859 runtime->GetMethodArgumentsCount(saveState->GetMethod());
860 }
861 ASSERT(!GetGraph()->IsBytecodeOptimizer());
862 // 1 for acc, number of env regs for dynamic method
863 vregsCount += 1U + GetGraph()->GetEnvCount();
864 #ifndef NDEBUG
865 ASSERT_PRINT(!saveState->GetInputsWereDeleted(), "Some vregs were deleted from the save state");
866 #endif
867 }
868
869 if (auto callInst = saveState->GetCallerInst()) {
870 CreateStackMapRec(callInst->GetSaveState(), requireVregMap, targetSite);
871 auto method = GetGraph()->IsAotMode() ? nullptr : callInst->GetCallMethod();
872 codeBuilder_->BeginInlineInfo(method, GetRuntime()->GetMethodId(callInst->GetCallMethod()), saveState->GetPc(),
873 vregsCount);
874 }
875
876 if (requireVregMap) {
877 CreateVRegMap(saveState, vregsCount, targetSite);
878 }
879
880 if (hasInlineInfo) {
881 codeBuilder_->EndInlineInfo();
882 }
883 }
884
CreateVRegMap(SaveStateInst * saveState,size_t vregsCount,Inst * targetSite)885 void Codegen::CreateVRegMap(SaveStateInst *saveState, size_t vregsCount, Inst *targetSite)
886 {
887 vregIndices_.clear();
888 vregIndices_.resize(vregsCount, {-1, -1});
889 FillVregIndices(saveState);
890
891 ASSERT(GetGraph()->IsAnalysisValid<LivenessAnalyzer>());
892 auto &la = GetGraph()->GetAnalysis<LivenessAnalyzer>();
893 auto targetLifeNumber = la.GetInstLifeIntervals(targetSite)->GetBegin();
894
895 for (auto &inputIndex : vregIndices_) {
896 if (inputIndex.first == -1 && inputIndex.second == -1) {
897 codeBuilder_->AddVReg(VRegInfo());
898 continue;
899 }
900 if (inputIndex.second != -1) {
901 auto imm = saveState->GetImmediate(inputIndex.second);
902 codeBuilder_->AddConstant(imm.value, IrTypeToMetainfoType(imm.type), imm.vregType);
903 continue;
904 }
905 ASSERT(inputIndex.first != -1);
906 auto vreg = saveState->GetVirtualRegister(inputIndex.first);
907 auto inputInst = saveState->GetDataFlowInput(inputIndex.first);
908 auto interval = la.GetInstLifeIntervals(inputInst)->FindSiblingAt(targetLifeNumber);
909 ASSERT(interval != nullptr);
910 CreateVreg(interval->GetLocation(), inputInst, vreg);
911 }
912 }
913
CreateVreg(const Location & location,Inst * inst,const VirtualRegister & vreg)914 void Codegen::CreateVreg(const Location &location, Inst *inst, const VirtualRegister &vreg)
915 {
916 switch (location.GetKind()) {
917 case LocationType::FP_REGISTER:
918 case LocationType::REGISTER: {
919 CreateVRegForRegister(location, inst, vreg);
920 break;
921 }
922 case LocationType::STACK_PARAMETER: {
923 auto slot = location.GetValue();
924 codeBuilder_->AddVReg(VRegInfo(GetFrameLayout().GetStackArgsStartSlot() - slot - CFrameSlots::Start(),
925 VRegInfo::Location::SLOT, IrTypeToMetainfoType(inst->GetType()),
926 vreg.GetVRegType()));
927 break;
928 }
929 case LocationType::STACK: {
930 auto slot = location.GetValue();
931 if (!Is64BitsArch(GetArch())) {
932 slot = ((location.GetValue() << 1U) + 1);
933 }
934 codeBuilder_->AddVReg(VRegInfo(GetFrameLayout().GetFirstSpillSlot() + slot, VRegInfo::Location::SLOT,
935 IrTypeToMetainfoType(inst->GetType()), vreg.GetVRegType()));
936 break;
937 }
938 case LocationType::IMMEDIATE: {
939 ASSERT(inst->IsConst());
940 auto type = inst->GetType();
941 // There are no int64 type for dynamic methods.
942 if (GetGraph()->IsDynamicMethod() && type == DataType::INT64) {
943 type = DataType::INT32;
944 }
945 codeBuilder_->AddConstant(inst->CastToConstant()->GetRawValue(), IrTypeToMetainfoType(type),
946 vreg.GetVRegType());
947 break;
948 }
949 default:
950 // Reg-to-reg spill fill must not occurs within SaveState
951 UNREACHABLE();
952 }
953 }
954
FillVregIndices(SaveStateInst * saveState)955 void Codegen::FillVregIndices(SaveStateInst *saveState)
956 {
957 for (size_t i = 0; i < saveState->GetInputsCount(); ++i) {
958 size_t vregIndex = saveState->GetVirtualRegister(i).Value();
959 if (vregIndex == VirtualRegister::BRIDGE) {
960 continue;
961 }
962 ASSERT(vregIndex < vregIndices_.size());
963 vregIndices_[vregIndex].first = i;
964 }
965 for (size_t i = 0; i < saveState->GetImmediatesCount(); i++) {
966 auto vregImm = saveState->GetImmediate(i);
967 if (vregImm.vreg == VirtualRegister::BRIDGE) {
968 continue;
969 }
970 ASSERT(vregImm.vreg < vregIndices_.size());
971 ASSERT(vregIndices_[vregImm.vreg].first == -1);
972 vregIndices_[vregImm.vreg].second = i;
973 }
974 }
975
CreateVRegForRegister(const Location & location,Inst * inst,const VirtualRegister & vreg)976 void Codegen::CreateVRegForRegister(const Location &location, Inst *inst, const VirtualRegister &vreg)
977 {
978 bool isOsr = GetGraph()->IsOsrMode();
979 bool isFp = (location.GetKind() == LocationType::FP_REGISTER);
980 auto regNum = location.GetValue();
981 auto reg = Reg(regNum, isFp ? FLOAT64_TYPE : INT64_TYPE);
982 if (!isOsr && GetRegfile()->GetZeroReg() == reg) {
983 codeBuilder_->AddConstant(0, IrTypeToMetainfoType(inst->GetType()), vreg.GetVRegType());
984 } else if (isOsr || GetRegfile()->IsCalleeRegister(reg)) {
985 if (isFp) {
986 ASSERT(inst->GetType() != DataType::REFERENCE);
987 ASSERT(isOsr || regNum >= GetFirstCalleeReg(GetArch(), true));
988 codeBuilder_->AddVReg(VRegInfo(regNum, VRegInfo::Location::FP_REGISTER,
989 IrTypeToMetainfoType(inst->GetType()), vreg.GetVRegType()));
990 } else {
991 ASSERT(isOsr || regNum >= GetFirstCalleeReg(GetArch(), false));
992 codeBuilder_->AddVReg(VRegInfo(regNum, VRegInfo::Location::REGISTER, IrTypeToMetainfoType(inst->GetType()),
993 vreg.GetVRegType()));
994 }
995 } else {
996 ASSERT(regNum >= GetFirstCallerReg(GetArch(), isFp));
997 auto lastSlot = GetFrameLayout().GetCallerLastSlot(isFp);
998 regNum -= GetFirstCallerReg(GetArch(), isFp);
999 codeBuilder_->AddVReg(VRegInfo(lastSlot - regNum, VRegInfo::Location::SLOT,
1000 IrTypeToMetainfoType(inst->GetType()), vreg.GetVRegType()));
1001 }
1002 }
1003
CreateOsrEntry(SaveStateInst * saveState)1004 void Codegen::CreateOsrEntry(SaveStateInst *saveState)
1005 {
1006 auto &stub = osrEntries_.emplace_back(GetAllocator()->New<OsrEntryStub>(this, saveState));
1007 GetEncoder()->BindLabel(stub->GetLabel());
1008 }
1009
CallIntrinsic(Inst * inst,RuntimeInterface::IntrinsicId id)1010 void Codegen::CallIntrinsic(Inst *inst, RuntimeInterface::IntrinsicId id)
1011 {
1012 SCOPED_DISASM_STR(this, "CallIntrinsic");
1013 // Call intrinsics isn't supported for IrToc, because we don't know address of intrinsics during IRToc compilation.
1014 // We store adresses of intrinsics in aot file in AOT mode.
1015 ASSERT(GetGraph()->SupportManagedCode());
1016 if (GetGraph()->IsAotMode()) {
1017 auto aotData = GetGraph()->GetAotData();
1018 intptr_t offset = aotData->GetEntrypointOffset(GetEncoder()->GetCursorOffset(), static_cast<int32_t>(id));
1019 GetEncoder()->MakeCallAot(offset);
1020 } else {
1021 GetEncoder()->MakeCall(reinterpret_cast<const void *>(GetRuntime()->GetIntrinsicAddress(
1022 inst->IsRuntimeCall(), GetRuntime()->GetMethodSourceLanguage(GetGraph()->GetMethod()), id)));
1023 }
1024 }
1025
EmitCallRuntimeCode(Inst * inst,std::variant<EntrypointId,Reg> entrypoint)1026 bool Codegen::EmitCallRuntimeCode(Inst *inst, std::variant<EntrypointId, Reg> entrypoint)
1027 {
1028 auto encoder = GetEncoder();
1029 if (std::holds_alternative<Reg>(entrypoint)) {
1030 auto reg = std::get<Reg>(entrypoint);
1031 ASSERT(reg.IsValid());
1032 encoder->MakeCall(reg);
1033 } else {
1034 auto id = std::get<EntrypointId>(entrypoint);
1035 MemRef entry(ThreadReg(), GetRuntime()->GetEntrypointTlsOffset(GetArch(), id));
1036 encoder->MakeCall(entry);
1037 }
1038
1039 SaveStateInst *saveState =
1040 (inst == nullptr || inst->IsSaveState()) ? static_cast<SaveStateInst *>(inst) : inst->GetSaveState();
1041 // StackMap should follow the call as the bridge function expects retaddr points to the stackmap
1042 if (saveState != nullptr) {
1043 CreateStackMap(inst);
1044 }
1045
1046 if (std::holds_alternative<EntrypointId>(entrypoint) &&
1047 GetRuntime()->IsEntrypointNoreturn(std::get<EntrypointId>(entrypoint))) {
1048 if constexpr (DEBUG_BUILD) { // NOLINT
1049 encoder->EncodeAbort();
1050 }
1051 return false;
1052 }
1053 ASSERT(saveState == nullptr || inst->IsRuntimeCall());
1054
1055 return true;
1056 }
1057
SaveRegistersForImplicitRuntime(Inst * inst,RegMask * paramsMask,RegMask * mask)1058 void Codegen::SaveRegistersForImplicitRuntime(Inst *inst, RegMask *paramsMask, RegMask *mask)
1059 {
1060 auto &rootsMask = inst->GetSaveState()->GetRootsRegsMask();
1061 for (auto i = 0U; i < inst->GetInputsCount(); i++) {
1062 auto input = inst->GetDataFlowInput(i);
1063 if (!input->IsMovableObject()) {
1064 continue;
1065 }
1066 auto location = inst->GetLocation(i);
1067 if (location.IsRegister() && location.IsRegisterValid()) {
1068 auto val = location.GetValue();
1069 auto reg = Reg(val, INT64_TYPE);
1070 GetEncoder()->SetRegister(mask, nullptr, reg, true);
1071 if (DataType::IsReference(inst->GetInputType(i)) && !rootsMask.test(val)) {
1072 paramsMask->set(val);
1073 rootsMask.set(val);
1074 }
1075 }
1076 }
1077 }
1078
CreateCheckForTLABWithConstSize(Inst * inst,Reg regTlabStart,Reg regTlabSize,size_t size,LabelHolder::LabelId label)1079 void Codegen::CreateCheckForTLABWithConstSize([[maybe_unused]] Inst *inst, Reg regTlabStart, Reg regTlabSize,
1080 size_t size, LabelHolder::LabelId label)
1081 {
1082 SCOPED_DISASM_STR(this, "CreateCheckForTLABWithConstSize");
1083 auto encoder = GetEncoder();
1084 if (encoder->CanEncodeImmAddSubCmp(size, WORD_SIZE, false)) {
1085 encoder->EncodeJump(label, regTlabSize, Imm(size), Condition::LO);
1086 // Change pointer to start free memory
1087 encoder->EncodeAdd(regTlabSize, regTlabStart, Imm(size));
1088 } else {
1089 ScopedTmpReg sizeReg(encoder);
1090 encoder->EncodeMov(sizeReg, Imm(size));
1091 encoder->EncodeJump(label, regTlabSize, sizeReg, Condition::LO);
1092 encoder->EncodeAdd(regTlabSize, regTlabStart, sizeReg);
1093 }
1094 }
1095
CreateDebugRuntimeCallsForNewObject(Inst * inst,Reg regTlabStart,size_t allocSize,RegMask preserved)1096 void Codegen::CreateDebugRuntimeCallsForNewObject(Inst *inst, [[maybe_unused]] Reg regTlabStart, size_t allocSize,
1097 RegMask preserved)
1098 {
1099 #if defined(PANDA_ASAN_ON) || defined(PANDA_TSAN_ON)
1100 CallRuntime(inst, EntrypointId::ANNOTATE_SANITIZERS, INVALID_REGISTER, preserved, regTlabStart,
1101 TypedImm(allocSize));
1102 #endif
1103 if (GetRuntime()->IsTrackTlabAlloc()) {
1104 CallRuntime(inst, EntrypointId::WRITE_TLAB_STATS, INVALID_REGISTER, preserved, regTlabStart,
1105 TypedImm(allocSize));
1106 }
1107 }
1108
CreateNewObjCall(NewObjectInst * newObj)1109 void Codegen::CreateNewObjCall(NewObjectInst *newObj)
1110 {
1111 auto dst = ConvertRegister(newObj->GetDstReg(), newObj->GetType());
1112 auto src = ConvertRegister(newObj->GetSrcReg(0), newObj->GetInputType(0));
1113 auto initClass = newObj->GetInput(0).GetInst();
1114 auto srcClass = ConvertRegister(newObj->GetSrcReg(0), DataType::POINTER);
1115 auto runtime = GetRuntime();
1116
1117 auto maxTlabSize = runtime->GetTLABMaxSize();
1118 if (maxTlabSize == 0 ||
1119 (initClass->GetOpcode() != Opcode::LoadAndInitClass && initClass->GetOpcode() != Opcode::LoadImmediate)) {
1120 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1121 return;
1122 }
1123 RuntimeInterface::ClassPtr klass;
1124 if (initClass->GetOpcode() == Opcode::LoadAndInitClass) {
1125 klass = initClass->CastToLoadAndInitClass()->GetClass();
1126 } else {
1127 ASSERT(initClass->GetOpcode() == Opcode::LoadImmediate);
1128 klass = initClass->CastToLoadImmediate()->GetClass();
1129 }
1130 if (klass == nullptr || !runtime->CanUseTlabForClass(klass)) {
1131 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1132 return;
1133 }
1134 auto classSize = runtime->GetClassSize(klass);
1135 auto alignment = runtime->GetTLABAlignment();
1136
1137 classSize = (classSize & ~(alignment - 1U)) + ((classSize % alignment) != 0U ? alignment : 0U);
1138 if (classSize > maxTlabSize) {
1139 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1140 return;
1141 }
1142 CallFastPath(newObj, EntrypointId::ALLOCATE_OBJECT_TLAB, dst, RegMask::GetZeroMask(), srcClass,
1143 TypedImm(classSize));
1144 }
1145
CreateNewObjCallOld(NewObjectInst * newObj)1146 void Codegen::CreateNewObjCallOld(NewObjectInst *newObj)
1147 {
1148 auto dst = ConvertRegister(newObj->GetDstReg(), newObj->GetType());
1149 auto src = ConvertRegister(newObj->GetSrcReg(0), newObj->GetInputType(0));
1150 auto initClass = newObj->GetInput(0).GetInst();
1151 auto runtime = GetRuntime();
1152 auto maxTlabSize = runtime->GetTLABMaxSize();
1153 auto encoder = GetEncoder();
1154
1155 if (maxTlabSize == 0 ||
1156 (initClass->GetOpcode() != Opcode::LoadAndInitClass && initClass->GetOpcode() != Opcode::LoadImmediate)) {
1157 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1158 return;
1159 }
1160 RuntimeInterface::ClassPtr klass;
1161 if (initClass->GetOpcode() == Opcode::LoadAndInitClass) {
1162 klass = initClass->CastToLoadAndInitClass()->GetClass();
1163 } else {
1164 ASSERT(initClass->GetOpcode() == Opcode::LoadImmediate);
1165 klass = initClass->CastToLoadImmediate()->GetClass();
1166 }
1167 if (klass == nullptr || !runtime->CanUseTlabForClass(klass)) {
1168 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1169 return;
1170 }
1171 auto classSize = runtime->GetClassSize(klass);
1172 auto alignment = runtime->GetTLABAlignment();
1173
1174 classSize = (classSize & ~(alignment - 1U)) + ((classSize % alignment) != 0U ? alignment : 0U);
1175 if (classSize > maxTlabSize) {
1176 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1177 return;
1178 }
1179 ScopedLiveTmpReg regTlabStart(encoder);
1180 ScopedLiveTmpReg regTlabSize(encoder);
1181
1182 auto slowPath = CreateSlowPath<SlowPathEntrypoint>(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS);
1183 CreateLoadTLABInformation(regTlabStart, regTlabSize);
1184 CreateCheckForTLABWithConstSize(newObj, regTlabStart, regTlabSize, classSize, slowPath->GetLabel());
1185
1186 RegMask preservedRegs;
1187 encoder->SetRegister(&preservedRegs, nullptr, src);
1188 CreateDebugRuntimeCallsForNewObject(newObj, regTlabStart, reinterpret_cast<size_t>(classSize), preservedRegs);
1189
1190 ScopedTmpReg tlabReg(encoder);
1191 // Load pointer to tlab
1192 encoder->EncodeLdr(tlabReg, false, MemRef(ThreadReg(), runtime->GetCurrentTLABOffset(GetArch())));
1193
1194 // Store pointer to the class
1195 encoder->EncodeStr(src, MemRef(regTlabStart, runtime->GetObjClassOffset(GetArch())));
1196 encoder->EncodeMov(dst, regTlabStart);
1197 regTlabStart.Release();
1198 // Store new pointer to start free memory in TLAB
1199 encoder->EncodeStrRelease(regTlabSize, MemRef(tlabReg, runtime->GetTLABFreePointerOffset(GetArch())));
1200 slowPath->BindBackLabel(encoder);
1201 }
1202
LoadClassFromObject(Reg classReg,Reg objReg)1203 void Codegen::LoadClassFromObject(Reg classReg, Reg objReg)
1204 {
1205 Reg reg = ConvertRegister(classReg.GetId(), DataType::REFERENCE);
1206 GetEncoder()->EncodeLdr(reg, false, MemRef(objReg, GetRuntime()->GetObjClassOffset(GetArch())));
1207 }
1208
CreateMultiArrayCall(CallInst * callInst)1209 void Codegen::CreateMultiArrayCall(CallInst *callInst)
1210 {
1211 SCOPED_DISASM_STR(this, "Create Call for MultiArray");
1212
1213 auto dstReg = ConvertRegister(callInst->GetDstReg(), callInst->GetType());
1214 auto numArgs = callInst->GetInputsCount() - 2U; // first is class, last is save_state
1215
1216 ScopedTmpReg classReg(GetEncoder());
1217 auto classType = ConvertDataType(DataType::REFERENCE, GetArch());
1218 Reg classOrig = classReg.GetReg().As(classType);
1219 auto location = callInst->GetLocation(0);
1220 ASSERT(location.IsFixedRegister() && location.IsRegisterValid());
1221
1222 GetEncoder()->EncodeMov(classOrig, ConvertRegister(location.GetValue(), DataType::INT32));
1223 CallRuntime(callInst, EntrypointId::CREATE_MULTI_ARRAY, dstReg, RegMask::GetZeroMask(), classReg, TypedImm(numArgs),
1224 SpReg());
1225 if (callInst->GetFlag(inst_flags::MEM_BARRIER)) {
1226 GetEncoder()->EncodeMemoryBarrier(memory_order::RELEASE);
1227 }
1228 }
1229
CreateJumpToClassResolverPltShared(Inst * inst,Reg tmpReg,RuntimeInterface::EntrypointId id)1230 void Codegen::CreateJumpToClassResolverPltShared(Inst *inst, Reg tmpReg, RuntimeInterface::EntrypointId id)
1231 {
1232 auto encoder = GetEncoder();
1233 auto graph = GetGraph();
1234 auto aotData = graph->GetAotData();
1235 auto offset = aotData->GetSharedSlowPathOffset(id, encoder->GetCursorOffset());
1236 if (offset == 0 || !encoder->CanMakeCallByOffset(offset)) {
1237 SlowPathShared *slowPath;
1238 auto search = slowPathsMap_.find(id);
1239 if (search != slowPathsMap_.end()) {
1240 slowPath = search->second;
1241 ASSERT(slowPath->GetTmpReg().GetId() == tmpReg.GetId());
1242 } else {
1243 slowPath = CreateSlowPath<SlowPathShared>(inst, id);
1244 slowPath->SetTmpReg(tmpReg);
1245 slowPathsMap_[id] = slowPath;
1246 }
1247 encoder->MakeCall(slowPath->GetLabel());
1248 } else {
1249 encoder->MakeCallByOffset(offset);
1250 }
1251 CreateStackMap(inst);
1252 }
1253
CreateLoadClassFromPLT(Inst * inst,Reg tmpReg,Reg dst,size_t classId)1254 void Codegen::CreateLoadClassFromPLT(Inst *inst, Reg tmpReg, Reg dst, size_t classId)
1255 {
1256 auto encoder = GetEncoder();
1257 auto graph = GetGraph();
1258 auto aotData = graph->GetAotData();
1259 intptr_t offset = aotData->GetClassSlotOffset(encoder->GetCursorOffset(), classId, false);
1260 auto label = encoder->CreateLabel();
1261 ASSERT(tmpReg.GetId() != dst.GetId());
1262 ASSERT(inst->IsRuntimeCall());
1263 encoder->MakeLoadAotTableAddr(offset, tmpReg, dst);
1264 encoder->EncodeJump(label, dst, Condition::NE);
1265
1266 // PLT Class Resolver has special calling convention:
1267 // First encoder temporary (tmp_reg) works as parameter and return value
1268 CHECK_EQ(tmpReg.GetId(), encoder->GetTarget().GetTempRegsMask().GetMinRegister());
1269
1270 CreateJumpToClassResolverPltShared(inst, tmpReg, EntrypointId::CLASS_RESOLVER);
1271
1272 encoder->EncodeMov(dst, tmpReg);
1273 encoder->BindLabel(label);
1274 }
1275
CreateLoadTLABInformation(Reg regTlabStart,Reg regTlabSize)1276 void Codegen::CreateLoadTLABInformation(Reg regTlabStart, Reg regTlabSize)
1277 {
1278 SCOPED_DISASM_STR(this, "LoadTLABInformation");
1279 auto runtime = GetRuntime();
1280 // Load pointer to tlab
1281 GetEncoder()->EncodeLdr(regTlabSize, false, MemRef(ThreadReg(), runtime->GetCurrentTLABOffset(GetArch())));
1282 // Load pointer to start free memory in TLAB
1283 GetEncoder()->EncodeLdr(regTlabStart, false, MemRef(regTlabSize, runtime->GetTLABFreePointerOffset(GetArch())));
1284 // Load pointer to end free memory in TLAB
1285 GetEncoder()->EncodeLdr(regTlabSize, false, MemRef(regTlabSize, runtime->GetTLABEndPointerOffset(GetArch())));
1286 // Calculate size of the free memory
1287 GetEncoder()->EncodeSub(regTlabSize, regTlabSize, regTlabStart);
1288 }
1289
1290 // The function alignment up the value from alignment_reg using tmp_reg.
CreateAlignmentValue(Reg alignmentReg,Reg tmpReg,size_t alignment)1291 void Codegen::CreateAlignmentValue(Reg alignmentReg, Reg tmpReg, size_t alignment)
1292 {
1293 auto andVal = ~(alignment - 1U);
1294 // zeroed lower bits
1295 GetEncoder()->EncodeAnd(tmpReg, alignmentReg, Imm(andVal));
1296 GetEncoder()->EncodeSub(alignmentReg, alignmentReg, tmpReg);
1297
1298 auto endLabel = GetEncoder()->CreateLabel();
1299
1300 // if zeroed value is different, add alignment
1301 GetEncoder()->EncodeJump(endLabel, alignmentReg, Condition::EQ);
1302 GetEncoder()->EncodeAdd(tmpReg, tmpReg, Imm(alignment));
1303
1304 GetEncoder()->BindLabel(endLabel);
1305 GetEncoder()->EncodeMov(alignmentReg, tmpReg);
1306 }
1307
GetObjectReg(Codegen * codegen,Inst * inst)1308 inline Reg GetObjectReg(Codegen *codegen, Inst *inst)
1309 {
1310 ASSERT(inst->IsCall() || inst->GetOpcode() == Opcode::ResolveVirtual);
1311 auto location = inst->GetLocation(0);
1312 ASSERT(location.IsFixedRegister() && location.IsRegisterValid());
1313 auto objReg = codegen->ConvertRegister(location.GetValue(), inst->GetInputType(0));
1314 ASSERT(objReg != INVALID_REGISTER);
1315 return objReg;
1316 }
1317
CreateCallIntrinsic(IntrinsicInst * inst)1318 void Codegen::CreateCallIntrinsic(IntrinsicInst *inst)
1319 {
1320 if (inst->HasArgumentsOnStack()) {
1321 // Since for some intrinsics(f.e. TimedWaitNanos) we need SaveState instruction and
1322 // more than two arguments, we need to place arguments on the stack, but in same time we need to
1323 // create boundary frame
1324 LOG(WARNING, COMPILER) << "Intrinsics with arguments on stack are not supported";
1325 GetEncoder()->SetFalseResult();
1326 return;
1327 }
1328
1329 ASSERT(!HasLiveCallerSavedRegs(inst));
1330 if (inst->HasImms() && GetGraph()->SupportManagedCode()) {
1331 EncodeImms(inst->GetImms(), (inst->HasIdInput() || inst->IsMethodFirstInput()));
1332 }
1333
1334 size_t explicitArgs;
1335 if (IsStackRangeIntrinsic(inst->GetIntrinsicId(), &explicitArgs)) {
1336 auto paramInfo = GetCallingConvention()->GetParameterInfo(explicitArgs);
1337 auto rangePtrReg =
1338 ConvertRegister(paramInfo->GetNextLocation(DataType::POINTER).GetRegister(), DataType::POINTER);
1339 if (inst->GetInputsCount() > explicitArgs + (inst->RequireState() ? 1U : 0U)) {
1340 auto rangeSpOffs = GetStackOffset(inst->GetLocation(explicitArgs));
1341 GetEncoder()->EncodeAdd(rangePtrReg, GetTarget().GetStackReg(), Imm(rangeSpOffs));
1342 }
1343 }
1344 if (inst->IsMethodFirstInput()) {
1345 Reg param0 = GetTarget().GetParamReg(0);
1346 if (GetGraph()->IsAotMode()) {
1347 LoadMethod(param0);
1348 } else {
1349 GetEncoder()->EncodeMov(param0, Imm(reinterpret_cast<uintptr_t>(inst->GetMethod())));
1350 }
1351 }
1352 CallIntrinsic(inst, inst->GetIntrinsicId());
1353
1354 if (inst->GetSaveState() != nullptr) {
1355 // StackMap should follow the call as the bridge function expects retaddr points to the stackmap
1356 CreateStackMap(inst);
1357 }
1358
1359 if (inst->GetType() != DataType::VOID) {
1360 auto arch = GetArch();
1361 auto returnType = inst->GetType();
1362 auto dstReg = ConvertRegister(inst->GetDstReg(), inst->GetType());
1363 auto returnReg = GetTarget().GetReturnReg(dstReg.GetType());
1364 // We must:
1365 // sign extended INT8 and INT16 to INT32
1366 // zero extended UINT8 and UINT16 to UINT32
1367 if (DataType::ShiftByType(returnType, arch) < DataType::ShiftByType(DataType::INT32, arch)) {
1368 bool isSigned = DataType::IsTypeSigned(returnType);
1369 GetEncoder()->EncodeCast(dstReg, isSigned, Reg(returnReg.GetId(), INT32_TYPE), isSigned);
1370 } else {
1371 GetEncoder()->EncodeMov(dstReg, returnReg);
1372 }
1373 }
1374 }
1375
IntfInlineCachePass(ResolveVirtualInst * resolver,Reg methodReg,Reg tmpReg,Reg objReg)1376 void Codegen::IntfInlineCachePass(ResolveVirtualInst *resolver, Reg methodReg, Reg tmpReg, Reg objReg)
1377 {
1378 // Cache structure:（offset addr)/(class addr) 32bit/32bit
1379 // -----------------------------------------------
1380 // (.aot_got)
1381 // ...
1382 // cache:offset/class <----------|
1383 // ... |
1384 // (.text) |
1385 // interface call start |
1386 // call runtime irtoc function
1387 // if call class == cache.class <------|
1388 // use cache.offset(method) <------|
1389 // else |
1390 // call RESOLVE_VIRTUAL_CALL_AOT |
1391 // save method‘s offset to cache >--|
1392 // return to (.text)
1393 // call method
1394 // -----------------------------------------------
1395 auto aotData = GetGraph()->GetAotData();
1396 uint64_t intfInlineCacheIndex = aotData->GetIntfInlineCacheIndex();
1397 // NOTE(liyiming): do LoadMethod in irtoc to reduce use tmp reg
1398 if (objReg.GetId() != tmpReg.GetId()) {
1399 auto regTmp64 = tmpReg.As(INT64_TYPE);
1400 uint64_t offset = aotData->GetInfInlineCacheSlotOffset(GetEncoder()->GetCursorOffset(), intfInlineCacheIndex);
1401 GetEncoder()->MakeLoadAotTableAddr(offset, regTmp64, INVALID_REGISTER);
1402 LoadMethod(methodReg);
1403 CallFastPath(resolver, EntrypointId::INTF_INLINE_CACHE, methodReg, {}, methodReg, objReg,
1404 TypedImm(resolver->GetCallMethodId()), regTmp64);
1405 } else {
1406 // we don't have tmp reg here, so use x3 directly
1407 Reg reg3 = Reg(3U, INT64_TYPE);
1408 ScopedTmpRegF64 vtmp(GetEncoder());
1409 GetEncoder()->EncodeMov(vtmp, reg3);
1410 uint64_t offset = aotData->GetInfInlineCacheSlotOffset(GetEncoder()->GetCursorOffset(), intfInlineCacheIndex);
1411 GetEncoder()->MakeLoadAotTableAddr(offset, reg3, INVALID_REGISTER);
1412 LoadMethod(methodReg);
1413 CallFastPath(resolver, EntrypointId::INTF_INLINE_CACHE, methodReg, {}, methodReg, objReg,
1414 TypedImm(resolver->GetCallMethodId()), reg3);
1415 GetEncoder()->EncodeMov(reg3, vtmp);
1416 }
1417
1418 intfInlineCacheIndex++;
1419 aotData->SetIntfInlineCacheIndex(intfInlineCacheIndex);
1420 }
1421
1422 /**
1423 * Returns a pointer to a caller of a virtual/static resolver.
1424 *
1425 * InstBuilder uses UnresolvedTypesInterface::AddTableSlot(method_ptr, ...) to allocate cache slots
1426 * for unresolved methods. The codegen uses UnresolvedTypesInterface::GetTableSlot(method_ptr, ...)
1427 * to get corresponding slots for unresolved methods.
1428 *
1429 * Since there is no valid method_ptr for an unresolved method at the point of IR construction
1430 * InstBuilder::BuildCallInst uses the pointer to the caller (GetGraph()->GetMethod()) as an argument
1431 * of UnresolvedTypesInterface::AddTableSlot. So the codegen has to deduce the pointer to the caller
1432 * and pass it to UnresolvedTypesInterface::GetTableSlot. If the caller was not inlined then
1433 * GetGraph()->GetMethod() works just fine, otherwise it is done by means of SaveState,
1434 * which is capable of remebering the corresponding caller instruction.
1435 *
1436 * To clarify the details let's consider the following sequence of calls:
1437 *
1438 * main() -> foo() -> bar()
1439 *
1440 * Suppose that foo() is a static method and bar() is an unresolved virtual method.
1441 *
1442 * 1) foo() is going to be inlined
1443 *
1444 * To inline foo() the inlining pass creates its IR by means of IRBuilder(..., caller_inst),
1445 * where caller_inst (the third argument) represents "CallStatic foo" instruction.
1446 * The caller_inst gets remembered in the corresponding SaveState created for bar's resolver
1447 * (see InstBuilder::CreateSaveState for the details).
1448 * At the same time "CallStatic foo" instruction contains a valid method pointer to foo()
1449 * (it can not be nullptr because we do not inline Unresolved).
1450 *
1451 * So we go through the following chain: bar's resolver->SaveState->caller_inst->pointer to the method.
1452 *
1453 * 2) foo() is not inlined
1454 *
1455 * In this case SaveState.caller_inst_ == nullptr (because foo() is not inlined).
1456 * Thus we just use plain GetGraph()->GetMethod().
1457 */
1458 template <typename T>
GetCallerOfUnresolvedMethod(T * resolver)1459 RuntimeInterface::MethodPtr Codegen::GetCallerOfUnresolvedMethod(T *resolver)
1460 {
1461 ASSERT(resolver->GetCallMethod() == nullptr);
1462 auto saveState = resolver->GetSaveState();
1463 ASSERT(saveState != nullptr);
1464 auto caller = saveState->GetCallerInst();
1465 auto method = (caller == nullptr ? GetGraph()->GetMethod() : caller->GetCallMethod());
1466 ASSERT(method != nullptr);
1467 return method;
1468 }
1469
EmitResolveUnknownVirtual(ResolveVirtualInst * resolver,Reg methodReg)1470 void Codegen::EmitResolveUnknownVirtual(ResolveVirtualInst *resolver, Reg methodReg)
1471 {
1472 SCOPED_DISASM_STR(this, "Create runtime call to resolve an unknown virtual method");
1473 ASSERT(resolver->GetOpcode() == Opcode::ResolveVirtual);
1474 auto method = GetCallerOfUnresolvedMethod(resolver);
1475 ScopedTmpReg tmpReg(GetEncoder(), ConvertDataType(DataType::REFERENCE, GetArch()));
1476 Reg objReg = GetObjectReg(this, resolver);
1477 if (GetGraph()->IsAotMode()) {
1478 LoadMethod(methodReg);
1479 CallRuntime(resolver, EntrypointId::RESOLVE_VIRTUAL_CALL_AOT, methodReg, {}, methodReg, objReg,
1480 TypedImm(resolver->GetCallMethodId()), TypedImm(0));
1481 } else {
1482 auto runtime = GetRuntime();
1483 auto utypes = runtime->GetUnresolvedTypes();
1484 auto skind = UnresolvedTypesInterface::SlotKind::VIRTUAL_METHOD;
1485 // Try to load vtable index
1486 auto methodSlotAddr = utypes->GetTableSlot(method, resolver->GetCallMethodId(), skind);
1487 GetEncoder()->EncodeMov(methodReg, Imm(methodSlotAddr));
1488 GetEncoder()->EncodeLdr(methodReg, false, MemRef(methodReg));
1489 // 0 means the virtual call is uninitialized or it is an interface call
1490 auto entrypoint = EntrypointId::RESOLVE_UNKNOWN_VIRTUAL_CALL;
1491 auto slowPath = CreateSlowPath<SlowPathUnresolved>(resolver, entrypoint);
1492 slowPath->SetUnresolvedType(method, resolver->GetCallMethodId());
1493 slowPath->SetDstReg(methodReg);
1494 slowPath->SetArgReg(objReg);
1495 slowPath->SetSlotAddr(methodSlotAddr);
1496 GetEncoder()->EncodeJump(slowPath->GetLabel(), methodReg, Condition::EQ);
1497 // Load klass into tmp_reg
1498 LoadClassFromObject(tmpReg, objReg);
1499 // Load from VTable, address = (klass + (index << shift)) + vtable_offset
1500 auto tmpReg64 = Reg(tmpReg.GetReg().GetId(), INT64_TYPE);
1501 GetEncoder()->EncodeAdd(tmpReg64, tmpReg, Shift(methodReg, GetVtableShift()));
1502 GetEncoder()->EncodeLdr(methodReg, false,
1503 MemRef(tmpReg64, runtime->GetVTableOffset(GetArch()) - (1U << GetVtableShift())));
1504 slowPath->BindBackLabel(GetEncoder());
1505 }
1506 }
1507
EmitResolveVirtualAot(ResolveVirtualInst * resolver,Reg methodReg)1508 void Codegen::EmitResolveVirtualAot(ResolveVirtualInst *resolver, Reg methodReg)
1509 {
1510 SCOPED_DISASM_STR(this, "AOT ResolveVirtual using PLT-GOT");
1511 ASSERT(resolver->IsRuntimeCall());
1512 ScopedTmpReg tmpReg(GetEncoder(), ConvertDataType(DataType::REFERENCE, GetArch()));
1513 auto methodReg64 = Reg(methodReg.GetId(), INT64_TYPE);
1514 auto tmpReg64 = Reg(tmpReg.GetReg().GetId(), INT64_TYPE);
1515 auto aotData = GetGraph()->GetAotData();
1516 intptr_t offset = aotData->GetVirtIndexSlotOffset(GetEncoder()->GetCursorOffset(), resolver->GetCallMethodId());
1517 GetEncoder()->MakeLoadAotTableAddr(offset, tmpReg64, methodReg64);
1518 auto label = GetEncoder()->CreateLabel();
1519 GetEncoder()->EncodeJump(label, methodReg, Condition::NE);
1520 GetEncoder()->EncodeMov(methodReg64, tmpReg64);
1521 // PLT CallVirtual Resolver has a very special calling convention:
1522 // First encoder temporary (method_reg) works as a parameter and return value
1523 CHECK_EQ(methodReg64.GetId(), GetTarget().GetTempRegsMask().GetMinRegister());
1524 MemRef entry(ThreadReg(), GetRuntime()->GetEntrypointTlsOffset(GetArch(), EntrypointId::CALL_VIRTUAL_RESOLVER));
1525 GetEncoder()->MakeCall(entry);
1526 // Need a stackmap to build correct boundary frame
1527 CreateStackMap(resolver);
1528 GetEncoder()->BindLabel(label);
1529 // Load class into method_reg
1530 Reg objReg = GetObjectReg(this, resolver);
1531 LoadClassFromObject(tmpReg, objReg);
1532 // Load from VTable, address = (klass + (index << shift)) + vtable_offset
1533 GetEncoder()->EncodeAdd(methodReg, tmpReg64, Shift(methodReg, GetVtableShift()));
1534 GetEncoder()->EncodeLdr(methodReg, false,
1535 MemRef(methodReg, GetRuntime()->GetVTableOffset(GetArch()) - (1U << GetVtableShift())));
1536 }
1537
EmitResolveVirtual(ResolveVirtualInst * resolver)1538 void Codegen::EmitResolveVirtual(ResolveVirtualInst *resolver)
1539 {
1540 auto methodReg = ConvertRegister(resolver->GetDstReg(), resolver->GetType());
1541 auto objectReg = ConvertRegister(resolver->GetSrcReg(0), DataType::REFERENCE);
1542 ScopedTmpReg tmpMethodReg(GetEncoder());
1543 if (resolver->GetCallMethod() == nullptr) {
1544 EmitResolveUnknownVirtual(resolver, tmpMethodReg);
1545 GetEncoder()->EncodeMov(methodReg, tmpMethodReg);
1546 } else if (GetRuntime()->IsInterfaceMethod(resolver->GetCallMethod())) {
1547 SCOPED_DISASM_STR(this, "Create runtime call to resolve a known virtual call");
1548 if (GetGraph()->IsAotMode()) {
1549 if (GetArch() == Arch::AARCH64) {
1550 ScopedTmpReg tmpReg(GetEncoder(), ConvertDataType(DataType::REFERENCE, GetArch()));
1551 IntfInlineCachePass(resolver, tmpMethodReg, tmpReg, objectReg);
1552 } else {
1553 LoadMethod(tmpMethodReg);
1554 CallRuntime(resolver, EntrypointId::RESOLVE_VIRTUAL_CALL_AOT, tmpMethodReg, {}, tmpMethodReg, objectReg,
1555 TypedImm(resolver->GetCallMethodId()), TypedImm(0));
1556 }
1557 } else {
1558 CallRuntime(resolver, EntrypointId::RESOLVE_VIRTUAL_CALL, tmpMethodReg, {},
1559 TypedImm(reinterpret_cast<size_t>(resolver->GetCallMethod())), objectReg);
1560 }
1561 GetEncoder()->EncodeMov(methodReg, tmpMethodReg);
1562 } else if (GetGraph()->IsAotNoChaMode()) {
1563 // ResolveVirtualAot requires method_reg to be the first tmp register.
1564 EmitResolveVirtualAot(resolver, tmpMethodReg);
1565 GetEncoder()->EncodeMov(methodReg, tmpMethodReg);
1566 } else {
1567 UNREACHABLE();
1568 }
1569 }
1570
EmitCallResolvedVirtual(CallInst * call)1571 void Codegen::EmitCallResolvedVirtual(CallInst *call)
1572 {
1573 SCOPED_DISASM_STR(this, "Create a call to resolved virtual method");
1574 ASSERT(call->GetOpcode() == Opcode::CallResolvedVirtual);
1575 if (call->GetSaveState() != nullptr && call->IsInlined()) {
1576 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
1577 if (!GetGraph()->IsAotMode()) {
1578 InsertTrace({Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
1579 Imm(reinterpret_cast<size_t>(call->GetCallMethod())),
1580 Imm(static_cast<size_t>(events::MethodEnterKind::INLINED))});
1581 }
1582 #endif
1583 return;
1584 }
1585 ASSERT(!HasLiveCallerSavedRegs(call));
1586 ASSERT(call->GetCallMethod() == nullptr || GetGraph()->GetRuntime()->IsInterfaceMethod(call->GetCallMethod()) ||
1587 GetGraph()->IsAotNoChaMode());
1588 Reg methodReg = ConvertRegister(call->GetSrcReg(0), DataType::POINTER);
1589 Reg param0 = GetTarget().GetParamReg(0);
1590 // Set method
1591 GetEncoder()->EncodeMov(param0, methodReg);
1592 GetEncoder()->MakeCall(MemRef(param0, GetRuntime()->GetCompiledEntryPointOffset(GetArch())));
1593 FinalizeCall(call);
1594 }
1595
EmitCallVirtual(CallInst * call)1596 void Codegen::EmitCallVirtual(CallInst *call)
1597 {
1598 SCOPED_DISASM_STR(this, "Create a call to virtual method");
1599 ASSERT(call->GetOpcode() == Opcode::CallVirtual);
1600 if (call->GetSaveState() != nullptr && call->IsInlined()) {
1601 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
1602 if (!GetGraph()->IsAotMode()) {
1603 InsertTrace({Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
1604 Imm(reinterpret_cast<size_t>(call->GetCallMethod())),
1605 Imm(static_cast<size_t>(events::MethodEnterKind::INLINED))});
1606 }
1607 #endif
1608 return;
1609 }
1610 auto runtime = GetRuntime();
1611 auto method = call->GetCallMethod();
1612 ASSERT(!HasLiveCallerSavedRegs(call));
1613 ASSERT(!call->IsUnresolved() && !runtime->IsInterfaceMethod(method) && !GetGraph()->IsAotNoChaMode());
1614 Reg methodReg = GetTarget().GetParamReg(0);
1615 ASSERT(!RegisterKeepCallArgument(call, methodReg));
1616 LoadClassFromObject(methodReg, GetObjectReg(this, call));
1617 // Get index
1618 auto vtableIndex = runtime->GetVTableIndex(method);
1619 // Load from VTable, address = klass + ((index << shift) + vtable_offset)
1620 auto totalOffset = runtime->GetVTableOffset(GetArch()) + (vtableIndex << GetVtableShift());
1621 // Class ref was loaded to method_reg
1622 GetEncoder()->EncodeLdr(methodReg, false, MemRef(methodReg, totalOffset));
1623 // Set method
1624 GetEncoder()->MakeCall(MemRef(methodReg, runtime->GetCompiledEntryPointOffset(GetArch())));
1625 FinalizeCall(call);
1626 }
1627
EnsureParamsFitIn32Bit(std::initializer_list<std::variant<Reg,TypedImm>> params)1628 [[maybe_unused]] bool Codegen::EnsureParamsFitIn32Bit(std::initializer_list<std::variant<Reg, TypedImm>> params)
1629 {
1630 for (auto ¶m : params) {
1631 if (std::holds_alternative<Reg>(param)) {
1632 auto reg = std::get<Reg>(param);
1633 if (reg.GetSize() > WORD_SIZE) {
1634 return false;
1635 }
1636 } else {
1637 auto immType = std::get<TypedImm>(param).GetType();
1638 if (immType.GetSize() > WORD_SIZE) {
1639 return false;
1640 }
1641 }
1642 }
1643 return true;
1644 }
1645
EmitResolveStatic(ResolveStaticInst * resolver)1646 void Codegen::EmitResolveStatic(ResolveStaticInst *resolver)
1647 {
1648 auto methodReg = ConvertRegister(resolver->GetDstReg(), resolver->GetType());
1649 if (GetGraph()->IsAotMode()) {
1650 LoadMethod(methodReg);
1651 CallRuntime(resolver, EntrypointId::GET_UNKNOWN_CALLEE_METHOD, methodReg, RegMask::GetZeroMask(), methodReg,
1652 TypedImm(resolver->GetCallMethodId()), TypedImm(0));
1653 return;
1654 }
1655 auto method = GetCallerOfUnresolvedMethod(resolver);
1656 ScopedTmpReg tmp(GetEncoder());
1657 auto utypes = GetRuntime()->GetUnresolvedTypes();
1658 auto skind = UnresolvedTypesInterface::SlotKind::METHOD;
1659 auto methodAddr = utypes->GetTableSlot(method, resolver->GetCallMethodId(), skind);
1660 GetEncoder()->EncodeMov(tmp.GetReg(), Imm(methodAddr));
1661 GetEncoder()->EncodeLdr(tmp.GetReg(), false, MemRef(tmp.GetReg()));
1662 auto slowPath = CreateSlowPath<SlowPathUnresolved>(resolver, EntrypointId::GET_UNKNOWN_CALLEE_METHOD);
1663 slowPath->SetUnresolvedType(method, resolver->GetCallMethodId());
1664 slowPath->SetDstReg(tmp.GetReg());
1665 slowPath->SetSlotAddr(methodAddr);
1666 GetEncoder()->EncodeJump(slowPath->GetLabel(), tmp.GetReg(), Condition::EQ);
1667 slowPath->BindBackLabel(GetEncoder());
1668 GetEncoder()->EncodeMov(methodReg, tmp.GetReg());
1669 }
1670
EmitCallResolvedStatic(CallInst * call)1671 void Codegen::EmitCallResolvedStatic(CallInst *call)
1672 {
1673 ASSERT(call->GetOpcode() == Opcode::CallResolvedStatic && call->GetCallMethod() == nullptr);
1674 Reg methodReg = ConvertRegister(call->GetSrcReg(0), DataType::POINTER);
1675 Reg param0 = GetTarget().GetParamReg(0);
1676 GetEncoder()->EncodeMov(param0, methodReg);
1677 GetEncoder()->MakeCall(MemRef(param0, GetRuntime()->GetCompiledEntryPointOffset(GetArch())));
1678 FinalizeCall(call);
1679 }
1680
EmitCallStatic(CallInst * call)1681 void Codegen::EmitCallStatic(CallInst *call)
1682 {
1683 ASSERT(call->IsStaticCall() && !call->IsUnresolved());
1684 if (call->GetSaveState() != nullptr && call->IsInlined()) {
1685 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
1686 if (!GetGraph()->IsAotMode()) {
1687 InsertTrace({Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
1688 Imm(reinterpret_cast<size_t>(call->GetCallMethod())),
1689 Imm(static_cast<size_t>(events::MethodEnterKind::INLINED))});
1690 }
1691 #endif
1692 return;
1693 }
1694 SCOPED_DISASM_STR(this, "Create a call to static");
1695 ASSERT(!HasLiveCallerSavedRegs(call));
1696 // Now MakeCallByOffset is not supported in Arch32Encoder (support ADR instruction)
1697 Reg param0 = GetTarget().GetParamReg(0);
1698 if (call->GetCallMethod() == GetGraph()->GetMethod() && GetArch() != Arch::AARCH32 && !GetGraph()->IsOsrMode() &&
1699 !GetGraph()->GetMethodProperties().GetHasDeopt()) {
1700 if (GetGraph()->IsAotMode()) {
1701 LoadMethod(param0);
1702 } else {
1703 GetEncoder()->EncodeMov(param0, Imm(reinterpret_cast<size_t>(GetGraph()->GetMethod())));
1704 }
1705 GetEncoder()->MakeCallByOffset(GetStartCodeOffset() - GetEncoder()->GetCursorOffset());
1706 } else {
1707 if (GetGraph()->IsAotMode()) {
1708 auto aotData = GetGraph()->GetAotData();
1709 intptr_t offset = aotData->GetPltSlotOffset(GetEncoder()->GetCursorOffset(), call->GetCallMethodId());
1710 // PLT CallStatic Resolver transparently uses param_0 (Method) register
1711 GetEncoder()->MakeLoadAotTable(offset, param0);
1712 } else { // usual JIT case
1713 auto method = call->GetCallMethod();
1714 GetEncoder()->EncodeMov(param0, Imm(reinterpret_cast<size_t>(method)));
1715 }
1716 GetEncoder()->MakeCall(MemRef(param0, GetRuntime()->GetCompiledEntryPointOffset(GetArch())));
1717 }
1718 FinalizeCall(call);
1719 }
1720
EmitCallDynamic(CallInst * call)1721 void Codegen::EmitCallDynamic(CallInst *call)
1722 {
1723 SCOPED_DISASM_STR(this, "Create a dynamic call");
1724 if (call->GetSaveState() != nullptr && call->IsInlined()) {
1725 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
1726 if (!GetGraph()->IsAotMode()) {
1727 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
1728 Imm(reinterpret_cast<size_t>(call->GetCallMethod())),
1729 Imm(static_cast<size_t>(events::MethodEnterKind::INLINED)));
1730 }
1731 #endif
1732 return;
1733 }
1734 RuntimeInterface *runtime = GetRuntime();
1735 Encoder *encoder = GetEncoder();
1736
1737 auto dstReg = ConvertRegister(call->GetDstReg(), call->GetType());
1738 Reg methodParamReg = GetTarget().GetParamReg(CallConvDynInfo::REG_METHOD).As(GetPtrRegType());
1739 Reg numArgsParamReg = GetTarget().GetParamReg(CallConvDynInfo::REG_NUM_ARGS);
1740 auto paramFuncObjLoc = Location::MakeStackArgument(CallConvDynInfo::SLOT_CALLEE);
1741
1742 ASSERT(!HasLiveCallerSavedRegs(call));
1743
1744 // Load method from callee object
1745 static_assert(coretypes::TaggedValue::TAG_OBJECT == 0);
1746 encoder->EncodeLdr(methodParamReg, false, MemRef(SpReg(), GetStackOffset(paramFuncObjLoc)));
1747 encoder->EncodeLdr(methodParamReg, false, MemRef(methodParamReg, runtime->GetFunctionTargetOffset(GetArch())));
1748
1749 ASSERT(call->GetInputsCount() > 1);
1750 auto numArgs = static_cast<uint32_t>(call->GetInputsCount() - 1); // '-1' means 1 for spill fill input
1751 encoder->EncodeMov(numArgsParamReg, Imm(numArgs));
1752
1753 size_t entryPointOffset = runtime->GetCompiledEntryPointOffset(GetArch());
1754 encoder->MakeCall(MemRef(methodParamReg, entryPointOffset));
1755
1756 CreateStackMap(call);
1757 // Dynamic callee may have moved sp if there was insufficient num_actual_args
1758 encoder->EncodeSub(
1759 SpReg(), FpReg(),
1760 Imm(GetFrameLayout().GetOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::BYTES>(0)));
1761
1762 if (dstReg.IsValid()) {
1763 Reg retReg = GetTarget().GetReturnReg(dstReg.GetType());
1764 encoder->EncodeMov(dstReg, retReg);
1765 }
1766 }
1767
FinalizeCall(CallInst * call)1768 void Codegen::FinalizeCall(CallInst *call)
1769 {
1770 ASSERT(!call->IsDynamicCall());
1771 CreateStackMap(call);
1772 auto returnType = call->GetType();
1773 auto dstReg = ConvertRegister(call->GetDstReg(), returnType);
1774 // Restore frame pointer in the TLS
1775 GetEncoder()->EncodeStr(FpReg(), MemRef(ThreadReg(), GetRuntime()->GetTlsFrameOffset(GetArch())));
1776 // Sign/Zero extend return_reg if necessary
1777 if (dstReg.IsValid()) {
1778 auto arch = GetArch();
1779 auto returnReg = GetTarget().GetReturnReg(dstReg.GetType());
1780 // INT8 and INT16 must be sign extended to INT32
1781 // UINT8 and UINT16 must be zero extended to UINT32
1782 if (DataType::ShiftByType(returnType, arch) < DataType::ShiftByType(DataType::INT32, arch)) {
1783 bool isSigned = DataType::IsTypeSigned(returnType);
1784 GetEncoder()->EncodeCast(dstReg, isSigned, Reg(returnReg.GetId(), INT32_TYPE), isSigned);
1785 } else {
1786 GetEncoder()->EncodeMov(dstReg, returnReg);
1787 }
1788 }
1789 }
1790
1791 template <typename T>
GetBarrierOperandValue(RuntimeInterface * runtime,panda::mem::BarrierPosition position,std::string_view name)1792 static T GetBarrierOperandValue(RuntimeInterface *runtime, panda::mem::BarrierPosition position, std::string_view name)
1793 {
1794 auto operand = runtime->GetBarrierOperand(position, name);
1795 return std::get<T>(operand.GetValue());
1796 }
1797
1798 template <bool IS_CLASS>
CreatePreWRB(Inst * inst,MemRef mem,RegMask preserved,bool storePair)1799 void Codegen::CreatePreWRB(Inst *inst, MemRef mem, RegMask preserved, bool storePair)
1800 {
1801 auto runtime = GetRuntime();
1802 auto *enc = GetEncoder();
1803
1804 SCOPED_DISASM_STR(this, "Pre WRB");
1805 ScopedTmpReg entrypointReg(enc, enc->IsLrAsTempRegEnabledAndReleased());
1806 GetEncoder()->EncodeLdr(entrypointReg, false,
1807 MemRef(ThreadReg(), runtime->GetTlsPreWrbEntrypointOffset(GetArch())));
1808
1809 // Check entrypoint address
1810 auto label = GetEncoder()->CreateLabel();
1811 enc->EncodeJump(label, entrypointReg, Condition::EQ);
1812 auto refType =
1813 inst->GetType() == DataType::REFERENCE ? DataType::GetIntTypeForReference(enc->GetArch()) : DataType::INT64;
1814 ScopedTmpReg tmpRef(enc, ConvertDataType(refType, GetArch()));
1815 auto prevOffset = enc->GetCursorOffset();
1816 // Load old value
1817 if (IsVolatileMemInst(inst)) {
1818 enc->EncodeLdrAcquire(tmpRef, false, mem);
1819 } else {
1820 enc->EncodeLdr(tmpRef, false, mem);
1821 }
1822 TryInsertImplicitNullCheck(inst, prevOffset);
1823 if constexpr (IS_CLASS) {
1824 enc->EncodeLdr(tmpRef, false, MemRef(tmpRef, runtime->GetManagedClassOffset(GetArch())));
1825 } else {
1826 CheckObject(tmpRef, label);
1827 }
1828 auto [liveRegs, liveVregs] = GetLiveRegisters<true>(inst);
1829 liveRegs |= preserved;
1830 CallBarrier(liveRegs, liveVregs, entrypointReg.GetReg(), tmpRef);
1831
1832 if (storePair) {
1833 // store pair doesn't support index and scalar
1834 ASSERT(!mem.HasIndex() && !mem.HasScale());
1835 // calculate offset for second store
1836 auto secondOffset = 1U << DataType::ShiftByType(DataType::REFERENCE, enc->GetArch());
1837 if (mem.HasDisp()) {
1838 secondOffset += mem.GetDisp();
1839 }
1840 // Load old value
1841 if (IsVolatileMemInst(inst)) {
1842 enc->EncodeLdrAcquire(tmpRef, false, MemRef(mem.GetBase(), secondOffset));
1843 } else {
1844 enc->EncodeLdr(tmpRef, false, MemRef(mem.GetBase(), secondOffset));
1845 }
1846 CheckObject(tmpRef, label);
1847 CallBarrier(liveRegs, liveVregs, entrypointReg.GetReg(), tmpRef);
1848 }
1849 enc->BindLabel(label);
1850 }
1851
EncodePostWRB(Inst * inst,MemRef mem,Reg reg1,Reg reg2,bool checkObject)1852 void Codegen::EncodePostWRB(Inst *inst, MemRef mem, Reg reg1, Reg reg2, bool checkObject)
1853 {
1854 auto refType {TypeInfo::FromDataType(DataType::REFERENCE, GetArch())};
1855 ASSERT(reg1.IsValid());
1856 reg1 = reg1.As(refType);
1857 if (reg2.IsValid()) {
1858 reg2 = reg2.As(refType);
1859 }
1860
1861 if (GetGraph()->SupportsIrtocBarriers()) {
1862 if (GetGraph()->IsOfflineCompilationMode()) {
1863 CreateOfflineIrtocPostWrb(inst, mem, reg1, reg2);
1864 } else {
1865 CreateOnlineIrtocPostWrb(inst, mem, reg1, reg2, checkObject);
1866 }
1867 return;
1868 }
1869
1870 auto barrierType {GetRuntime()->GetPostType()};
1871 ASSERT(barrierType == panda::mem::BarrierType::POST_INTERGENERATIONAL_BARRIER ||
1872 barrierType == panda::mem::BarrierType::POST_INTERREGION_BARRIER);
1873
1874 if (barrierType == panda::mem::BarrierType::POST_INTERREGION_BARRIER) {
1875 CreatePostInterRegionBarrier(inst, mem, reg1, reg2, checkObject);
1876 } else {
1877 auto base {mem.GetBase().As(refType)};
1878 CreatePostInterGenerationalBarrier(base);
1879 }
1880 }
1881
CreatePostWRBForDynamic(Inst * inst,MemRef mem,Reg reg1,Reg reg2)1882 void Codegen::CreatePostWRBForDynamic(Inst *inst, MemRef mem, Reg reg1, Reg reg2)
1883 {
1884 int storeIndex;
1885 if (reg2 == INVALID_REGISTER) {
1886 if (inst->GetOpcode() == Opcode::StoreObject || inst->GetOpcode() == Opcode::StoreI ||
1887 inst->GetOpcode() == Opcode::StoreArrayI) {
1888 storeIndex = 1_I;
1889 } else {
1890 ASSERT(inst->GetOpcode() == Opcode::StoreArray || inst->GetOpcode() == Opcode::Store);
1891 storeIndex = 2_I;
1892 }
1893 if (StoreValueCanBeObject(inst->GetDataFlowInput(storeIndex))) {
1894 EncodePostWRB(inst, mem, reg1, reg2, true);
1895 }
1896 } else {
1897 if (inst->GetOpcode() == Opcode::StoreArrayPairI) {
1898 storeIndex = 1_I;
1899 } else {
1900 ASSERT(inst->GetOpcode() == Opcode::StoreArrayPair);
1901 storeIndex = 2_I;
1902 }
1903 bool firstIsObject = StoreValueCanBeObject(inst->GetDataFlowInput(storeIndex));
1904 bool secondIsObject = StoreValueCanBeObject(inst->GetDataFlowInput(storeIndex + 1));
1905 if (firstIsObject || secondIsObject) {
1906 if (firstIsObject && !secondIsObject) {
1907 reg2 = INVALID_REGISTER;
1908 } else if (!firstIsObject && secondIsObject) {
1909 reg1 = reg2;
1910 reg2 = INVALID_REGISTER;
1911 }
1912 EncodePostWRB(inst, mem, reg1, reg2, true);
1913 }
1914 }
1915 }
1916
CreatePostWRB(Inst * inst,MemRef mem,Reg reg1,Reg reg2)1917 void Codegen::CreatePostWRB(Inst *inst, MemRef mem, Reg reg1, Reg reg2)
1918 {
1919 ASSERT(reg1 != INVALID_REGISTER);
1920
1921 if (!GetGraph()->SupportsIrtocBarriers() || !GetGraph()->IsOfflineCompilationMode()) {
1922 auto barrierType = GetRuntime()->GetPostType();
1923 if (barrierType == panda::mem::BarrierType::POST_WRB_NONE) {
1924 return;
1925 }
1926 ASSERT(barrierType == panda::mem::BarrierType::POST_INTERGENERATIONAL_BARRIER ||
1927 barrierType == panda::mem::BarrierType::POST_INTERREGION_BARRIER);
1928 }
1929
1930 // For dynamic methods, another check
1931 if (GetGraph()->IsDynamicMethod()) {
1932 CreatePostWRBForDynamic(inst, mem, reg1, reg2);
1933 return;
1934 }
1935 Inst *secondValue;
1936 Inst *val = InstStoredValue(inst, &secondValue);
1937 ASSERT(secondValue == nullptr || reg2 != INVALID_REGISTER);
1938 if (val->GetOpcode() == Opcode::NullPtr) {
1939 // We can don't encode Post barrier for nullptr
1940 if (secondValue == nullptr || secondValue->GetOpcode() == Opcode::NullPtr) {
1941 return;
1942 }
1943 // CallPostWRB only for second reg
1944 EncodePostWRB(inst, mem, reg2, INVALID_REGISTER, !IsInstNotNull(secondValue));
1945 return;
1946 }
1947 // Create PostWRB only for first value
1948 if (secondValue != nullptr && secondValue->GetOpcode() == Opcode::NullPtr) {
1949 reg2 = INVALID_REGISTER;
1950 }
1951 bool checkObject = true;
1952 if (reg2 == INVALID_REGISTER) {
1953 if (IsInstNotNull(val)) {
1954 checkObject = false;
1955 }
1956 } else {
1957 if (IsInstNotNull(val) && IsInstNotNull(secondValue)) {
1958 checkObject = false;
1959 }
1960 }
1961 EncodePostWRB(inst, mem, reg1, reg2, checkObject);
1962 }
1963
CheckObject(Reg reg,LabelHolder::LabelId label)1964 void Codegen::CheckObject(Reg reg, LabelHolder::LabelId label)
1965 {
1966 auto graph = GetGraph();
1967 auto *enc = GetEncoder();
1968
1969 // interpreter use x20 in IrToc and we don't have enough temporary registers
1970 // remove after PR 98 or PR 47
1971 if (graph->IsDynamicMethod()) {
1972 ASSERT(reg.IsScalar());
1973 reg = reg.As(INT64_TYPE);
1974 auto tagMask = graph->GetRuntime()->GetTaggedTagMask();
1975 // Check that the value is object(not int and not double)
1976 enc->EncodeJumpTest(label, reg, Imm(tagMask), Condition::TST_NE);
1977 auto specialMask = graph->GetRuntime()->GetTaggedSpecialMask();
1978 // Check that the value is not special value
1979 enc->EncodeJumpTest(label, reg, Imm(~specialMask), Condition::TST_EQ);
1980 } else {
1981 enc->EncodeJump(label, reg, Condition::EQ);
1982 }
1983 }
1984
CreateOfflineIrtocPostWrb(Inst * inst,MemRef mem,Reg reg1,Reg reg2)1985 void Codegen::CreateOfflineIrtocPostWrb(Inst *inst, MemRef mem, Reg reg1, Reg reg2)
1986 {
1987 ASSERT(reg1.IsValid());
1988
1989 bool hasObj2 {reg2.IsValid() && reg1 != reg2};
1990
1991 auto paramRegs {GetTarget().GetParamRegsMask(hasObj2 ? 4U : 3U) & GetLiveRegisters(inst).first};
1992 SaveCallerRegisters(paramRegs, VRegMask(), false);
1993
1994 if (hasObj2) {
1995 FillPostWrbCallParams(mem, reg1, reg2);
1996 } else {
1997 FillPostWrbCallParams(mem, reg1);
1998 }
1999
2000 // load function pointer from tls field
2001 auto offset {hasObj2 ? cross_values::GetManagedThreadPostWrbTwoObjectsOffset(GetArch())
2002 : cross_values::GetManagedThreadPostWrbOneObjectOffset(GetArch())};
2003 MemRef entry(ThreadReg(), offset);
2004 GetEncoder()->MakeCall(entry);
2005
2006 LoadCallerRegisters(paramRegs, VRegMask(), false);
2007 }
2008
CheckObj(Encoder * enc,Codegen * cg,Reg base,Reg reg1,LabelHolder::LabelId skipLabel,bool checkNull)2009 void CheckObj(Encoder *enc, Codegen *cg, Reg base, Reg reg1, LabelHolder::LabelId skipLabel, bool checkNull)
2010 {
2011 if (checkNull) {
2012 // Fast null check in-place for one register
2013 enc->EncodeJump(skipLabel, reg1, Condition::EQ);
2014 }
2015
2016 ScopedTmpReg tmp(enc, cg->ConvertDataType(DataType::REFERENCE, cg->GetArch()));
2017 auto regionSizeBit = GetBarrierOperandValue<uint8_t>(
2018 cg->GetRuntime(), panda::mem::BarrierPosition::BARRIER_POSITION_POST, "REGION_SIZE_BITS");
2019 enc->EncodeXor(tmp, base, reg1);
2020 enc->EncodeShr(tmp, tmp, Imm(regionSizeBit));
2021 enc->EncodeJump(skipLabel, tmp, Condition::EQ);
2022 }
2023
WrapOneArg(Encoder * enc,Reg param,Reg base,MemRef mem,size_t additionalOffset=0)2024 void WrapOneArg(Encoder *enc, Reg param, Reg base, MemRef mem, size_t additionalOffset = 0)
2025 {
2026 if (mem.HasIndex()) {
2027 ASSERT(mem.GetScale() == 0 && !mem.HasDisp());
2028 enc->EncodeAdd(param, base, mem.GetIndex());
2029 if (additionalOffset != 0) {
2030 enc->EncodeAdd(param, param, Imm(additionalOffset));
2031 }
2032 } else if (mem.HasDisp()) {
2033 ASSERT(!mem.HasIndex());
2034 enc->EncodeAdd(param, base, Imm(mem.GetDisp() + additionalOffset));
2035 } else {
2036 enc->EncodeAdd(param, base, Imm(additionalOffset));
2037 }
2038 }
2039
2040 /**
2041 * Post-write barrier for StorePair case.
2042 * The code below may mark 0, 1 or 2 cards. The idea is following:
2043 * if (the second object isn't null and is allocated in other from base object's region)
2044 * MarkOneCard(CardOfSecondField(mem))
2045 * if (address of the second field isn't aligned at the size of a card)
2046 * # i.e. each of store address (fields of the base objects) are related to the same card
2047 * goto Done
2048 *
2049 * if (the first object isn't null and is allocated in other from base object's region)
2050 * MarkOneCard(CardOfFirstField(mem))
2051 *
2052 * label: Done
2053 */
CreateOnlineIrtocPostWrbRegionTwoRegs(Inst * inst,MemRef mem,Reg reg1,Reg reg2,bool checkObject)2054 void Codegen::CreateOnlineIrtocPostWrbRegionTwoRegs(Inst *inst, MemRef mem, Reg reg1, Reg reg2, bool checkObject)
2055 {
2056 auto entrypointId {EntrypointId::POST_INTER_REGION_BARRIER_SLOW};
2057 auto enc {GetEncoder()};
2058 auto base {mem.GetBase().As(TypeInfo::FromDataType(DataType::REFERENCE, GetArch()))};
2059 auto paramReg0 = enc->GetTarget().GetParamReg(0);
2060 MemRef entry(ThreadReg(), GetRuntime()->GetEntrypointTlsOffset(GetArch(), entrypointId));
2061 constexpr size_t PARAMS_NUM = 1U;
2062 auto paramRegs {GetTarget().GetParamRegsMask(PARAMS_NUM) & GetLiveRegisters(inst).first};
2063
2064 auto lblMarkCardAndExit = enc->CreateLabel();
2065 auto lblCheck1Obj = enc->CreateLabel();
2066 auto done = enc->CreateLabel();
2067
2068 CheckObj(enc, this, base, reg2, lblCheck1Obj, checkObject);
2069 SaveCallerRegisters(paramRegs, VRegMask(), false);
2070 WrapOneArg(enc, paramReg0, base, mem, reg1.GetSize() / BITS_PER_BYTE);
2071 {
2072 ScopedTmpReg tmp(enc, ConvertDataType(DataType::REFERENCE, GetArch()));
2073 enc->EncodeAnd(tmp, paramReg0, Imm(cross_values::GetCardAlignmentMask(GetArch())));
2074 enc->EncodeJump(lblMarkCardAndExit, tmp, Condition::NE);
2075 }
2076
2077 enc->MakeCall(entry);
2078 LoadCallerRegisters(paramRegs, VRegMask(), false);
2079
2080 enc->BindLabel(lblCheck1Obj);
2081 CheckObj(enc, this, base, reg1, done, checkObject);
2082 SaveCallerRegisters(paramRegs, VRegMask(), false);
2083 WrapOneArg(enc, paramReg0, base, mem);
2084 enc->BindLabel(lblMarkCardAndExit);
2085 enc->MakeCall(entry);
2086 LoadCallerRegisters(paramRegs, VRegMask(), false);
2087 enc->BindLabel(done);
2088 }
2089
CreateOnlineIrtocPostWrbRegionOneReg(Inst * inst,MemRef mem,Reg reg1,bool checkObject)2090 void Codegen::CreateOnlineIrtocPostWrbRegionOneReg(Inst *inst, MemRef mem, Reg reg1, bool checkObject)
2091 {
2092 auto entrypointId {EntrypointId::POST_INTER_REGION_BARRIER_SLOW};
2093 auto enc {GetEncoder()};
2094 auto base {mem.GetBase().As(TypeInfo::FromDataType(DataType::REFERENCE, GetArch()))};
2095 auto paramReg0 = enc->GetTarget().GetParamReg(0);
2096 MemRef entry(ThreadReg(), GetRuntime()->GetEntrypointTlsOffset(GetArch(), entrypointId));
2097 constexpr size_t PARAMS_NUM = 1;
2098 auto paramRegs {GetTarget().GetParamRegsMask(PARAMS_NUM) & GetLiveRegisters(inst).first};
2099
2100 auto skip = enc->CreateLabel();
2101
2102 CheckObj(enc, this, base, reg1, skip, checkObject);
2103 SaveCallerRegisters(paramRegs, VRegMask(), false);
2104 WrapOneArg(enc, paramReg0, base, mem);
2105 enc->MakeCall(entry);
2106 LoadCallerRegisters(paramRegs, VRegMask(), false);
2107 enc->BindLabel(skip);
2108 }
2109
CreateOnlineIrtocPostWrb(Inst * inst,MemRef mem,Reg reg1,Reg reg2,bool checkObject)2110 void Codegen::CreateOnlineIrtocPostWrb(Inst *inst, MemRef mem, Reg reg1, Reg reg2, bool checkObject)
2111 {
2112 SCOPED_DISASM_STR(this, "Post Online Irtoc-WRB");
2113 ASSERT(reg1.IsValid());
2114
2115 auto enc {GetEncoder()};
2116
2117 bool hasObj2 {reg2.IsValid() && reg1 != reg2};
2118 if (GetRuntime()->GetPostType() == panda::mem::BarrierType::POST_INTERREGION_BARRIER) {
2119 if (hasObj2) {
2120 CreateOnlineIrtocPostWrbRegionTwoRegs(inst, mem, reg1, reg2, checkObject);
2121 } else {
2122 CreateOnlineIrtocPostWrbRegionOneReg(inst, mem, reg1, checkObject);
2123 }
2124 } else {
2125 auto entrypointId {EntrypointId::POST_INTER_GENERATIONAL_BARRIER0};
2126 auto base {mem.GetBase().As(TypeInfo::FromDataType(DataType::REFERENCE, GetArch()))};
2127 constexpr size_t PARAMS_NUM = 1;
2128 auto paramRegs {GetTarget().GetParamRegsMask(PARAMS_NUM) & GetLiveRegisters(inst).first};
2129
2130 SaveCallerRegisters(paramRegs, VRegMask(), false);
2131 auto paramReg0 = enc->GetTarget().GetParamReg(0);
2132 enc->EncodeMov(paramReg0, base);
2133 MemRef entry(ThreadReg(), GetRuntime()->GetEntrypointTlsOffset(GetArch(), entrypointId));
2134 enc->MakeCall(entry);
2135 LoadCallerRegisters(paramRegs, VRegMask(), false);
2136 }
2137 }
2138
CreatePostInterRegionBarrier(Inst * inst,MemRef mem,Reg reg1,Reg reg2,bool checkObject)2139 void Codegen::CreatePostInterRegionBarrier(Inst *inst, MemRef mem, Reg reg1, Reg reg2, bool checkObject)
2140 {
2141 SCOPED_DISASM_STR(this, "Post IR-WRB");
2142 auto *enc = GetEncoder();
2143 ASSERT(GetRuntime()->GetPostType() == panda::mem::BarrierType::POST_INTERREGION_BARRIER);
2144 ASSERT(reg1 != INVALID_REGISTER);
2145
2146 auto label = GetEncoder()->CreateLabel();
2147
2148 if (checkObject) {
2149 CheckObject(reg1, label);
2150 }
2151
2152 auto regionSizeBit = GetBarrierOperandValue<uint8_t>(
2153 GetRuntime(), panda::mem::BarrierPosition::BARRIER_POSITION_POST, "REGION_SIZE_BITS");
2154
2155 auto base {mem.GetBase().As(TypeInfo::FromDataType(DataType::REFERENCE, GetArch()))};
2156 ScopedTmpReg tmp(enc, ConvertDataType(DataType::REFERENCE, GetArch()));
2157 // Compare first store value with mem
2158 enc->EncodeXor(tmp, base, reg1);
2159 enc->EncodeShr(tmp, tmp, Imm(regionSizeBit));
2160
2161 enc->EncodeJump(label, tmp, Condition::EQ);
2162 auto [liveRegs, liveVregs] = GetLiveRegisters<true>(inst);
2163
2164 if (mem.HasIndex()) {
2165 ASSERT(mem.GetScale() == 0 && !mem.HasDisp());
2166 enc->EncodeAdd(tmp, base, mem.GetIndex());
2167 CallBarrier(liveRegs, liveVregs, EntrypointId::POST_WRB_UPDATE_CARD_FUNC_NO_BRIDGE, tmp, reg1);
2168 } else if (mem.HasDisp()) {
2169 ASSERT(!mem.HasIndex());
2170 enc->EncodeAdd(tmp, base, Imm(mem.GetDisp()));
2171 CallBarrier(liveRegs, liveVregs, EntrypointId::POST_WRB_UPDATE_CARD_FUNC_NO_BRIDGE, tmp, reg1);
2172 } else {
2173 CallBarrier(liveRegs, liveVregs, EntrypointId::POST_WRB_UPDATE_CARD_FUNC_NO_BRIDGE, base, reg1);
2174 }
2175 enc->BindLabel(label);
2176
2177 if (reg2.IsValid() && reg1 != reg2) {
2178 auto label1 = GetEncoder()->CreateLabel();
2179 if (checkObject) {
2180 CheckObject(reg2, label1);
2181 }
2182
2183 // Compare second store value with mem
2184 enc->EncodeXor(tmp, base, reg2);
2185 enc->EncodeShr(tmp, tmp, Imm(regionSizeBit));
2186 enc->EncodeJump(label1, tmp, Condition::EQ);
2187
2188 enc->EncodeAdd(tmp, base, Imm(reg1.GetSize() / BITS_PER_BYTE));
2189 if (mem.HasIndex()) {
2190 ASSERT(mem.GetScale() == 0 && !mem.HasDisp());
2191 enc->EncodeAdd(tmp, tmp, mem.GetIndex());
2192 } else if (mem.HasDisp()) {
2193 ASSERT(!mem.HasIndex());
2194 enc->EncodeAdd(tmp, tmp, Imm(mem.GetDisp()));
2195 }
2196 CallBarrier(liveRegs, liveVregs, EntrypointId::POST_WRB_UPDATE_CARD_FUNC_NO_BRIDGE, tmp, reg2);
2197 enc->BindLabel(label1);
2198 }
2199 }
2200
CalculateCardIndex(Reg baseReg,ScopedTmpReg * tmp,ScopedTmpReg * tmp1)2201 void Codegen::CalculateCardIndex(Reg baseReg, ScopedTmpReg *tmp, ScopedTmpReg *tmp1)
2202 {
2203 auto tmpType = tmp->GetReg().GetType();
2204 auto tmp1Type = tmp1->GetReg().GetType();
2205 auto *enc = GetEncoder();
2206 auto cardBits =
2207 GetBarrierOperandValue<uint8_t>(GetRuntime(), panda::mem::BarrierPosition::BARRIER_POSITION_POST, "CARD_BITS");
2208
2209 ASSERT(baseReg != INVALID_REGISTER);
2210 if (baseReg.GetSize() < Reg(*tmp).GetSize()) {
2211 tmp->ChangeType(baseReg.GetType());
2212 tmp1->ChangeType(baseReg.GetType());
2213 }
2214 enc->EncodeSub(*tmp, baseReg, *tmp);
2215 enc->EncodeShr(*tmp, *tmp, Imm(cardBits));
2216 tmp->ChangeType(tmpType);
2217 tmp1->ChangeType(tmp1Type);
2218 }
2219
CreatePostInterGenerationalBarrier(Reg base)2220 void Codegen::CreatePostInterGenerationalBarrier(Reg base)
2221 {
2222 SCOPED_DISASM_STR(this, "Post IG-WRB");
2223 auto runtime = GetRuntime();
2224 auto *enc = GetEncoder();
2225 ASSERT(runtime->GetPostType() == panda::mem::BarrierType::POST_INTERGENERATIONAL_BARRIER);
2226 ScopedTmpReg tmp(enc);
2227 ScopedTmpReg tmp1(enc);
2228 // * load AddressOf(MIN_ADDR) -> min_addr
2229 if (GetGraph()->IsOfflineCompilationMode()) {
2230 GetEncoder()->EncodeLdr(tmp, false, MemRef(ThreadReg(), runtime->GetTlsCardTableMinAddrOffset(GetArch())));
2231 } else {
2232 auto minAddress = reinterpret_cast<uintptr_t>(
2233 GetBarrierOperandValue<void *>(runtime, panda::mem::BarrierPosition::BARRIER_POSITION_POST, "MIN_ADDR"));
2234 enc->EncodeMov(tmp, Imm(minAddress));
2235 }
2236 // * card_index = (AddressOf(obj.field) - min_addr) >> CARD_BITS // shift right
2237 CalculateCardIndex(base, &tmp, &tmp1);
2238 // * load AddressOf(CARD_TABLE_ADDR) -> card_table_addr
2239 if (GetGraph()->IsOfflineCompilationMode()) {
2240 GetEncoder()->EncodeLdr(tmp1.GetReg().As(INT64_TYPE), false,
2241 MemRef(ThreadReg(), runtime->GetTlsCardTableAddrOffset(GetArch())));
2242 } else {
2243 auto cardTableAddr = reinterpret_cast<uintptr_t>(GetBarrierOperandValue<uint8_t *>(
2244 runtime, panda::mem::BarrierPosition::BARRIER_POSITION_POST, "CARD_TABLE_ADDR"));
2245 enc->EncodeMov(tmp1, Imm(cardTableAddr));
2246 }
2247 // * card_addr = card_table_addr + card_index
2248 enc->EncodeAdd(tmp, tmp1, tmp);
2249 // * store card_addr <- DIRTY_VAL
2250 auto dirtyVal =
2251 GetBarrierOperandValue<uint8_t>(runtime, panda::mem::BarrierPosition::BARRIER_POSITION_POST, "DIRTY_VAL");
2252
2253 auto tmp1B = ConvertRegister(tmp1.GetReg().GetId(), DataType::INT8);
2254 enc->EncodeMov(tmp1B, Imm(dirtyVal));
2255 enc->EncodeStr(tmp1B, MemRef(tmp));
2256 }
2257
HasLiveCallerSavedRegs(Inst * inst)2258 bool Codegen::HasLiveCallerSavedRegs(Inst *inst)
2259 {
2260 auto [liveRegs, liveFpRegs] = GetLiveRegisters<false>(inst);
2261 liveRegs &= GetCallerRegsMask(GetArch(), false);
2262 liveFpRegs &= GetCallerRegsMask(GetArch(), true);
2263 return liveRegs.Any() || liveFpRegs.Any();
2264 }
2265
SaveCallerRegisters(RegMask liveRegs,VRegMask liveVregs,bool adjustRegs)2266 void Codegen::SaveCallerRegisters(RegMask liveRegs, VRegMask liveVregs, bool adjustRegs)
2267 {
2268 SCOPED_DISASM_STR(this, "Save caller saved regs");
2269 auto base = GetFrameInfo()->GetCallersRelativeFp() ? GetTarget().GetFrameReg() : GetTarget().GetStackReg();
2270 liveRegs &= ~GetEncoder()->GetAvailableScratchRegisters();
2271 liveVregs &= ~GetEncoder()->GetAvailableScratchFpRegisters();
2272 if (adjustRegs) {
2273 liveRegs &= GetRegfile()->GetCallerSavedRegMask();
2274 liveVregs &= GetRegfile()->GetCallerSavedVRegMask();
2275 } else {
2276 liveRegs &= GetCallerRegsMask(GetArch(), false);
2277 liveVregs &= GetCallerRegsMask(GetArch(), true);
2278 }
2279 if (GetFrameInfo()->GetPushCallers()) {
2280 GetEncoder()->PushRegisters(liveRegs, liveVregs);
2281 } else {
2282 GetEncoder()->SaveRegisters(liveRegs, false, GetFrameInfo()->GetCallersOffset(), base,
2283 GetCallerRegsMask(GetArch(), false));
2284 GetEncoder()->SaveRegisters(liveVregs, true, GetFrameInfo()->GetFpCallersOffset(), base,
2285 GetFrameInfo()->GetPositionedCallers() ? GetCallerRegsMask(GetArch(), true)
2286 : RegMask());
2287 }
2288 }
2289
LoadCallerRegisters(RegMask liveRegs,VRegMask liveVregs,bool adjustRegs)2290 void Codegen::LoadCallerRegisters(RegMask liveRegs, VRegMask liveVregs, bool adjustRegs)
2291 {
2292 SCOPED_DISASM_STR(this, "Restore caller saved regs");
2293 auto base = GetFrameInfo()->GetCallersRelativeFp() ? GetTarget().GetFrameReg() : GetTarget().GetStackReg();
2294 liveRegs &= ~GetEncoder()->GetAvailableScratchRegisters();
2295 liveVregs &= ~GetEncoder()->GetAvailableScratchFpRegisters();
2296 if (adjustRegs) {
2297 liveRegs &= GetRegfile()->GetCallerSavedRegMask();
2298 liveVregs &= GetRegfile()->GetCallerSavedVRegMask();
2299 } else {
2300 liveRegs &= GetCallerRegsMask(GetArch(), false);
2301 liveVregs &= GetCallerRegsMask(GetArch(), true);
2302 }
2303 if (GetFrameInfo()->GetPushCallers()) {
2304 GetEncoder()->PopRegisters(liveRegs, liveVregs);
2305 } else {
2306 GetEncoder()->LoadRegisters(liveRegs, false, GetFrameInfo()->GetCallersOffset(), base,
2307 GetCallerRegsMask(GetArch(), false));
2308 GetEncoder()->LoadRegisters(liveVregs, true, GetFrameInfo()->GetFpCallersOffset(), base,
2309 GetCallerRegsMask(GetArch(), true));
2310 }
2311 }
2312
RegisterKeepCallArgument(CallInst * callInst,Reg reg)2313 bool Codegen::RegisterKeepCallArgument(CallInst *callInst, Reg reg)
2314 {
2315 for (auto i = 0U; i < callInst->GetInputsCount(); i++) {
2316 auto location = callInst->GetLocation(i);
2317 if (location.IsRegister() && location.GetValue() == reg.GetId()) {
2318 return true;
2319 }
2320 }
2321 return false;
2322 }
2323
LoadMethod(Reg dst)2324 void Codegen::LoadMethod(Reg dst)
2325 {
2326 ASSERT((CFrameMethod::GetOffsetFromSpInBytes(GetFrameLayout()) -
2327 (GetFrameLayout().GetMethodOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::BYTES>())) ==
2328 0);
2329 auto spOffset = CFrameMethod::GetOffsetFromSpInBytes(GetFrameLayout());
2330 auto mem = MemRef(SpReg(), spOffset);
2331 GetEncoder()->EncodeLdr(dst, false, mem);
2332 }
2333
StoreFreeSlot(Reg src)2334 void Codegen::StoreFreeSlot(Reg src)
2335 {
2336 ASSERT(src.GetSize() <= (GetFrameLayout().GetSlotSize() << 3U));
2337 auto spOffset =
2338 GetFrameLayout().GetFreeSlotOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::BYTES>();
2339 auto mem = MemRef(SpReg(), spOffset);
2340 GetEncoder()->EncodeStr(src, mem);
2341 }
2342
LoadFreeSlot(Reg dst)2343 void Codegen::LoadFreeSlot(Reg dst)
2344 {
2345 ASSERT(dst.GetSize() <= (GetFrameLayout().GetSlotSize() << 3U));
2346 auto spOffset =
2347 GetFrameLayout().GetFreeSlotOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::BYTES>();
2348 auto mem = MemRef(SpReg(), spOffset);
2349 GetEncoder()->EncodeLdr(dst, false, mem);
2350 }
2351
CreateReturn(const Inst * inst)2352 void Codegen::CreateReturn(const Inst *inst)
2353 {
2354 if (GetGraph()->GetMethodProperties().GetLastReturn() == inst) {
2355 GetEncoder()->BindLabel(GetLabelExit());
2356 GenerateEpilogue();
2357 } else {
2358 GetEncoder()->EncodeJump(GetLabelExit());
2359 }
2360 }
2361
EncodeDynamicCast(Inst * inst,Reg dst,bool dstSigned,Reg src)2362 void Codegen::EncodeDynamicCast(Inst *inst, Reg dst, bool dstSigned, Reg src)
2363 {
2364 CHECK_EQ(src.GetSize(), BITS_PER_UINT64);
2365 CHECK_GE(dst.GetSize(), BITS_PER_UINT32);
2366
2367 bool isDst64 {dst.GetSize() == BITS_PER_UINT64};
2368 dst = dst.As(INT32_TYPE);
2369
2370 auto enc {GetEncoder()};
2371 if (g_options.IsCpuFeatureEnabled(CpuFeature::JSCVT)) {
2372 // no slow path intended
2373 enc->EncodeFastPathDynamicCast(dst, src, LabelHolder::INVALID_LABEL);
2374 } else {
2375 auto slowPath {CreateSlowPath<SlowPathJsCastDoubleToInt32>(inst)};
2376 slowPath->SetDstReg(dst);
2377 slowPath->SetSrcReg(src);
2378
2379 enc->EncodeFastPathDynamicCast(dst, src, slowPath->GetLabel());
2380 slowPath->BindBackLabel(enc);
2381 }
2382
2383 if (isDst64) {
2384 enc->EncodeCast(dst.As(INT64_TYPE), dstSigned, dst, dstSigned);
2385 }
2386 }
2387
2388 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
2389 #define UNARY_OPERATION(opc) \
2390 void EncodeVisitor::Visit##opc(GraphVisitor *visitor, Inst *inst) \
2391 { \
2392 EncodeVisitor *enc = static_cast<EncodeVisitor *>(visitor); \
2393 auto type = inst->GetType(); \
2394 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); \
2395 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type); \
2396 enc->GetEncoder()->Encode##opc(dst, src0); \
2397 }
2398
2399 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
2400 #define BINARY_OPERATION(opc) \
2401 void EncodeVisitor::Visit##opc(GraphVisitor *visitor, Inst *inst) \
2402 { \
2403 auto type = inst->GetType(); \
2404 EncodeVisitor *enc = static_cast<EncodeVisitor *>(visitor); \
2405 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); \
2406 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type); \
2407 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type); \
2408 enc->GetEncoder()->Encode##opc(dst, src0, src1); \
2409 }
2410
2411 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
2412 #define BINARY_SHIFTED_REGISTER_OPERATION(opc) \
2413 void EncodeVisitor::Visit##opc##SR(GraphVisitor *visitor, Inst *inst) \
2414 { \
2415 auto type = inst->GetType(); \
2416 EncodeVisitor *enc = static_cast<EncodeVisitor *>(visitor); \
2417 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); \
2418 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type); \
2419 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type); \
2420 auto immShiftInst = static_cast<BinaryShiftedRegisterOperation *>(inst); \
2421 auto immValue = static_cast<uint32_t>(immShiftInst->GetImm()) & (dst.GetSize() - 1); \
2422 auto shift = Shift(src1, immShiftInst->GetShiftType(), immValue); \
2423 enc->GetEncoder()->Encode##opc(dst, src0, shift); \
2424 }
2425
2426 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
2427 #define INST_DEF(OPCODE, TYPE) TYPE(OPCODE)
2428
2429 ENCODE_MATH_LIST(INST_DEF)
ENCODE_INST_WITH_SHIFTED_OPERAND(INST_DEF)2430 ENCODE_INST_WITH_SHIFTED_OPERAND(INST_DEF)
2431
2432 #undef UNARY_OPERATION
2433 #undef BINARY_OPERATION
2434 #undef BINARY_SHIFTED_REGISTER_OPERATION
2435 #undef INST_DEF
2436
2437 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
2438 #define BINARY_IMM_OPERATION(opc) \
2439 void EncodeVisitor::Visit##opc##I(GraphVisitor *visitor, Inst *inst) \
2440 { \
2441 auto binop = inst->CastTo##opc##I(); \
2442 EncodeVisitor *enc = static_cast<EncodeVisitor *>(visitor); \
2443 auto type = inst->GetType(); \
2444 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); \
2445 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type); \
2446 enc->GetEncoder()->Encode##opc(dst, src0, Imm(binop->GetImm())); \
2447 }
2448
2449 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
2450 #define BINARY_IMM_OPS(DEF) DEF(Add) DEF(Sub) DEF(Shl) DEF(AShr) DEF(And) DEF(Or) DEF(Xor)
2451
2452 BINARY_IMM_OPS(BINARY_IMM_OPERATION)
2453
2454 #undef BINARY_IMM_OPS
2455 #undef BINARY_IMM_OPERATION
2456
2457 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
2458 #define BINARY_SIGN_UNSIGN_OPERATION(opc) \
2459 void EncodeVisitor::Visit##opc(GraphVisitor *visitor, Inst *inst) \
2460 { \
2461 auto type = inst->GetType(); \
2462 EncodeVisitor *enc = static_cast<EncodeVisitor *>(visitor); \
2463 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); \
2464 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type); \
2465 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type); \
2466 auto arch = enc->GetCodegen()->GetArch(); \
2467 if (!Codegen::InstEncodedWithLibCall(inst, arch)) { \
2468 enc->GetEncoder()->Encode##opc(dst, IsTypeSigned(type), src0, src1); \
2469 return; \
2470 } \
2471 ASSERT(arch == Arch::AARCH32); \
2472 if (enc->cg_->GetGraph()->IsAotMode()) { \
2473 enc->GetEncoder()->SetFalseResult(); \
2474 return; \
2475 } \
2476 auto [liveRegs, liveVregs] = enc->GetCodegen()->GetLiveRegisters(inst); \
2477 enc->GetEncoder()->SetRegister(&liveRegs, &liveVregs, dst, false); \
2478 enc->GetCodegen()->SaveCallerRegisters(liveRegs, liveVregs, true); \
2479 enc->GetEncoder()->Encode##opc(dst, IsTypeSigned(type), src0, src1); \
2480 enc->GetCodegen()->LoadCallerRegisters(liveRegs, liveVregs, true); \
2481 }
2482
2483 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
2484 #define SIGN_UNSIGN_OPS(DEF) DEF(Div) DEF(Min) DEF(Max)
2485
2486 SIGN_UNSIGN_OPS(BINARY_SIGN_UNSIGN_OPERATION)
2487
2488 #undef SIGN_UNSIGN_OPS
2489 #undef BINARY_SIGN_UNSIGN_OPERATION
2490
2491 void EncodeVisitor::VisitMod(GraphVisitor *visitor, Inst *inst)
2492 {
2493 auto type = inst->GetType();
2494 auto *enc = static_cast<EncodeVisitor *>(visitor);
2495 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
2496 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
2497 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
2498 auto arch = enc->GetCodegen()->GetArch();
2499 if (!Codegen::InstEncodedWithLibCall(inst, arch)) {
2500 enc->GetEncoder()->EncodeMod(dst, IsTypeSigned(type), src0, src1);
2501 return;
2502 }
2503
2504 if (DataType::IsFloatType(type)) {
2505 RuntimeInterface::IntrinsicId entry =
2506 ((type == DataType::FLOAT32) ? RuntimeInterface::IntrinsicId::LIB_CALL_FMODF
2507 : RuntimeInterface::IntrinsicId::LIB_CALL_FMOD);
2508 auto [liveRegs, liveVregs] = enc->GetCodegen()->GetLiveRegisters(inst);
2509 enc->GetCodegen()->SaveCallerRegisters(liveRegs, liveVregs, true);
2510
2511 enc->GetCodegen()->FillCallParams(src0, src1);
2512 enc->GetCodegen()->CallIntrinsic(inst, entry);
2513
2514 auto retVal = enc->GetCodegen()->GetTarget().GetReturnFpReg();
2515 if (retVal.GetType().IsFloat()) {
2516 // ret_val is FLOAT64 for Arm64, AMD64 and ARM32 HardFP, but dst can be FLOAT32
2517 // so we convert ret_val to FLOAT32
2518 enc->GetEncoder()->EncodeMov(dst, Reg(retVal.GetId(), dst.GetType()));
2519 } else {
2520 // case for ARM32 SoftFP
2521 enc->GetEncoder()->EncodeMov(dst,
2522 Reg(retVal.GetId(), dst.GetSize() == WORD_SIZE ? INT32_TYPE : INT64_TYPE));
2523 }
2524
2525 enc->GetEncoder()->SetRegister(&liveRegs, &liveVregs, dst, false);
2526 enc->GetCodegen()->LoadCallerRegisters(liveRegs, liveVregs, true);
2527 return;
2528 }
2529
2530 ASSERT(arch == Arch::AARCH32);
2531 // Fix after supporting AOT mode for arm32
2532 if (enc->cg_->GetGraph()->IsAotMode()) {
2533 enc->GetEncoder()->SetFalseResult();
2534 return;
2535 }
2536 auto [liveRegs, liveVregs] = enc->GetCodegen()->GetLiveRegisters(inst);
2537 enc->GetEncoder()->SetRegister(&liveRegs, &liveVregs, dst, false);
2538 enc->GetCodegen()->SaveCallerRegisters(liveRegs, liveVregs, true);
2539 enc->GetEncoder()->EncodeMod(dst, IsTypeSigned(type), src0, src1);
2540 enc->GetCodegen()->LoadCallerRegisters(liveRegs, liveVregs, true);
2541 }
2542
VisitShrI(GraphVisitor * visitor,Inst * inst)2543 void EncodeVisitor::VisitShrI(GraphVisitor *visitor, Inst *inst)
2544 {
2545 auto binop = inst->CastToShrI();
2546 auto enc = static_cast<EncodeVisitor *>(visitor);
2547 auto type = inst->GetType();
2548 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
2549 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
2550 enc->GetEncoder()->EncodeShr(dst, src0, Imm(binop->GetImm()));
2551 }
2552
VisitMAdd(GraphVisitor * visitor,Inst * inst)2553 void EncodeVisitor::VisitMAdd(GraphVisitor *visitor, Inst *inst)
2554 {
2555 auto type = inst->GetType();
2556 auto enc = static_cast<EncodeVisitor *>(visitor);
2557 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
2558 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
2559 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
2560 constexpr int64_t IMM_2 = 2;
2561 auto src2 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(IMM_2), type);
2562 enc->GetEncoder()->EncodeMAdd(dst, src0, src1, src2);
2563 }
2564
VisitMSub(GraphVisitor * visitor,Inst * inst)2565 void EncodeVisitor::VisitMSub(GraphVisitor *visitor, Inst *inst)
2566 {
2567 auto type = inst->GetType();
2568 auto enc = static_cast<EncodeVisitor *>(visitor);
2569 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
2570 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
2571 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
2572 constexpr int64_t IMM_2 = 2;
2573 auto src2 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(IMM_2), type);
2574 enc->GetEncoder()->EncodeMSub(dst, src0, src1, src2);
2575 }
2576
VisitMNeg(GraphVisitor * visitor,Inst * inst)2577 void EncodeVisitor::VisitMNeg(GraphVisitor *visitor, Inst *inst)
2578 {
2579 auto type = inst->GetType();
2580 auto enc = static_cast<EncodeVisitor *>(visitor);
2581 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
2582 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
2583 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
2584 enc->GetEncoder()->EncodeMNeg(dst, src0, src1);
2585 }
2586
VisitOrNot(GraphVisitor * visitor,Inst * inst)2587 void EncodeVisitor::VisitOrNot(GraphVisitor *visitor, Inst *inst)
2588 {
2589 auto type = inst->GetType();
2590 auto enc = static_cast<EncodeVisitor *>(visitor);
2591 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
2592 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
2593 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
2594 enc->GetEncoder()->EncodeOrNot(dst, src0, src1);
2595 }
2596
VisitAndNot(GraphVisitor * visitor,Inst * inst)2597 void EncodeVisitor::VisitAndNot(GraphVisitor *visitor, Inst *inst)
2598 {
2599 auto type = inst->GetType();
2600 auto enc = static_cast<EncodeVisitor *>(visitor);
2601 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
2602 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
2603 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
2604 enc->GetEncoder()->EncodeAndNot(dst, src0, src1);
2605 }
2606
VisitXorNot(GraphVisitor * visitor,Inst * inst)2607 void EncodeVisitor::VisitXorNot(GraphVisitor *visitor, Inst *inst)
2608 {
2609 auto type = inst->GetType();
2610 auto enc = static_cast<EncodeVisitor *>(visitor);
2611 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
2612 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
2613 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
2614 enc->GetEncoder()->EncodeXorNot(dst, src0, src1);
2615 }
2616
VisitNegSR(GraphVisitor * visitor,Inst * inst)2617 void EncodeVisitor::VisitNegSR(GraphVisitor *visitor, Inst *inst)
2618 {
2619 auto type = inst->GetType();
2620 auto enc = static_cast<EncodeVisitor *>(visitor);
2621 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
2622 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
2623 auto immShiftInst = static_cast<UnaryShiftedRegisterOperation *>(inst);
2624 enc->GetEncoder()->EncodeNeg(dst, Shift(src, immShiftInst->GetShiftType(), immShiftInst->GetImm()));
2625 }
2626
VisitCast(GraphVisitor * visitor,Inst * inst)2627 void EncodeVisitor::VisitCast(GraphVisitor *visitor, Inst *inst)
2628 {
2629 auto enc = static_cast<EncodeVisitor *>(visitor);
2630 auto srcType = inst->GetInputType(0);
2631 auto dstType = inst->GetType();
2632
2633 ASSERT(dstType != DataType::ANY);
2634
2635 bool srcSigned = IsTypeSigned(srcType);
2636 bool dstSigned = IsTypeSigned(dstType);
2637
2638 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), srcType);
2639 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), dstType);
2640 if (dstType == DataType::BOOL) {
2641 enc->GetEncoder()->EncodeCastToBool(dst, src);
2642 return;
2643 }
2644
2645 if (inst->CastToCast()->IsDynamicCast()) {
2646 enc->GetCodegen()->EncodeDynamicCast(inst, dst, dstSigned, src);
2647 return;
2648 }
2649
2650 auto arch = enc->GetCodegen()->GetArch();
2651 if (!Codegen::InstEncodedWithLibCall(inst, arch)) {
2652 enc->GetEncoder()->EncodeCast(dst, dstSigned, src, srcSigned);
2653 return;
2654 }
2655 ASSERT(arch == Arch::AARCH32);
2656 // Fix after supporting AOT mode for arm32
2657 if (enc->cg_->GetGraph()->IsAotMode()) {
2658 enc->GetEncoder()->SetFalseResult();
2659 return;
2660 }
2661 auto [liveRegs, liveVregs] = enc->GetCodegen()->GetLiveRegisters(inst);
2662 enc->GetEncoder()->SetRegister(&liveRegs, &liveVregs, dst, false);
2663 enc->GetCodegen()->SaveCallerRegisters(liveRegs, liveVregs, true);
2664 enc->GetEncoder()->EncodeCast(dst, dstSigned, src, srcSigned);
2665 enc->GetCodegen()->LoadCallerRegisters(liveRegs, liveVregs, true);
2666 }
2667
VisitBitcast(GraphVisitor * visitor,Inst * inst)2668 void EncodeVisitor::VisitBitcast(GraphVisitor *visitor, Inst *inst)
2669 {
2670 auto enc = static_cast<EncodeVisitor *>(visitor);
2671 auto codegen = enc->GetCodegen();
2672 auto srcType = inst->GetInputType(0);
2673 auto dstType = inst->GetType();
2674 auto dst = codegen->ConvertRegister(inst->GetDstReg(), dstType);
2675 auto src = codegen->ConvertRegister(inst->GetSrcReg(0), srcType);
2676 enc->GetEncoder()->EncodeMov(dst, src);
2677 }
2678
VisitPhi(GraphVisitor * visitor,Inst * inst)2679 void EncodeVisitor::VisitPhi([[maybe_unused]] GraphVisitor *visitor, [[maybe_unused]] Inst *inst) {}
2680
VisitConstant(GraphVisitor * visitor,Inst * inst)2681 void EncodeVisitor::VisitConstant(GraphVisitor *visitor, Inst *inst)
2682 {
2683 auto *enc = static_cast<EncodeVisitor *>(visitor);
2684 if (inst->GetDstReg() == INVALID_REG) {
2685 return;
2686 }
2687 if (inst->GetDstReg() == enc->cg_->GetGraph()->GetZeroReg()) {
2688 ASSERT(IsZeroConstant(inst));
2689 ASSERT(enc->GetRegfile()->GetZeroReg() != INVALID_REGISTER);
2690 return;
2691 }
2692 auto *constInst = inst->CastToConstant();
2693 auto type = inst->GetType();
2694 if (enc->cg_->GetGraph()->IsDynamicMethod() && type == DataType::INT64) {
2695 type = DataType::INT32;
2696 }
2697
2698 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
2699 #ifndef NDEBUG
2700 switch (type) {
2701 case DataType::FLOAT32:
2702 enc->GetEncoder()->EncodeMov(dst, Imm(constInst->GetFloatValue()));
2703 break;
2704 case DataType::FLOAT64:
2705 enc->GetEncoder()->EncodeMov(dst, Imm(constInst->GetDoubleValue()));
2706 break;
2707 default:
2708 enc->GetEncoder()->EncodeMov(dst, Imm(constInst->GetRawValue()));
2709 }
2710 #else
2711 enc->GetEncoder()->EncodeMov(dst, Imm(constInst->GetRawValue()));
2712 #endif
2713 }
2714
VisitNullPtr(GraphVisitor * visitor,Inst * inst)2715 void EncodeVisitor::VisitNullPtr(GraphVisitor *visitor, Inst *inst)
2716 {
2717 auto *enc = static_cast<EncodeVisitor *>(visitor);
2718 if (inst->GetDstReg() == enc->cg_->GetGraph()->GetZeroReg()) {
2719 ASSERT_PRINT(enc->GetRegfile()->GetZeroReg() != INVALID_REGISTER,
2720 "NullPtr doesn't have correct destination register");
2721 return;
2722 }
2723 auto type = inst->GetType();
2724 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
2725 enc->GetEncoder()->EncodeMov(dst, Imm(0));
2726 }
2727
VisitLoadUndefined(GraphVisitor * visitor,Inst * inst)2728 void EncodeVisitor::VisitLoadUndefined(GraphVisitor *visitor, Inst *inst)
2729 {
2730 auto *enc = static_cast<EncodeVisitor *>(visitor);
2731 auto type = inst->GetType();
2732 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
2733 auto runtime = enc->GetCodegen()->GetGraph()->GetRuntime();
2734 auto graph = enc->GetCodegen()->GetGraph();
2735 if (graph->IsJitOrOsrMode()) {
2736 enc->GetEncoder()->EncodeMov(dst, Imm(runtime->GetUndefinedObject()));
2737 } else {
2738 auto ref = MemRef(enc->GetCodegen()->ThreadReg(), runtime->GetTlsUndefinedObjectOffset(graph->GetArch()));
2739 enc->GetEncoder()->EncodeLdr(dst, false, ref);
2740 }
2741 }
2742
2743 // Next visitors use calling convention
VisitCallIndirect(CallIndirectInst * inst)2744 void Codegen::VisitCallIndirect(CallIndirectInst *inst)
2745 {
2746 auto location = inst->GetLocation(0);
2747 ASSERT(location.IsFixedRegister() && location.IsRegisterValid());
2748 auto src = Reg(location.GetValue(), GetTarget().GetPtrRegType());
2749 auto dst = ConvertRegister(inst->GetDstReg(), inst->GetType());
2750
2751 GetEncoder()->MakeCall(src);
2752 if (inst->HasUsers()) {
2753 GetEncoder()->EncodeMov(dst, GetTarget().GetReturnReg(dst.GetType()));
2754 }
2755 }
2756
VisitCall(CallInst * inst)2757 void Codegen::VisitCall(CallInst *inst)
2758 {
2759 ASSERT(GetGraph()->GetRelocationHandler() != nullptr);
2760 ASSERT(!HasLiveCallerSavedRegs(inst));
2761
2762 RelocationInfo relocation;
2763 relocation.data = inst->GetCallMethodId();
2764 GetEncoder()->MakeCall(&relocation);
2765 GetGraph()->GetRelocationHandler()->AddRelocation(relocation);
2766
2767 if (inst->HasUsers()) {
2768 auto dstReg = ConvertRegister(inst->GetDstReg(), inst->GetType());
2769 ASSERT(dstReg.IsValid());
2770 GetEncoder()->EncodeMov(dstReg, GetTarget().GetReturnReg(dstReg.GetType()));
2771 }
2772 }
2773
VisitCallIndirect(GraphVisitor * visitor,Inst * inst)2774 void EncodeVisitor::VisitCallIndirect(GraphVisitor *visitor, Inst *inst)
2775 {
2776 static_cast<EncodeVisitor *>(visitor)->GetCodegen()->VisitCallIndirect(inst->CastToCallIndirect());
2777 }
2778
VisitCall(GraphVisitor * visitor,Inst * inst)2779 void EncodeVisitor::VisitCall(GraphVisitor *visitor, Inst *inst)
2780 {
2781 static_cast<EncodeVisitor *>(visitor)->GetCodegen()->VisitCall(inst->CastToCall());
2782 }
2783
VisitCompare(GraphVisitor * visitor,Inst * inst)2784 void EncodeVisitor::VisitCompare(GraphVisitor *visitor, Inst *inst)
2785 {
2786 auto *enc = static_cast<EncodeVisitor *>(visitor);
2787
2788 auto type = inst->CastToCompare()->GetOperandsType();
2789 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
2790 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
2791 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
2792 auto cc = enc->GetCodegen()->ConvertCc(inst->CastToCompare()->GetCc());
2793 if (IsTestCc(cc)) {
2794 enc->GetEncoder()->EncodeCompareTest(dst, src0, src1, cc);
2795 } else {
2796 enc->GetEncoder()->EncodeCompare(dst, src0, src1, cc);
2797 }
2798 }
2799
VisitCmp(GraphVisitor * visitor,Inst * inst)2800 void EncodeVisitor::VisitCmp(GraphVisitor *visitor, Inst *inst)
2801 {
2802 auto *enc = static_cast<EncodeVisitor *>(visitor);
2803
2804 auto cmpInst = inst->CastToCmp();
2805 auto type = cmpInst->GetOperandsType();
2806
2807 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
2808 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
2809 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
2810
2811 Condition cc;
2812 if (DataType::IsFloatType(type)) {
2813 // check whether MI is fully correct here
2814 cc = cmpInst->IsFcmpg() ? (Condition::MI) : (Condition::LT);
2815 } else if (IsTypeSigned(type)) {
2816 cc = Condition::LT;
2817 } else {
2818 cc = Condition::LO;
2819 }
2820 enc->GetEncoder()->EncodeCmp(dst, src0, src1, cc);
2821 }
2822
VisitReturnVoid(GraphVisitor * visitor,Inst * inst)2823 void EncodeVisitor::VisitReturnVoid(GraphVisitor *visitor, Inst *inst)
2824 {
2825 auto *enc = static_cast<EncodeVisitor *>(visitor);
2826 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
2827 enc->GetEncoder()->EncodeMemoryBarrier(memory_order::RELEASE);
2828 }
2829
2830 enc->GetCodegen()->CreateReturn(inst);
2831 }
2832
VisitReturn(GraphVisitor * visitor,Inst * inst)2833 void EncodeVisitor::VisitReturn(GraphVisitor *visitor, Inst *inst)
2834 {
2835 auto *enc = static_cast<EncodeVisitor *>(visitor);
2836 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), inst->GetType());
2837 enc->GetEncoder()->EncodeMov(enc->GetCodegen()->GetTarget().GetReturnReg(src.GetType()), src);
2838
2839 enc->GetCodegen()->CreateReturn(inst);
2840 }
2841
VisitReturnI(GraphVisitor * visitor,Inst * inst)2842 void EncodeVisitor::VisitReturnI(GraphVisitor *visitor, Inst *inst)
2843 {
2844 auto *enc = static_cast<EncodeVisitor *>(visitor);
2845 auto codegen = enc->GetCodegen();
2846 auto rzero = enc->GetRegfile()->GetZeroReg();
2847 int64_t immVal = inst->CastToReturnI()->GetImm();
2848 Imm imm = codegen->ConvertImmWithExtend(immVal, inst->GetType());
2849
2850 auto returnReg = codegen->GetTarget().GetReturnReg(codegen->ConvertDataType(inst->GetType(), codegen->GetArch()));
2851
2852 if (immVal == 0 && codegen->GetTarget().SupportZeroReg() && !DataType::IsFloatType(inst->GetType())) {
2853 enc->GetEncoder()->EncodeMov(returnReg, rzero);
2854 } else {
2855 enc->GetEncoder()->EncodeMov(returnReg, imm);
2856 }
2857
2858 enc->GetCodegen()->CreateReturn(inst);
2859 }
2860
2861 #if defined(EVENT_METHOD_EXIT_ENABLED) && EVENT_METHOD_EXIT_ENABLED != 0
GetCallInstFromReturnInlined(Inst * return_inlined)2862 static CallInst *GetCallInstFromReturnInlined(Inst *return_inlined)
2863 {
2864 auto ss = return_inlined->GetSaveState();
2865 for (auto &user : ss->GetUsers()) {
2866 auto inst = user.GetInst();
2867 if (inst->IsCall() && static_cast<CallInst *>(inst)->IsInlined()) {
2868 return static_cast<CallInst *>(inst);
2869 }
2870 }
2871 return nullptr;
2872 }
2873 #endif
2874
VisitReturnInlined(GraphVisitor * visitor,Inst * inst)2875 void EncodeVisitor::VisitReturnInlined(GraphVisitor *visitor, Inst *inst)
2876 {
2877 auto *enc = static_cast<EncodeVisitor *>(visitor);
2878 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
2879 enc->GetEncoder()->EncodeMemoryBarrier(memory_order::RELEASE);
2880 }
2881
2882 #if defined(EVENT_METHOD_EXIT_ENABLED) && EVENT_METHOD_EXIT_ENABLED != 0
2883 if (!enc->cg_->GetGraph()->IsAotMode()) {
2884 auto callInst = GetCallInstFromReturnInlined(inst->CastToReturnInlined());
2885 ASSERT(callInst != nullptr);
2886 static_cast<EncodeVisitor *>(visitor)->GetCodegen()->InsertTrace(
2887 {Imm(static_cast<size_t>(TraceId::METHOD_EXIT)), Imm(reinterpret_cast<size_t>(callInst->GetCallMethod())),
2888 Imm(static_cast<size_t>(events::MethodExitKind::INLINED))});
2889 }
2890 #endif
2891 }
2892
VisitLoadConstArray(GraphVisitor * visitor,Inst * inst)2893 void EncodeVisitor::VisitLoadConstArray(GraphVisitor *visitor, Inst *inst)
2894 {
2895 auto *enc = static_cast<EncodeVisitor *>(visitor);
2896 auto method = inst->CastToLoadConstArray()->GetMethod();
2897 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
2898 auto arrayType = inst->CastToLoadConstArray()->GetTypeId();
2899
2900 enc->GetCodegen()->CallRuntimeWithMethod(inst, method, EntrypointId::RESOLVE_LITERAL_ARRAY, dst,
2901 TypedImm(arrayType));
2902 }
2903
VisitFillConstArray(GraphVisitor * visitor,Inst * inst)2904 void EncodeVisitor::VisitFillConstArray(GraphVisitor *visitor, Inst *inst)
2905 {
2906 auto type = inst->GetType();
2907 ASSERT(type != DataType::REFERENCE);
2908 auto *enc = static_cast<EncodeVisitor *>(visitor);
2909 auto encoder = enc->GetEncoder();
2910 auto runtime = enc->cg_->GetGraph()->GetRuntime();
2911 auto arrayType = inst->CastToFillConstArray()->GetTypeId();
2912 auto arch = enc->cg_->GetGraph()->GetArch();
2913 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE);
2914 auto method = inst->CastToFillConstArray()->GetMethod();
2915 auto offset = runtime->GetArrayDataOffset(enc->GetCodegen()->GetArch());
2916 auto arraySize = inst->CastToFillConstArray()->GetImm() << DataType::ShiftByType(type, arch);
2917
2918 auto arrayReg = enc->GetCodegen()->ConvertInstTmpReg(inst, DataType::GetIntTypeForReference(arch));
2919 encoder->EncodeAdd(arrayReg, src, Imm(offset));
2920
2921 ASSERT(arraySize != 0);
2922
2923 if (enc->cg_->GetGraph()->IsAotMode()) {
2924 auto arrOffset = runtime->GetOffsetToConstArrayData(method, arrayType);
2925 auto pfOffset = runtime->GetPandaFileOffset(arch);
2926 ScopedTmpReg methodReg(encoder);
2927 enc->GetCodegen()->LoadMethod(methodReg);
2928 // load pointer to panda file
2929 encoder->EncodeLdr(methodReg, false, MemRef(methodReg, pfOffset));
2930 // load pointer to binary file
2931 encoder->EncodeLdr(methodReg, false, MemRef(methodReg, runtime->GetBinaryFileBaseOffset(enc->GetArch())));
2932 // Get pointer to array data
2933 encoder->EncodeAdd(methodReg, methodReg, Imm(arrOffset));
2934 // call memcpy
2935 RuntimeInterface::IntrinsicId entry = RuntimeInterface::IntrinsicId::LIB_CALL_MEM_COPY;
2936 auto [liveRegs, liveVregs] = enc->GetCodegen()->GetLiveRegisters(inst);
2937 enc->GetCodegen()->SaveCallerRegisters(liveRegs, liveVregs, true);
2938
2939 enc->GetCodegen()->FillCallParams(arrayReg, methodReg, TypedImm(arraySize));
2940 enc->GetCodegen()->CallIntrinsic(inst, entry);
2941 enc->GetCodegen()->LoadCallerRegisters(liveRegs, liveVregs, true);
2942 } else {
2943 auto data = runtime->GetPointerToConstArrayData(method, arrayType);
2944 // call memcpy
2945 RuntimeInterface::IntrinsicId entry = RuntimeInterface::IntrinsicId::LIB_CALL_MEM_COPY;
2946 auto [liveRegs, liveVregs] = enc->GetCodegen()->GetLiveRegisters(inst);
2947 enc->GetCodegen()->SaveCallerRegisters(liveRegs, liveVregs, true);
2948
2949 enc->GetCodegen()->FillCallParams(arrayReg, TypedImm(data), TypedImm(arraySize));
2950 enc->GetCodegen()->CallIntrinsic(inst, entry);
2951 enc->GetCodegen()->LoadCallerRegisters(liveRegs, liveVregs, true);
2952 }
2953 }
2954
GetEntryPointId(uint64_t elementSize,RuntimeInterface::EntrypointId & eid)2955 static void GetEntryPointId(uint64_t elementSize, RuntimeInterface::EntrypointId &eid)
2956 {
2957 switch (elementSize) {
2958 case sizeof(uint8_t):
2959 eid = RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB8;
2960 break;
2961 case sizeof(uint16_t):
2962 eid = RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB16;
2963 break;
2964 case sizeof(uint32_t):
2965 eid = RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB32;
2966 break;
2967 case sizeof(uint64_t):
2968 eid = RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB64;
2969 break;
2970 default:
2971 UNREACHABLE();
2972 }
2973 }
2974
VisitNewArray(Inst * inst)2975 void Codegen::VisitNewArray(Inst *inst)
2976 {
2977 auto method = inst->CastToNewArray()->GetMethod();
2978 auto dst = ConvertRegister(inst->GetDstReg(), inst->GetType());
2979 auto srcClass = ConvertRegister(inst->GetSrcReg(NewArrayInst::INDEX_CLASS), DataType::POINTER);
2980 auto srcSize = ConvertRegister(inst->GetSrcReg(NewArrayInst::INDEX_SIZE), DataType::Type::INT32);
2981 auto arrayType = inst->CastToNewArray()->GetTypeId();
2982 auto runtime = GetGraph()->GetRuntime();
2983 auto encoder = GetEncoder();
2984
2985 auto maxTlabSize = runtime->GetTLABMaxSize();
2986 // NOTE(msherstennikov): support NewArray fast path for arm32
2987 if (maxTlabSize == 0 || GetArch() == Arch::AARCH32) {
2988 CallRuntime(inst, EntrypointId::CREATE_ARRAY, dst, RegMask::GetZeroMask(), srcClass, srcSize);
2989 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
2990 encoder->EncodeMemoryBarrier(memory_order::RELEASE);
2991 }
2992 return;
2993 }
2994
2995 auto lenInst = inst->GetDataFlowInput(0);
2996 auto classArraySize = runtime->GetClassArraySize(GetArch());
2997 uint64_t arraySize = 0;
2998 uint64_t elementSize = runtime->GetArrayElementSize(method, arrayType);
2999 uint64_t alignment = runtime->GetTLABAlignment();
3000 if (lenInst->GetOpcode() == Opcode::Constant) {
3001 ASSERT(lenInst->GetType() == DataType::INT64);
3002 arraySize = lenInst->CastToConstant()->GetIntValue() * elementSize + classArraySize;
3003 arraySize = (arraySize & ~(alignment - 1U)) + ((arraySize % alignment) != 0U ? alignment : 0U);
3004 if (arraySize > maxTlabSize) {
3005 CallRuntime(inst, EntrypointId::CREATE_ARRAY, dst, RegMask::GetZeroMask(), srcClass, srcSize);
3006 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
3007 encoder->EncodeMemoryBarrier(memory_order::RELEASE);
3008 }
3009 return;
3010 }
3011 }
3012
3013 EntrypointId eid;
3014 GetEntryPointId(elementSize, eid);
3015 CallFastPath(inst, eid, dst, RegMask::GetZeroMask(), srcClass, srcSize);
3016 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
3017 encoder->EncodeMemoryBarrier(memory_order::RELEASE);
3018 }
3019 }
3020
VisitNewArray(GraphVisitor * visitor,Inst * inst)3021 void EncodeVisitor::VisitNewArray(GraphVisitor *visitor, Inst *inst)
3022 {
3023 auto *enc = static_cast<EncodeVisitor *>(visitor);
3024 return enc->GetCodegen()->VisitNewArray(inst);
3025 }
3026
VisitParameter(GraphVisitor * visitor,Inst * inst)3027 void EncodeVisitor::VisitParameter(GraphVisitor *visitor, Inst *inst)
3028 {
3029 /*
3030 Default register parameters pushed in ir_builder
3031 In regalloc filled spill/fill parameters part.
3032 */
3033 auto *enc = static_cast<EncodeVisitor *>(visitor);
3034 auto codegen = enc->GetCodegen();
3035 auto paramInst = inst->CastToParameter();
3036 auto sf = paramInst->GetLocationData();
3037 if (sf.GetSrc() == sf.GetDst()) {
3038 return;
3039 }
3040
3041 auto tmpSf = codegen->GetGraph()->CreateInstSpillFill();
3042 tmpSf->AddSpillFill(sf);
3043 SpillFillEncoder(codegen, tmpSf).EncodeSpillFill();
3044 }
3045
VisitStoreArray(GraphVisitor * visitor,Inst * inst)3046 void EncodeVisitor::VisitStoreArray(GraphVisitor *visitor, Inst *inst)
3047 {
3048 auto *enc = static_cast<EncodeVisitor *>(visitor);
3049 auto array = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE);
3050 auto index = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::INT32);
3051 constexpr int64_t IMM_2 = 2;
3052 auto storedValue = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(IMM_2), inst->GetType());
3053 int32_t offset = enc->cg_->GetGraph()->GetRuntime()->GetArrayDataOffset(enc->GetCodegen()->GetArch());
3054 int32_t scale = DataType::ShiftByType(inst->GetType(), enc->GetCodegen()->GetArch());
3055
3056 auto memRef = [enc, inst, array, index, offset, scale]() {
3057 if (inst->CastToStoreArray()->GetNeedBarrier()) {
3058 auto tmpOffset =
3059 enc->GetCodegen()->ConvertInstTmpReg(inst, DataType::GetIntTypeForReference(enc->GetArch()));
3060 enc->GetEncoder()->EncodeShl(tmpOffset, index, Imm(scale));
3061 enc->GetEncoder()->EncodeAdd(tmpOffset, tmpOffset, Imm(offset));
3062 return MemRef(array, tmpOffset, 0);
3063 }
3064
3065 auto tmp = enc->GetCodegen()->ConvertInstTmpReg(inst, DataType::REFERENCE);
3066 enc->GetEncoder()->EncodeAdd(tmp, array, Imm(offset));
3067 return MemRef(tmp, index, scale);
3068 };
3069
3070 auto mem = memRef();
3071 if (inst->CastToStoreArray()->GetNeedBarrier()) {
3072 enc->GetCodegen()->CreatePreWRB(inst, mem, MakeMask(array.GetId(), storedValue.GetId()));
3073 }
3074 auto prevOffset = enc->GetEncoder()->GetCursorOffset();
3075 enc->GetEncoder()->EncodeStr(storedValue, mem);
3076 enc->GetCodegen()->TryInsertImplicitNullCheck(inst, prevOffset);
3077 if (inst->CastToStoreArray()->GetNeedBarrier()) {
3078 enc->GetCodegen()->CreatePostWRB(inst, mem, storedValue, INVALID_REGISTER);
3079 }
3080 }
3081
VisitSpillFill(GraphVisitor * visitor,Inst * inst)3082 void EncodeVisitor::VisitSpillFill(GraphVisitor *visitor, Inst *inst)
3083 {
3084 auto codegen = static_cast<EncodeVisitor *>(visitor)->GetCodegen();
3085 SpillFillEncoder(codegen, inst).EncodeSpillFill();
3086 }
3087
VisitSaveState(GraphVisitor * visitor,Inst * inst)3088 void EncodeVisitor::VisitSaveState([[maybe_unused]] GraphVisitor *visitor, [[maybe_unused]] Inst *inst)
3089 {
3090 // Nothing to do, SaveState is processed in its users.
3091 }
3092
VisitSaveStateDeoptimize(GraphVisitor * visitor,Inst * inst)3093 void EncodeVisitor::VisitSaveStateDeoptimize([[maybe_unused]] GraphVisitor *visitor, [[maybe_unused]] Inst *inst)
3094 {
3095 // Nothing to do, SaveStateDeoptimize is processed in its users.
3096 }
3097
VisitSaveStateOsr(GraphVisitor * visitor,Inst * inst)3098 void EncodeVisitor::VisitSaveStateOsr(GraphVisitor *visitor, Inst *inst)
3099 {
3100 static_cast<EncodeVisitor *>(visitor)->GetCodegen()->CreateOsrEntry(inst->CastToSaveStateOsr());
3101 }
3102
VisitLoadArray(GraphVisitor * visitor,Inst * inst)3103 void EncodeVisitor::VisitLoadArray(GraphVisitor *visitor, Inst *inst)
3104 {
3105 auto instLoadArray = inst->CastToLoadArray();
3106 auto enc = static_cast<EncodeVisitor *>(visitor);
3107 auto runtime = enc->cg_->GetGraph()->GetRuntime();
3108 ASSERT(instLoadArray->IsArray() || !runtime->IsCompressedStringsEnabled());
3109 if (static_cast<LoadInst *>(inst)->GetNeedBarrier()) {
3110 // Consider inserting barriers for GC
3111 }
3112 auto type = inst->GetType();
3113 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // array
3114 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::INT32); // index
3115 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); // load value
3116 int32_t offset = instLoadArray->IsArray() ? runtime->GetArrayDataOffset(enc->GetCodegen()->GetArch())
3117 : runtime->GetStringDataOffset(enc->GetArch());
3118 auto encoder = enc->GetEncoder();
3119 auto arch = encoder->GetArch();
3120 int32_t shift = DataType::ShiftByType(type, arch);
3121 ScopedTmpReg scopedTmp(encoder, Codegen::ConvertDataType(DataType::GetIntTypeForReference(arch), arch));
3122 auto tmp = scopedTmp.GetReg();
3123 encoder->EncodeAdd(tmp, src0, Imm(offset));
3124 auto mem = MemRef(tmp, src1, shift);
3125 auto prevOffset = enc->GetEncoder()->GetCursorOffset();
3126 encoder->EncodeLdr(dst, IsTypeSigned(type), mem);
3127 enc->GetCodegen()->TryInsertImplicitNullCheck(inst, prevOffset);
3128 }
3129
VisitLoadCompressedStringChar(GraphVisitor * visitor,Inst * inst)3130 void EncodeVisitor::VisitLoadCompressedStringChar(GraphVisitor *visitor, Inst *inst)
3131 {
3132 auto *enc = static_cast<EncodeVisitor *>(visitor);
3133 auto runtime = enc->cg_->GetGraph()->GetRuntime();
3134 auto type = inst->GetType();
3135 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // array
3136 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::INT32); // index
3137 auto src2 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(2), DataType::INT32); // length
3138 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); // load value
3139 int32_t offset = runtime->GetStringDataOffset(enc->GetArch());
3140 auto encoder = enc->GetEncoder();
3141 auto arch = encoder->GetArch();
3142 int32_t shift = DataType::ShiftByType(type, arch);
3143 ScopedTmpReg scopedTmp(encoder, Codegen::ConvertDataType(DataType::GetIntTypeForReference(arch), arch));
3144 auto tmp = scopedTmp.GetReg();
3145
3146 ASSERT(encoder->CanEncodeCompressedStringCharAt());
3147 auto mask = runtime->GetStringCompressionMask();
3148 if (mask != 1) {
3149 UNREACHABLE(); // mask is hardcoded in JCL, but verify it just in case it's changed
3150 }
3151 enc->GetEncoder()->EncodeCompressedStringCharAt(dst, src0, src1, src2, tmp, offset, shift);
3152 }
3153
VisitLenArray(GraphVisitor * visitor,Inst * inst)3154 void EncodeVisitor::VisitLenArray(GraphVisitor *visitor, Inst *inst)
3155 {
3156 auto *enc = static_cast<EncodeVisitor *>(visitor);
3157 auto type = inst->GetType();
3158 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // array
3159 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); // len array
3160
3161 auto lenArrayInst = inst->CastToLenArray();
3162 auto runtime = enc->cg_->GetGraph()->GetRuntime();
3163 int64_t offset = lenArrayInst->IsArray() ? runtime->GetArrayLengthOffset(enc->GetCodegen()->GetArch())
3164 : runtime->GetStringLengthOffset(enc->GetArch());
3165 auto mem = MemRef(src0, offset);
3166
3167 auto prevOffset = enc->GetEncoder()->GetCursorOffset();
3168 enc->GetEncoder()->EncodeLdr(dst, IsTypeSigned(type), mem);
3169 enc->GetCodegen()->TryInsertImplicitNullCheck(inst, prevOffset);
3170 }
3171
VisitBuiltin(GraphVisitor * visitor,Inst * inst)3172 void EncodeVisitor::VisitBuiltin([[maybe_unused]] GraphVisitor *visitor, [[maybe_unused]] Inst *inst)
3173 {
3174 UNREACHABLE();
3175 }
3176
VisitNullCheck(GraphVisitor * visitor,Inst * inst)3177 void EncodeVisitor::VisitNullCheck(GraphVisitor *visitor, Inst *inst)
3178 {
3179 if (inst->CastToNullCheck()->IsImplicit()) {
3180 return;
3181 }
3182 auto *enc = static_cast<EncodeVisitor *>(visitor);
3183 enc->GetCodegen()->template CreateUnaryCheck<SlowPathImplicitNullCheck>(inst, EntrypointId::NULL_POINTER_EXCEPTION,
3184 DeoptimizeType::NULL_CHECK, Condition::EQ);
3185 }
3186
VisitBoundsCheck(GraphVisitor * visitor,Inst * inst)3187 void EncodeVisitor::VisitBoundsCheck(GraphVisitor *visitor, Inst *inst)
3188 {
3189 auto *enc = static_cast<EncodeVisitor *>(visitor);
3190 auto lenType = inst->GetInputType(0);
3191 auto len = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), lenType);
3192 auto indexType = inst->GetInputType(1);
3193 auto index = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), indexType);
3194 [[maybe_unused]] constexpr int64_t IMM_2 = 2;
3195 ASSERT(inst->GetInput(IMM_2).GetInst()->GetOpcode() == Opcode::SaveState ||
3196 inst->GetInput(IMM_2).GetInst()->GetOpcode() == Opcode::SaveStateDeoptimize);
3197
3198 EntrypointId entrypoint = inst->CastToBoundsCheck()->IsArray() ? EntrypointId::ARRAY_INDEX_OUT_OF_BOUNDS_EXCEPTION
3199 : EntrypointId::STRING_INDEX_OUT_OF_BOUNDS_EXCEPTION;
3200
3201 LabelHolder::LabelId label;
3202 if (inst->CanDeoptimize()) {
3203 DeoptimizeType type = DeoptimizeType::BOUNDS_CHECK_WITH_DEOPT;
3204 label = enc->GetCodegen()->CreateSlowPath<SlowPathDeoptimize>(inst, type)->GetLabel();
3205 } else {
3206 label = enc->GetCodegen()->CreateSlowPath<SlowPathCheck>(inst, entrypoint)->GetLabel();
3207 }
3208 enc->GetEncoder()->EncodeJump(label, index, len, Condition::HS);
3209 }
3210
VisitRefTypeCheck(GraphVisitor * visitor,Inst * inst)3211 void EncodeVisitor::VisitRefTypeCheck(GraphVisitor *visitor, Inst *inst)
3212 {
3213 auto *enc = static_cast<EncodeVisitor *>(visitor);
3214 auto encoder = enc->GetEncoder();
3215 auto arrayReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE);
3216 auto refReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::REFERENCE);
3217 [[maybe_unused]] constexpr int64_t IMM_2 = 2;
3218 ASSERT(inst->GetInput(IMM_2).GetInst()->GetOpcode() == Opcode::SaveState);
3219 auto runtime = enc->cg_->GetGraph()->GetRuntime();
3220
3221 auto slowPath =
3222 enc->GetCodegen()->CreateSlowPath<SlowPathRefCheck>(inst, EntrypointId::CHECK_STORE_ARRAY_REFERENCE);
3223
3224 slowPath->SetRegs(arrayReg, refReg);
3225 slowPath->CreateBackLabel(encoder);
3226
3227 // We don't check if stored object is nullptr
3228 encoder->EncodeJump(slowPath->GetBackLabel(), refReg, Condition::EQ);
3229
3230 ScopedTmpReg tmpReg(encoder, Codegen::ConvertDataType(DataType::REFERENCE, enc->GetCodegen()->GetArch()));
3231 ScopedTmpReg tmpReg1(encoder, Codegen::ConvertDataType(DataType::REFERENCE, enc->GetCodegen()->GetArch()));
3232
3233 // Get Class from array
3234 enc->GetCodegen()->LoadClassFromObject(tmpReg, arrayReg);
3235
3236 // Get element type Class from array class
3237 encoder->EncodeLdr(tmpReg, false, MemRef(tmpReg, runtime->GetClassComponentTypeOffset(enc->GetArch())));
3238
3239 // Get Class from stored object
3240 enc->GetCodegen()->LoadClassFromObject(tmpReg1, refReg);
3241
3242 // If the object's and array element's types match we do not check further
3243 encoder->EncodeJump(slowPath->GetBackLabel(), tmpReg, tmpReg1, Condition::EQ);
3244
3245 // If the array's element class is not Object (baseclass == null)
3246 // we call CheckStoreArrayReference, otherwise we fall through
3247 encoder->EncodeLdr(tmpReg1, false, MemRef(tmpReg, runtime->GetClassBaseOffset(enc->GetArch())));
3248 encoder->EncodeJump(slowPath->GetLabel(), tmpReg1, Condition::NE);
3249
3250 slowPath->BindBackLabel(encoder);
3251 }
3252
VisitZeroCheck(GraphVisitor * visitor,Inst * inst)3253 void EncodeVisitor::VisitZeroCheck(GraphVisitor *visitor, Inst *inst)
3254 {
3255 auto *enc = static_cast<EncodeVisitor *>(visitor);
3256 enc->GetCodegen()->template CreateUnaryCheck<SlowPathCheck>(inst, EntrypointId::ARITHMETIC_EXCEPTION,
3257 DeoptimizeType::ZERO_CHECK, Condition::EQ);
3258 }
3259
VisitNegativeCheck(GraphVisitor * visitor,Inst * inst)3260 void EncodeVisitor::VisitNegativeCheck(GraphVisitor *visitor, Inst *inst)
3261 {
3262 auto *enc = static_cast<EncodeVisitor *>(visitor);
3263 enc->GetCodegen()->template CreateUnaryCheck<SlowPathCheck>(inst, EntrypointId::NEGATIVE_ARRAY_SIZE_EXCEPTION,
3264 DeoptimizeType::NEGATIVE_CHECK, Condition::LT);
3265 }
3266
VisitNotPositiveCheck(GraphVisitor * visitor,Inst * inst)3267 void EncodeVisitor::VisitNotPositiveCheck(GraphVisitor *visitor, Inst *inst)
3268 {
3269 auto *enc = static_cast<EncodeVisitor *>(visitor);
3270 auto srcType = inst->GetInputType(0);
3271 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), srcType);
3272 ASSERT(inst->GetInput(1).GetInst()->GetOpcode() == Opcode::SaveState ||
3273 inst->GetInput(1).GetInst()->GetOpcode() == Opcode::SaveStateDeoptimize);
3274
3275 ASSERT(inst->CanDeoptimize());
3276 DeoptimizeType type = DeoptimizeType::NEGATIVE_CHECK;
3277 auto label = enc->GetCodegen()->CreateSlowPath<SlowPathDeoptimize>(inst, type)->GetLabel();
3278 enc->GetEncoder()->EncodeJump(label, src, Condition::LE);
3279 }
3280
VisitDeoptimizeIf(GraphVisitor * visitor,Inst * inst)3281 void EncodeVisitor::VisitDeoptimizeIf(GraphVisitor *visitor, Inst *inst)
3282 {
3283 auto *enc = static_cast<EncodeVisitor *>(visitor);
3284 auto srcType = inst->GetInput(0).GetInst()->GetType();
3285 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), srcType);
3286
3287 auto slowPath =
3288 enc->GetCodegen()->CreateSlowPath<SlowPathDeoptimize>(inst, inst->CastToDeoptimizeIf()->GetDeoptimizeType());
3289
3290 // create jump to slow path if src is true
3291 enc->GetEncoder()->EncodeJump(slowPath->GetLabel(), src, Condition::NE);
3292 }
3293
VisitDeoptimizeCompare(GraphVisitor * visitor,Inst * inst)3294 void EncodeVisitor::VisitDeoptimizeCompare(GraphVisitor *visitor, Inst *inst)
3295 {
3296 auto *enc = static_cast<EncodeVisitor *>(visitor);
3297 auto deopt = inst->CastToDeoptimizeCompare();
3298 ASSERT(deopt->GetOperandsType() != DataType::NO_TYPE);
3299 auto src0 = enc->GetCodegen()->ConvertRegister(deopt->GetSrcReg(0), deopt->GetOperandsType());
3300 auto src1 = enc->GetCodegen()->ConvertRegister(deopt->GetSrcReg(1), deopt->GetOperandsType());
3301 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathDeoptimize>(
3302 inst, inst->CastToDeoptimizeCompare()->GetDeoptimizeType());
3303 enc->GetEncoder()->EncodeJump(slowPath->GetLabel(), src0, src1, enc->GetCodegen()->ConvertCc(deopt->GetCc()));
3304 }
3305
VisitDeoptimizeCompareImm(GraphVisitor * visitor,Inst * inst)3306 void EncodeVisitor::VisitDeoptimizeCompareImm(GraphVisitor *visitor, Inst *inst)
3307 {
3308 auto *enc = static_cast<EncodeVisitor *>(visitor);
3309 auto encoder = enc->GetEncoder();
3310 auto deopt = inst->CastToDeoptimizeCompareImm();
3311 ASSERT(deopt->GetOperandsType() != DataType::NO_TYPE);
3312 auto cc = deopt->GetCc();
3313 auto src0 = enc->GetCodegen()->ConvertRegister(deopt->GetSrcReg(0), deopt->GetOperandsType());
3314 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathDeoptimize>(
3315 inst, inst->CastToDeoptimizeCompareImm()->GetDeoptimizeType());
3316
3317 if (deopt->GetImm() == 0) {
3318 Arch arch = enc->GetCodegen()->GetArch();
3319 DataType::Type type = deopt->GetInput(0).GetInst()->GetType();
3320 ASSERT(!IsFloatType(type));
3321 if (IsTypeSigned(type) && (cc == ConditionCode::CC_LT || cc == ConditionCode::CC_GE)) {
3322 auto signBit = GetTypeSize(type, arch) - 1;
3323 if (cc == ConditionCode::CC_LT) {
3324 // x < 0
3325 encoder->EncodeBitTestAndBranch(slowPath->GetLabel(), src0, signBit, true);
3326 return;
3327 }
3328 if (cc == ConditionCode::CC_GE) {
3329 // x >= 0
3330 encoder->EncodeBitTestAndBranch(slowPath->GetLabel(), src0, signBit, false);
3331 return;
3332 }
3333 }
3334 if (enc->GetCodegen()->GetTarget().SupportZeroReg()) {
3335 auto zreg = enc->GetRegfile()->GetZeroReg();
3336 encoder->EncodeJump(slowPath->GetLabel(), src0, zreg, enc->GetCodegen()->ConvertCc(cc));
3337 } else {
3338 encoder->EncodeJump(slowPath->GetLabel(), src0, Imm(0), enc->GetCodegen()->ConvertCc(cc));
3339 }
3340 return;
3341 }
3342 encoder->EncodeJump(slowPath->GetLabel(), src0, Imm(deopt->GetImm()), enc->GetCodegen()->ConvertCc(cc));
3343 }
3344
VisitLoadString(GraphVisitor * visitor,Inst * inst)3345 void EncodeVisitor::VisitLoadString(GraphVisitor *visitor, Inst *inst)
3346 {
3347 auto *enc = static_cast<EncodeVisitor *>(visitor);
3348 auto method = inst->CastToLoadString()->GetMethod();
3349 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
3350 auto stringType = inst->CastToLoadString()->GetTypeId();
3351 auto graph = enc->cg_->GetGraph();
3352 auto encoder = enc->GetEncoder();
3353 ASSERT(inst->IsRuntimeCall());
3354
3355 // Static constructor invoked only once, so there is no sense in replacing regular
3356 // ResolveString runtime call with optimized version that will only slow down constructor's execution.
3357 auto isCctor = graph->GetRuntime()->IsMethodStaticConstructor(method);
3358 ObjectPointerType stringPtr {0};
3359
3360 if (graph->IsAotMode() && g_options.IsCompilerAotLoadStringPlt() && !isCctor) {
3361 auto aotData = graph->GetAotData();
3362 intptr_t slotOffset = aotData->GetStringSlotOffset(encoder->GetCursorOffset(), stringType);
3363 ScopedTmpRegU64 addrReg(encoder);
3364 ScopedTmpRegU64 tmpDst(encoder);
3365 encoder->MakeLoadAotTableAddr(slotOffset, addrReg, tmpDst);
3366
3367 auto slowPath =
3368 enc->GetCodegen()->CreateSlowPath<SlowPathResolveStringAot>(inst, EntrypointId::RESOLVE_STRING_AOT);
3369 slowPath->SetDstReg(dst);
3370 slowPath->SetAddrReg(addrReg);
3371 slowPath->SetStringId(stringType);
3372 slowPath->SetMethod(method);
3373 encoder->EncodeJump(slowPath->GetLabel(), tmpDst, Imm(RuntimeInterface::RESOLVE_STRING_AOT_COUNTER_LIMIT),
3374 Condition::LT);
3375 encoder->EncodeMov(dst, Reg(tmpDst.GetReg().GetId(), dst.GetType()));
3376 slowPath->BindBackLabel(encoder);
3377 } else if (!graph->IsAotMode() && (stringPtr = graph->GetRuntime()->GetNonMovableString(method, stringType)) != 0) {
3378 encoder->EncodeMov(dst, Imm(stringPtr));
3379 EVENT_JIT_USE_RESOLVED_STRING(graph->GetRuntime()->GetMethodName(method), stringType);
3380 } else {
3381 enc->GetCodegen()->CallRuntimeWithMethod(inst, method, EntrypointId::RESOLVE_STRING, dst, TypedImm(stringType));
3382 }
3383 }
3384
VisitLoadObject(GraphVisitor * visitor,Inst * inst)3385 void EncodeVisitor::VisitLoadObject(GraphVisitor *visitor, Inst *inst)
3386 {
3387 auto *enc = static_cast<EncodeVisitor *>(visitor);
3388 auto loadObj = inst->CastToLoadObject();
3389 if (loadObj->GetNeedBarrier()) {
3390 // Consider inserting barriers for GC
3391 }
3392 auto type = inst->GetType();
3393 auto typeInput = inst->GetInput(0).GetInst()->GetType();
3394 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), typeInput); // obj
3395 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); // load value
3396
3397 auto graph = enc->cg_->GetGraph();
3398 auto field = loadObj->GetObjField();
3399 size_t offset = GetObjectOffset(graph, loadObj->GetObjectType(), field, loadObj->GetTypeId());
3400 auto mem = MemRef(src, offset);
3401 auto prevOffset = enc->GetEncoder()->GetCursorOffset();
3402 if (loadObj->GetVolatile()) {
3403 enc->GetEncoder()->EncodeLdrAcquire(dst, IsTypeSigned(type), mem);
3404 } else {
3405 enc->GetEncoder()->EncodeLdr(dst, IsTypeSigned(type), mem);
3406 }
3407 enc->GetCodegen()->TryInsertImplicitNullCheck(inst, prevOffset);
3408 }
3409
VisitResolveObjectField(GraphVisitor * visitor,Inst * inst)3410 void EncodeVisitor::VisitResolveObjectField(GraphVisitor *visitor, Inst *inst)
3411 {
3412 auto *enc = static_cast<EncodeVisitor *>(visitor);
3413 auto graph = enc->cg_->GetGraph();
3414 auto resolver = inst->CastToResolveObjectField();
3415 if (resolver->GetNeedBarrier()) {
3416 // Inserts barriers for GC
3417 }
3418 auto type = inst->GetType();
3419 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
3420 auto typeId = resolver->GetTypeId();
3421 auto method = resolver->GetMethod();
3422 if (graph->IsAotMode()) {
3423 enc->GetCodegen()->CallRuntimeWithMethod(inst, method, EntrypointId::GET_FIELD_OFFSET, dst, TypedImm(typeId));
3424 } else {
3425 auto skind = UnresolvedTypesInterface::SlotKind::FIELD;
3426 auto fieldOffsetAddr = graph->GetRuntime()->GetUnresolvedTypes()->GetTableSlot(method, typeId, skind);
3427 ScopedTmpReg tmpReg(enc->GetEncoder());
3428 // load field offset and if it's 0 then call runtime EntrypointId::GET_FIELD_OFFSET
3429 enc->GetEncoder()->EncodeMov(tmpReg, Imm(fieldOffsetAddr));
3430 enc->GetEncoder()->EncodeLdr(tmpReg, false, MemRef(tmpReg));
3431 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathUnresolved>(inst, EntrypointId::GET_FIELD_OFFSET);
3432 slowPath->SetUnresolvedType(method, typeId);
3433 slowPath->SetDstReg(tmpReg);
3434 slowPath->SetSlotAddr(fieldOffsetAddr);
3435 enc->GetEncoder()->EncodeJump(slowPath->GetLabel(), tmpReg, Condition::EQ);
3436 slowPath->BindBackLabel(enc->GetEncoder());
3437 enc->GetEncoder()->EncodeMov(dst, tmpReg);
3438 }
3439 }
3440
VisitLoadResolvedObjectField(GraphVisitor * visitor,Inst * inst)3441 void EncodeVisitor::VisitLoadResolvedObjectField([[maybe_unused]] GraphVisitor *visitor, Inst *inst)
3442 {
3443 auto *enc = static_cast<EncodeVisitor *>(visitor);
3444 auto load = inst->CastToLoadResolvedObjectField();
3445 if (load->GetNeedBarrier()) {
3446 // Inserts barriers for GC
3447 }
3448 auto type = inst->GetType();
3449 auto obj = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // obj
3450 auto ofs = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::UINT32); // field offset
3451 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); // load value
3452 ScopedTmpReg tmpReg(enc->GetEncoder());
3453 enc->GetEncoder()->EncodeAdd(tmpReg, obj, ofs);
3454 enc->GetEncoder()->EncodeLdrAcquire(dst, IsTypeSigned(type), MemRef(tmpReg));
3455 }
3456
VisitLoad(GraphVisitor * visitor,Inst * inst)3457 void EncodeVisitor::VisitLoad(GraphVisitor *visitor, Inst *inst)
3458 {
3459 auto *enc = static_cast<EncodeVisitor *>(visitor);
3460 auto loadByOffset = inst->CastToLoad();
3461
3462 auto type = inst->GetType();
3463 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0U), DataType::POINTER); // pointer
3464 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1U), DataType::UINT32); // offset
3465 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); // load value
3466
3467 auto mem = MemRef(src0, src1, loadByOffset->GetScale());
3468
3469 if (loadByOffset->GetVolatile()) {
3470 enc->GetEncoder()->EncodeLdrAcquire(dst, IsTypeSigned(type), mem);
3471 } else {
3472 enc->GetEncoder()->EncodeLdr(dst, IsTypeSigned(type), mem);
3473 }
3474 }
3475
VisitLoadI(GraphVisitor * visitor,Inst * inst)3476 void EncodeVisitor::VisitLoadI(GraphVisitor *visitor, Inst *inst)
3477 {
3478 auto *enc = static_cast<EncodeVisitor *>(visitor);
3479 auto loadByOffset = inst->CastToLoadI();
3480
3481 auto type = inst->GetType();
3482 auto base = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0U), DataType::POINTER); // pointer
3483 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); // load value
3484
3485 if (loadByOffset->GetVolatile()) {
3486 enc->GetEncoder()->EncodeLdrAcquire(dst, IsTypeSigned(type), MemRef(base, loadByOffset->GetImm()));
3487 } else {
3488 enc->GetEncoder()->EncodeLdr(dst, IsTypeSigned(type), MemRef(base, loadByOffset->GetImm()));
3489 }
3490 }
3491
VisitStoreI(GraphVisitor * visitor,Inst * inst)3492 void EncodeVisitor::VisitStoreI(GraphVisitor *visitor, Inst *inst)
3493 {
3494 auto *enc = static_cast<EncodeVisitor *>(visitor);
3495 auto storeInst = inst->CastToStoreI();
3496
3497 auto base = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0U), DataType::POINTER);
3498 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1U), inst->GetType());
3499
3500 auto mem = MemRef(base, storeInst->GetImm());
3501 if (inst->CastToStoreI()->GetNeedBarrier()) {
3502 enc->GetCodegen()->CreatePreWRB(inst, mem, MakeMask(base.GetId(), src.GetId()));
3503 }
3504
3505 if (storeInst->GetVolatile()) {
3506 enc->GetEncoder()->EncodeStrRelease(src, mem);
3507 } else {
3508 enc->GetEncoder()->EncodeStr(src, mem);
3509 }
3510
3511 if (inst->CastToStoreI()->GetNeedBarrier()) {
3512 enc->GetCodegen()->CreatePostWRB(inst, mem, src, INVALID_REGISTER);
3513 }
3514 }
3515
VisitStoreObject(GraphVisitor * visitor,Inst * inst)3516 void EncodeVisitor::VisitStoreObject(GraphVisitor *visitor, Inst *inst)
3517 {
3518 auto *enc = static_cast<EncodeVisitor *>(visitor);
3519 auto storeObj = inst->CastToStoreObject();
3520 auto codegen = enc->GetCodegen();
3521 auto src0 = codegen->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // obj
3522 auto src1 = codegen->ConvertRegister(inst->GetSrcReg(1), inst->GetType()); // store value
3523
3524 auto graph = enc->cg_->GetGraph();
3525 auto field = storeObj->GetObjField();
3526 size_t offset = GetObjectOffset(graph, storeObj->GetObjectType(), field, storeObj->GetTypeId());
3527 if (!enc->GetCodegen()->OffsetFitReferenceTypeSize(offset)) {
3528 // such code should not be executed
3529 enc->GetEncoder()->EncodeAbort();
3530 return;
3531 }
3532 auto mem = MemRef(src0, offset);
3533 auto encoder = enc->GetEncoder();
3534 if (inst->CastToStoreObject()->GetNeedBarrier()) {
3535 if (storeObj->GetObjectType() == ObjectType::MEM_DYN_CLASS) {
3536 codegen->CreatePreWRB<true>(inst, mem, MakeMask(src0.GetId(), src1.GetId()));
3537 } else {
3538 codegen->CreatePreWRB(inst, mem, MakeMask(src0.GetId(), src1.GetId()));
3539 }
3540 }
3541 auto prevOffset = encoder->GetCursorOffset();
3542 if (storeObj->GetVolatile()) {
3543 encoder->EncodeStrRelease(src1, mem);
3544 } else {
3545 encoder->EncodeStr(src1, mem);
3546 }
3547 codegen->TryInsertImplicitNullCheck(inst, prevOffset);
3548 if (inst->CastToStoreObject()->GetNeedBarrier()) {
3549 ScopedTmpRegLazy tmp(encoder);
3550 if (storeObj->GetObjectType() == ObjectType::MEM_DYN_CLASS) {
3551 tmp.AcquireIfInvalid();
3552 encoder->EncodeLdr(tmp, false, MemRef(src1, graph->GetRuntime()->GetManagedClassOffset(graph->GetArch())));
3553 src1 = tmp;
3554 }
3555 codegen->CreatePostWRB(inst, mem, src1, INVALID_REGISTER);
3556 }
3557 }
3558
VisitStoreResolvedObjectField(GraphVisitor * visitor,Inst * inst)3559 void EncodeVisitor::VisitStoreResolvedObjectField(GraphVisitor *visitor, Inst *inst)
3560 {
3561 auto *enc = static_cast<EncodeVisitor *>(visitor);
3562 auto store = inst->CastToStoreResolvedObjectField();
3563 auto obj = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // object
3564 auto val = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), inst->GetType()); // store value
3565 auto ofs = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(2), DataType::UINT32); // field offset
3566 auto mem = MemRef(obj, ofs, 0);
3567 if (store->GetNeedBarrier()) {
3568 enc->GetCodegen()->CreatePreWRB(inst, mem);
3569 }
3570 // Unknown store, assume it can be volatile
3571 enc->GetEncoder()->EncodeStrRelease(val, mem);
3572 if (store->GetNeedBarrier()) {
3573 enc->GetCodegen()->CreatePostWRB(inst, mem, val, INVALID_REGISTER);
3574 }
3575 }
3576
VisitStore(GraphVisitor * visitor,Inst * inst)3577 void EncodeVisitor::VisitStore(GraphVisitor *visitor, Inst *inst)
3578 {
3579 auto *enc = static_cast<EncodeVisitor *>(visitor);
3580 auto storeByOffset = inst->CastToStore();
3581 auto type = inst->GetType();
3582 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0U), DataType::POINTER); // pointer
3583 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1U), DataType::UINT32); // offset
3584 auto src2 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(2U), type); // store value
3585
3586 auto mem = MemRef(src0, src1, storeByOffset->GetScale());
3587 if (inst->CastToStore()->GetNeedBarrier()) {
3588 enc->GetCodegen()->CreatePreWRB(inst, mem, MakeMask(src0.GetId(), src2.GetId()));
3589 }
3590
3591 if (storeByOffset->GetVolatile()) {
3592 enc->GetEncoder()->EncodeStrRelease(src2, mem);
3593 } else {
3594 enc->GetEncoder()->EncodeStr(src2, mem);
3595 }
3596
3597 if (inst->CastToStore()->GetNeedBarrier()) {
3598 enc->GetCodegen()->CreatePostWRB(inst, mem, src2, INVALID_REGISTER);
3599 }
3600 }
3601
VisitInitClass(GraphVisitor * visitor,Inst * inst)3602 void EncodeVisitor::VisitInitClass(GraphVisitor *visitor, Inst *inst)
3603 {
3604 auto *enc = static_cast<EncodeVisitor *>(visitor);
3605 auto graph = enc->cg_->GetGraph();
3606 auto runtime = graph->GetRuntime();
3607 auto classId = inst->CastToInitClass()->GetTypeId();
3608 auto encoder = enc->GetEncoder();
3609 ASSERT(inst->IsRuntimeCall());
3610
3611 if (graph->IsAotMode()) {
3612 ScopedTmpReg tmpReg(encoder);
3613 ScopedTmpReg classReg(encoder);
3614
3615 auto aotData = graph->GetAotData();
3616 intptr_t offset = aotData->GetClassSlotOffset(encoder->GetCursorOffset(), classId, true);
3617 encoder->MakeLoadAotTableAddr(offset, tmpReg, classReg);
3618
3619 auto label = encoder->CreateLabel();
3620 encoder->EncodeJump(label, classReg, Condition::NE);
3621
3622 // PLT Class Init Resolver has special calling convention:
3623 // First encoder temporary (tmp_reg) works as parameter and return value (which is unnecessary here)
3624 CHECK_EQ(tmpReg.GetReg().GetId(), encoder->GetTarget().GetTempRegsMask().GetMinRegister());
3625 enc->GetCodegen()->CreateJumpToClassResolverPltShared(inst, tmpReg.GetReg(), EntrypointId::CLASS_INIT_RESOLVER);
3626 encoder->BindLabel(label);
3627 } else { // JIT mode
3628 auto klass = reinterpret_cast<uintptr_t>(inst->CastToInitClass()->GetClass());
3629 ASSERT(klass != 0);
3630 if (!runtime->IsClassInitialized(klass)) {
3631 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathEntrypoint>(inst, EntrypointId::INITIALIZE_CLASS);
3632
3633 auto stateOffset = runtime->GetClassStateOffset(enc->GetArch());
3634 int64_t initValue = runtime->GetClassInitializedValue();
3635 ScopedTmpReg tmpReg(encoder);
3636 encoder->EncodeMov(tmpReg, Imm(klass + stateOffset));
3637 auto tmpI8 = enc->GetCodegen()->ConvertRegister(tmpReg.GetReg().GetId(), DataType::INT8);
3638 encoder->EncodeLdr(tmpI8, false, MemRef(tmpReg));
3639
3640 encoder->EncodeJump(slowPath->GetLabel(), tmpI8, Imm(initValue), Condition::NE);
3641
3642 slowPath->BindBackLabel(encoder);
3643 }
3644 }
3645 }
3646
VisitLoadClass(GraphVisitor * visitor,Inst * inst)3647 void EncodeVisitor::VisitLoadClass(GraphVisitor *visitor, Inst *inst)
3648 {
3649 auto *enc = static_cast<EncodeVisitor *>(visitor);
3650 auto encoder = enc->GetEncoder();
3651 auto loadClass = inst->CastToLoadClass();
3652 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
3653 auto graph = enc->cg_->GetGraph();
3654 auto typeId = loadClass->GetTypeId();
3655 ASSERT(inst->IsRuntimeCall());
3656
3657 if (graph->IsAotMode()) {
3658 auto methodClassId = graph->GetRuntime()->GetClassIdForMethod(graph->GetMethod());
3659 if (methodClassId == typeId) {
3660 auto dstPtr = dst.As(Codegen::ConvertDataType(DataType::POINTER, graph->GetArch()));
3661 enc->GetCodegen()->LoadMethod(dstPtr);
3662 auto mem = MemRef(dstPtr, graph->GetRuntime()->GetClassOffset(graph->GetArch()));
3663 encoder->EncodeLdr(dst.As(Codegen::ConvertDataType(DataType::REFERENCE, graph->GetArch())), false, mem);
3664 return;
3665 }
3666 ScopedTmpReg tmpReg(encoder);
3667 enc->GetCodegen()->CreateLoadClassFromPLT(inst, tmpReg, dst, typeId);
3668 } else { // JIT mode
3669 auto klass = loadClass->GetClass();
3670 if (klass == nullptr) {
3671 FillLoadClassUnresolved(visitor, inst);
3672 } else {
3673 encoder->EncodeMov(dst, Imm(reinterpret_cast<uintptr_t>(klass)));
3674 }
3675 }
3676 }
3677
FillLoadClassUnresolved(GraphVisitor * visitor,Inst * inst)3678 void EncodeVisitor::FillLoadClassUnresolved(GraphVisitor *visitor, Inst *inst)
3679 {
3680 auto *enc = static_cast<EncodeVisitor *>(visitor);
3681 auto encoder = enc->GetEncoder();
3682 auto loadClass = inst->CastToLoadClass();
3683 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
3684 auto graph = enc->cg_->GetGraph();
3685 auto typeId = loadClass->GetTypeId();
3686 auto method = loadClass->GetMethod();
3687 auto utypes = graph->GetRuntime()->GetUnresolvedTypes();
3688 auto klassAddr = utypes->GetTableSlot(method, typeId, UnresolvedTypesInterface::SlotKind::CLASS);
3689 Reg dstPtr(dst.GetId(), enc->GetCodegen()->GetPtrRegType());
3690 encoder->EncodeMov(dstPtr, Imm(klassAddr));
3691 encoder->EncodeLdr(dst, false, MemRef(dstPtr));
3692 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathUnresolved>(inst, EntrypointId::RESOLVE_CLASS);
3693 slowPath->SetUnresolvedType(method, typeId);
3694 slowPath->SetDstReg(dst);
3695 slowPath->SetSlotAddr(klassAddr);
3696
3697 encoder->EncodeJump(slowPath->GetLabel(), dst, Condition::EQ);
3698 slowPath->BindBackLabel(encoder);
3699 }
3700
VisitGetGlobalVarAddress(GraphVisitor * visitor,Inst * inst)3701 void EncodeVisitor::VisitGetGlobalVarAddress(GraphVisitor *visitor, Inst *inst)
3702 {
3703 auto *enc = static_cast<EncodeVisitor *>(visitor);
3704 auto graph = enc->cg_->GetGraph();
3705 auto runtime = graph->GetRuntime();
3706 auto id = inst->CastToGetGlobalVarAddress()->GetTypeId();
3707 auto encoder = enc->GetEncoder();
3708 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType()); // load value
3709 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::ANY);
3710 auto entrypointId = runtime->GetGlobalVarEntrypointId();
3711 ASSERT(inst->IsRuntimeCall());
3712 if (graph->IsAotMode()) {
3713 ScopedTmpReg addr(encoder);
3714 auto aotData = graph->GetAotData();
3715 Reg constPool = src;
3716 ScopedTmpRegLazy tmpReg(encoder);
3717 if (dst.GetId() == src.GetId()) {
3718 tmpReg.AcquireIfInvalid();
3719 constPool = tmpReg.GetReg().As(src.GetType());
3720 encoder->EncodeMov(constPool, src);
3721 }
3722 intptr_t offset = aotData->GetCommonSlotOffset(encoder->GetCursorOffset(), id);
3723 encoder->MakeLoadAotTableAddr(offset, addr, dst);
3724
3725 auto label = encoder->CreateLabel();
3726 encoder->EncodeJump(label, dst, Condition::NE);
3727
3728 enc->GetCodegen()->CallRuntime(inst, entrypointId, dst, RegMask::GetZeroMask(), constPool, TypedImm(id));
3729 encoder->EncodeStr(dst, MemRef(addr));
3730 encoder->BindLabel(label);
3731 } else {
3732 auto address = inst->CastToGetGlobalVarAddress()->GetAddress();
3733 if (address == 0) {
3734 address = runtime->GetGlobalVarAddress(graph->GetMethod(), id);
3735 }
3736 if (address == 0) {
3737 enc->GetCodegen()->CallRuntime(inst, entrypointId, dst, RegMask::GetZeroMask(), src, TypedImm(id));
3738 } else {
3739 encoder->EncodeMov(dst, Imm(address));
3740 }
3741 }
3742 }
3743
VisitLoadRuntimeClass(GraphVisitor * visitor,Inst * inst)3744 void EncodeVisitor::VisitLoadRuntimeClass(GraphVisitor *visitor, Inst *inst)
3745 {
3746 auto *enc = static_cast<EncodeVisitor *>(visitor);
3747 auto codegen = enc->GetCodegen();
3748 auto dst = codegen->ConvertRegister(inst->GetDstReg(), inst->GetType()); // load value
3749 size_t classAddrOffset = codegen->GetGraph()->GetRuntime()->GetTlsPromiseClassPointerOffset(codegen->GetArch());
3750
3751 auto mem = MemRef(codegen->ThreadReg(), classAddrOffset);
3752 enc->GetEncoder()->EncodeLdr(dst, false, mem);
3753 }
3754
VisitLoadAndInitClass(GraphVisitor * visitor,Inst * inst)3755 void EncodeVisitor::VisitLoadAndInitClass(GraphVisitor *visitor, Inst *inst)
3756 {
3757 auto *enc = static_cast<EncodeVisitor *>(visitor);
3758 auto graph = enc->cg_->GetGraph();
3759 auto runtime = graph->GetRuntime();
3760 auto classId = inst->CastToLoadAndInitClass()->GetTypeId();
3761 auto encoder = enc->GetEncoder();
3762 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType()); // load value
3763 ASSERT(inst->IsRuntimeCall());
3764
3765 if (graph->IsAotMode()) {
3766 auto methodClassId = runtime->GetClassIdForMethod(graph->GetMethod());
3767 if (methodClassId == classId) {
3768 auto dstPtr = dst.As(Codegen::ConvertDataType(DataType::POINTER, graph->GetArch()));
3769 enc->GetCodegen()->LoadMethod(dstPtr);
3770 auto mem = MemRef(dstPtr, graph->GetRuntime()->GetClassOffset(graph->GetArch()));
3771 encoder->EncodeLdr(dst.As(Codegen::ConvertDataType(DataType::REFERENCE, graph->GetArch())), false, mem);
3772 return;
3773 }
3774
3775 ScopedTmpReg tmpReg(encoder);
3776
3777 auto aotData = graph->GetAotData();
3778 intptr_t offset = aotData->GetClassSlotOffset(encoder->GetCursorOffset(), classId, true);
3779 encoder->MakeLoadAotTableAddr(offset, tmpReg, dst);
3780
3781 auto label = encoder->CreateLabel();
3782 encoder->EncodeJump(label, dst, Condition::NE);
3783
3784 // PLT Class Init Resolver has special calling convention:
3785 // First encoder temporary (tmp_reg) works as parameter and return value
3786 CHECK_EQ(tmpReg.GetReg().GetId(), encoder->GetTarget().GetTempRegsMask().GetMinRegister());
3787 enc->GetCodegen()->CreateJumpToClassResolverPltShared(inst, tmpReg.GetReg(), EntrypointId::CLASS_INIT_RESOLVER);
3788
3789 encoder->EncodeMov(dst, tmpReg);
3790 encoder->BindLabel(label);
3791 } else { // JIT mode
3792 auto klass = reinterpret_cast<uintptr_t>(inst->CastToLoadAndInitClass()->GetClass());
3793 encoder->EncodeMov(dst, Imm(klass));
3794
3795 if (runtime->IsClassInitialized(klass)) {
3796 return;
3797 }
3798
3799 auto methodClass = runtime->GetClass(graph->GetMethod());
3800 if (methodClass == inst->CastToLoadAndInitClass()->GetClass()) {
3801 return;
3802 }
3803
3804 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathEntrypoint>(inst, EntrypointId::INITIALIZE_CLASS);
3805
3806 auto stateOffset = runtime->GetClassStateOffset(enc->GetArch());
3807 int64_t initValue = runtime->GetClassInitializedValue();
3808
3809 ScopedTmpReg stateReg(encoder, INT8_TYPE);
3810 encoder->EncodeLdr(stateReg, false, MemRef(dst, stateOffset));
3811
3812 encoder->EncodeJump(slowPath->GetLabel(), stateReg, Imm(initValue), Condition::NE);
3813
3814 slowPath->BindBackLabel(encoder);
3815 }
3816 }
3817
VisitUnresolvedLoadAndInitClass(GraphVisitor * visitor,Inst * inst)3818 void EncodeVisitor::VisitUnresolvedLoadAndInitClass(GraphVisitor *visitor, Inst *inst)
3819 {
3820 auto *enc = static_cast<EncodeVisitor *>(visitor);
3821 auto graph = enc->cg_->GetGraph();
3822 auto classId = inst->CastToUnresolvedLoadAndInitClass()->GetTypeId();
3823 auto encoder = enc->GetEncoder();
3824 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType()); // load value
3825 ASSERT(inst->IsRuntimeCall());
3826
3827 if (graph->IsAotMode()) {
3828 ScopedTmpReg tmpReg(encoder);
3829
3830 auto aotData = graph->GetAotData();
3831 intptr_t offset = aotData->GetClassSlotOffset(encoder->GetCursorOffset(), classId, true);
3832 encoder->MakeLoadAotTableAddr(offset, tmpReg, dst);
3833
3834 auto label = encoder->CreateLabel();
3835 encoder->EncodeJump(label, dst, Condition::NE);
3836
3837 // PLT Class Init Resolver has special calling convention:
3838 // First encoder temporary (tmp_reg) works as parameter and return value
3839 CHECK_EQ(tmpReg.GetReg().GetId(), encoder->GetTarget().GetTempRegsMask().GetMinRegister());
3840 enc->GetCodegen()->CreateJumpToClassResolverPltShared(inst, tmpReg.GetReg(), EntrypointId::CLASS_INIT_RESOLVER);
3841
3842 encoder->EncodeMov(dst, tmpReg);
3843 encoder->BindLabel(label);
3844 } else { // JIT mode
3845 auto method = inst->CastToUnresolvedLoadAndInitClass()->GetMethod();
3846 auto utypes = graph->GetRuntime()->GetUnresolvedTypes();
3847 auto klassAddr = utypes->GetTableSlot(method, classId, UnresolvedTypesInterface::SlotKind::CLASS);
3848 Reg dstPtr(dst.GetId(), enc->GetCodegen()->GetPtrRegType());
3849 encoder->EncodeMov(dstPtr, Imm(klassAddr));
3850 encoder->EncodeLdr(dst, false, MemRef(dstPtr));
3851
3852 auto slowPath =
3853 enc->GetCodegen()->CreateSlowPath<SlowPathUnresolved>(inst, EntrypointId::INITIALIZE_CLASS_BY_ID);
3854 slowPath->SetUnresolvedType(method, classId);
3855 slowPath->SetDstReg(dst);
3856 slowPath->SetSlotAddr(klassAddr);
3857
3858 encoder->EncodeJump(slowPath->GetLabel(), dst, Condition::EQ);
3859 slowPath->BindBackLabel(encoder);
3860 }
3861 }
3862
VisitLoadStatic(GraphVisitor * visitor,Inst * inst)3863 void EncodeVisitor::VisitLoadStatic(GraphVisitor *visitor, Inst *inst)
3864 {
3865 auto *enc = static_cast<EncodeVisitor *>(visitor);
3866 auto loadStatic = inst->CastToLoadStatic();
3867 if (loadStatic->GetNeedBarrier()) {
3868 // Consider inserting barriers for GC
3869 }
3870 auto type = inst->GetType();
3871 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); // load value
3872 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // class
3873
3874 auto graph = enc->cg_->GetGraph();
3875 auto field = loadStatic->GetObjField();
3876 auto offset = graph->GetRuntime()->GetFieldOffset(field);
3877 auto mem = MemRef(src0, offset);
3878 if (loadStatic->GetVolatile()) {
3879 enc->GetEncoder()->EncodeLdrAcquire(dst, IsTypeSigned(type), mem);
3880 } else {
3881 enc->GetEncoder()->EncodeLdr(dst, IsTypeSigned(type), mem);
3882 }
3883 }
3884
VisitResolveObjectFieldStatic(GraphVisitor * visitor,Inst * inst)3885 void EncodeVisitor::VisitResolveObjectFieldStatic(GraphVisitor *visitor, Inst *inst)
3886 {
3887 auto *enc = static_cast<EncodeVisitor *>(visitor);
3888 auto graph = enc->cg_->GetGraph();
3889 auto resolver = inst->CastToResolveObjectFieldStatic();
3890 ASSERT(resolver->GetType() == DataType::REFERENCE);
3891 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
3892 auto typeId = resolver->GetTypeId();
3893 auto method = resolver->GetMethod();
3894 EntrypointId entrypoint = EntrypointId::GET_UNKNOWN_STATIC_FIELD_MEMORY_ADDRESS; // REFERENCE
3895 UnresolvedTypesInterface::SlotKind slotKind = UnresolvedTypesInterface::SlotKind::FIELD;
3896
3897 if (graph->IsAotMode()) {
3898 enc->GetCodegen()->CallRuntimeWithMethod(inst, method, entrypoint, dst, TypedImm(typeId), TypedImm(0));
3899 } else {
3900 ScopedTmpReg tmpReg(enc->GetEncoder());
3901 auto fieldAddr = graph->GetRuntime()->GetUnresolvedTypes()->GetTableSlot(method, typeId, slotKind);
3902 enc->GetEncoder()->EncodeMov(tmpReg, Imm(fieldAddr));
3903 enc->GetEncoder()->EncodeLdr(tmpReg, false, MemRef(tmpReg));
3904 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathUnresolved>(inst, entrypoint);
3905 slowPath->SetUnresolvedType(method, typeId);
3906 slowPath->SetDstReg(tmpReg);
3907 slowPath->SetSlotAddr(fieldAddr);
3908 enc->GetEncoder()->EncodeJump(slowPath->GetLabel(), tmpReg, Condition::EQ);
3909 slowPath->BindBackLabel(enc->GetEncoder());
3910 enc->GetEncoder()->EncodeMov(dst, tmpReg);
3911 }
3912 }
3913
VisitLoadResolvedObjectFieldStatic(GraphVisitor * visitor,Inst * inst)3914 void EncodeVisitor::VisitLoadResolvedObjectFieldStatic(GraphVisitor *visitor, Inst *inst)
3915 {
3916 auto *enc = static_cast<EncodeVisitor *>(visitor);
3917 auto load = inst->CastToLoadResolvedObjectFieldStatic();
3918 if (load->GetNeedBarrier()) {
3919 // Insert barriers for GC
3920 }
3921 auto type = inst->GetType();
3922 auto fieldAddr = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE);
3923 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
3924 // Unknown load, assume it can be volatile
3925 enc->GetEncoder()->EncodeLdrAcquire(dst, IsTypeSigned(type), MemRef(fieldAddr));
3926 }
3927
VisitStoreStatic(GraphVisitor * visitor,Inst * inst)3928 void EncodeVisitor::VisitStoreStatic(GraphVisitor *visitor, Inst *inst)
3929 {
3930 auto *enc = static_cast<EncodeVisitor *>(visitor);
3931 auto storeStatic = inst->CastToStoreStatic();
3932 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // class
3933 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), inst->GetType()); // store value
3934
3935 auto graph = enc->cg_->GetGraph();
3936 auto runtime = graph->GetRuntime();
3937 auto field = storeStatic->GetObjField();
3938 auto offset = runtime->GetFieldOffset(field);
3939 auto mem = MemRef(src0, offset);
3940
3941 if (inst->CastToStoreStatic()->GetNeedBarrier()) {
3942 enc->GetCodegen()->CreatePreWRB(inst, mem, MakeMask(src1.GetId()));
3943 }
3944 if (storeStatic->GetVolatile()) {
3945 enc->GetEncoder()->EncodeStrRelease(src1, mem);
3946 } else {
3947 enc->GetEncoder()->EncodeStr(src1, mem);
3948 }
3949 if (inst->CastToStoreStatic()->GetNeedBarrier()) {
3950 auto arch = enc->GetEncoder()->GetArch();
3951 auto tmpReg = enc->GetCodegen()->ConvertInstTmpReg(inst, DataType::GetIntTypeForReference(arch));
3952 enc->GetEncoder()->EncodeLdr(tmpReg, false, MemRef(src0, runtime->GetManagedClassOffset(enc->GetArch())));
3953 auto classHeaderMem = MemRef(tmpReg);
3954 enc->GetCodegen()->CreatePostWRB(inst, classHeaderMem, src1, INVALID_REGISTER);
3955 }
3956 }
3957
VisitLoadObjectDynamic(GraphVisitor * visitor,Inst * inst)3958 void EncodeVisitor::VisitLoadObjectDynamic(GraphVisitor *visitor, Inst *inst)
3959 {
3960 auto *enc = static_cast<EncodeVisitor *>(visitor);
3961 auto *codegen = enc->GetCodegen();
3962 codegen->CreateLoadObjectDynamic(inst);
3963 }
3964
VisitStoreObjectDynamic(GraphVisitor * visitor,Inst * inst)3965 void EncodeVisitor::VisitStoreObjectDynamic(GraphVisitor *visitor, Inst *inst)
3966 {
3967 auto *enc = static_cast<EncodeVisitor *>(visitor);
3968 auto *codegen = enc->GetCodegen();
3969 codegen->CreateStoreObjectDynamic(inst);
3970 }
3971
VisitUnresolvedStoreStatic(GraphVisitor * visitor,Inst * inst)3972 void EncodeVisitor::VisitUnresolvedStoreStatic(GraphVisitor *visitor, Inst *inst)
3973 {
3974 auto *enc = static_cast<EncodeVisitor *>(visitor);
3975 auto storeStatic = inst->CastToUnresolvedStoreStatic();
3976 ASSERT(storeStatic->GetType() == DataType::REFERENCE);
3977 ASSERT(storeStatic->GetNeedBarrier());
3978 auto typeId = storeStatic->GetTypeId();
3979 auto value = enc->GetCodegen()->ConvertRegister(storeStatic->GetSrcReg(0), storeStatic->GetType());
3980 auto entrypoint = RuntimeInterface::EntrypointId::UNRESOLVED_STORE_STATIC_BARRIERED;
3981 auto method = storeStatic->GetMethod();
3982 ASSERT(method != nullptr);
3983 enc->GetCodegen()->CallRuntimeWithMethod(storeStatic, method, entrypoint, Reg(), TypedImm(typeId), value);
3984 }
3985
VisitStoreResolvedObjectFieldStatic(GraphVisitor * visitor,Inst * inst)3986 void EncodeVisitor::VisitStoreResolvedObjectFieldStatic(GraphVisitor *visitor, Inst *inst)
3987 {
3988 auto *enc = static_cast<EncodeVisitor *>(visitor);
3989 auto store = inst->CastToStoreResolvedObjectFieldStatic();
3990 ASSERT(store->GetType() != DataType::REFERENCE);
3991 ASSERT(!store->GetNeedBarrier());
3992 auto val = enc->GetCodegen()->ConvertRegister(store->GetSrcReg(1), store->GetType());
3993 auto reg = enc->GetCodegen()->ConvertRegister(store->GetSrcReg(0), DataType::REFERENCE);
3994 // Non-barriered case. Unknown store, assume it can be volatile
3995 enc->GetEncoder()->EncodeStrRelease(val, MemRef(reg));
3996 }
3997
VisitNewObject(GraphVisitor * visitor,Inst * inst)3998 void EncodeVisitor::VisitNewObject(GraphVisitor *visitor, Inst *inst)
3999 {
4000 auto *enc = static_cast<EncodeVisitor *>(visitor);
4001 // NOTE(msherstennikov): use irtoced entrypoint once spill-fills will be supported for entrypoints mode.
4002 if (enc->cg_->GetArch() == Arch::AARCH32) {
4003 enc->GetCodegen()->CreateNewObjCallOld(inst->CastToNewObject());
4004 } else {
4005 enc->GetCodegen()->CreateNewObjCall(inst->CastToNewObject());
4006 }
4007 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
4008 enc->GetEncoder()->EncodeMemoryBarrier(memory_order::RELEASE);
4009 }
4010 }
4011
VisitUnresolvedLoadType(GraphVisitor * visitor,Inst * inst)4012 void EncodeVisitor::VisitUnresolvedLoadType(GraphVisitor *visitor, Inst *inst)
4013 {
4014 auto *enc = static_cast<EncodeVisitor *>(visitor);
4015 auto encoder = enc->GetEncoder();
4016 auto codegen = enc->GetCodegen();
4017 auto loadType = inst->CastToUnresolvedLoadType();
4018 if (loadType->GetNeedBarrier()) {
4019 // Consider inserting barriers for GC
4020 }
4021 auto dst = codegen->ConvertRegister(inst->GetDstReg(), inst->GetType());
4022 auto graph = enc->cg_->GetGraph();
4023 auto typeId = loadType->GetTypeId();
4024
4025 auto runtime = graph->GetRuntime();
4026 auto method = loadType->GetMethod();
4027
4028 if (graph->IsAotMode()) {
4029 ScopedTmpReg tmpReg(encoder);
4030 // Load pointer to klass from PLT
4031 codegen->CreateLoadClassFromPLT(inst, tmpReg, dst, typeId);
4032 // Finally load Object
4033 encoder->EncodeLdr(dst, false, MemRef(dst, runtime->GetManagedClassOffset(enc->GetArch())));
4034 } else {
4035 auto utypes = runtime->GetUnresolvedTypes();
4036 auto clsAddr = utypes->GetTableSlot(method, typeId, UnresolvedTypesInterface::SlotKind::MANAGED_CLASS);
4037 Reg dstPtr(dst.GetId(), codegen->GetPtrRegType());
4038 encoder->EncodeMov(dstPtr, Imm(clsAddr));
4039 encoder->EncodeLdr(dst, false, MemRef(dstPtr));
4040
4041 auto slowPath = codegen->CreateSlowPath<SlowPathUnresolved>(inst, EntrypointId::RESOLVE_CLASS_OBJECT);
4042 slowPath->SetUnresolvedType(method, typeId);
4043 slowPath->SetDstReg(dst);
4044 slowPath->SetSlotAddr(clsAddr);
4045 encoder->EncodeJump(slowPath->GetLabel(), dst, Condition::EQ);
4046 slowPath->BindBackLabel(encoder);
4047 }
4048 }
4049
VisitLoadType(GraphVisitor * visitor,Inst * inst)4050 void EncodeVisitor::VisitLoadType(GraphVisitor *visitor, Inst *inst)
4051 {
4052 auto *enc = static_cast<EncodeVisitor *>(visitor);
4053 auto encoder = enc->GetEncoder();
4054 auto loadType = inst->CastToLoadType();
4055 if (loadType->GetNeedBarrier()) {
4056 // Consider inserting barriers for GC
4057 }
4058 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
4059 auto graph = enc->cg_->GetGraph();
4060 auto typeId = loadType->GetTypeId();
4061
4062 auto runtime = graph->GetRuntime();
4063 auto method = loadType->GetMethod();
4064
4065 if (graph->IsAotMode()) {
4066 auto methodClassId = runtime->GetClassIdForMethod(graph->GetMethod());
4067 if (methodClassId == typeId) {
4068 auto dstPtr = dst.As(Codegen::ConvertDataType(DataType::POINTER, graph->GetArch()));
4069 enc->GetCodegen()->LoadMethod(dstPtr);
4070 auto mem = MemRef(dstPtr, graph->GetRuntime()->GetClassOffset(graph->GetArch()));
4071 encoder->EncodeLdr(dst.As(Codegen::ConvertDataType(DataType::REFERENCE, graph->GetArch())), false, mem);
4072 } else {
4073 ScopedTmpReg tmpReg(encoder);
4074 // Load pointer to klass from PLT
4075 enc->GetCodegen()->CreateLoadClassFromPLT(inst, tmpReg, dst, typeId);
4076 }
4077 // Finally load ManagedClass object
4078 encoder->EncodeLdr(dst, false, MemRef(dst, runtime->GetManagedClassOffset(enc->GetArch())));
4079 } else { // JIT mode
4080 auto klass = reinterpret_cast<uintptr_t>(runtime->ResolveType(method, typeId));
4081 auto managedKlass = runtime->GetManagedType(klass);
4082 encoder->EncodeMov(dst, Imm(managedKlass));
4083 }
4084 }
4085
FillUnresolvedClass(GraphVisitor * visitor,Inst * inst)4086 void EncodeVisitor::FillUnresolvedClass(GraphVisitor *visitor, Inst *inst)
4087 {
4088 auto *enc = static_cast<EncodeVisitor *>(visitor);
4089 auto graph = enc->cg_->GetGraph();
4090 auto encoder = enc->GetEncoder();
4091 auto classReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::REFERENCE);
4092 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathEntrypoint>(inst, EntrypointId::CHECK_CAST);
4093 encoder->EncodeJump(slowPath->GetLabel(), classReg, Condition::EQ);
4094 slowPath->CreateBackLabel(encoder);
4095 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // obj
4096 encoder->EncodeJump(slowPath->GetBackLabel(), src, Condition::EQ);
4097 ScopedTmpReg tmpReg(encoder, Codegen::ConvertDataType(DataType::REFERENCE, graph->GetArch()));
4098 enc->GetCodegen()->LoadClassFromObject(tmpReg, src);
4099 encoder->EncodeJump(slowPath->GetLabel(), tmpReg, classReg, Condition::NE);
4100 slowPath->BindBackLabel(encoder);
4101 }
4102
FillObjectClass(GraphVisitor * visitor,Reg tmpReg,LabelHolder::LabelId throwLabel)4103 void EncodeVisitor::FillObjectClass(GraphVisitor *visitor, Reg tmpReg, LabelHolder::LabelId throwLabel)
4104 {
4105 auto *enc = static_cast<EncodeVisitor *>(visitor);
4106 auto graph = enc->cg_->GetGraph();
4107 auto runtime = graph->GetRuntime();
4108 auto encoder = enc->GetEncoder();
4109
4110 Reg typeReg(tmpReg.GetId(), INT8_TYPE);
4111 // Load type class
4112 encoder->EncodeLdr(typeReg, false, MemRef(tmpReg, runtime->GetClassTypeOffset(enc->GetArch())));
4113 // Jump to EH if type not reference
4114 encoder->EncodeJump(throwLabel, typeReg, Imm(runtime->GetReferenceTypeMask()), Condition::NE);
4115 }
4116
4117 /* The CheckCast class should be a subclass of input class:
4118 ......................
4119 bool Class::IsSubClassOf(const Class *klass) const {
4120 const Class *current = this;
4121 do {
4122 if (current == klass) {
4123 return true;
4124 }
4125 current = current->GetBase();
4126 } while (current != nullptr);
4127 return false;
4128 }
4129 */
4130
FillOtherClass(GraphVisitor * visitor,Inst * inst,Reg tmpReg,LabelHolder::LabelId throwLabel)4131 void EncodeVisitor::FillOtherClass(GraphVisitor *visitor, Inst *inst, Reg tmpReg, LabelHolder::LabelId throwLabel)
4132 {
4133 auto *enc = static_cast<EncodeVisitor *>(visitor);
4134 auto graph = enc->cg_->GetGraph();
4135 auto encoder = enc->GetEncoder();
4136 auto loopLabel = encoder->CreateLabel();
4137
4138 // First compare `current == klass` we make before switch
4139 encoder->BindLabel(loopLabel);
4140 // Load base klass
4141 encoder->EncodeLdr(tmpReg, false, MemRef(tmpReg, graph->GetRuntime()->GetClassBaseOffset(enc->GetArch())));
4142 encoder->EncodeJump(throwLabel, tmpReg, Condition::EQ);
4143 auto classReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::REFERENCE);
4144 encoder->EncodeJump(loopLabel, tmpReg, classReg, Condition::NE);
4145 }
4146
FillArrayObjectClass(GraphVisitor * visitor,Reg tmpReg,LabelHolder::LabelId throwLabel)4147 void EncodeVisitor::FillArrayObjectClass(GraphVisitor *visitor, Reg tmpReg, LabelHolder::LabelId throwLabel)
4148 {
4149 auto *enc = static_cast<EncodeVisitor *>(visitor);
4150 auto graph = enc->cg_->GetGraph();
4151 auto runtime = graph->GetRuntime();
4152 auto encoder = enc->GetEncoder();
4153
4154 Reg typeReg(tmpReg.GetId(), INT8_TYPE);
4155 // Load Component class
4156 encoder->EncodeLdr(tmpReg, false, MemRef(tmpReg, runtime->GetClassComponentTypeOffset(enc->GetArch())));
4157 // Jump to EH if src is not array class
4158 encoder->EncodeJump(throwLabel, tmpReg, Condition::EQ);
4159 // Load type of the component class
4160 encoder->EncodeLdr(typeReg, false, MemRef(tmpReg, runtime->GetClassTypeOffset(enc->GetArch())));
4161 // Jump to EH if type not reference
4162 encoder->EncodeJump(throwLabel, typeReg, Imm(runtime->GetReferenceTypeMask()), Condition::NE);
4163 }
4164
FillArrayClass(GraphVisitor * visitor,Inst * inst,Reg tmpReg,LabelHolder::LabelId throwLabel)4165 void EncodeVisitor::FillArrayClass(GraphVisitor *visitor, Inst *inst, Reg tmpReg, LabelHolder::LabelId throwLabel)
4166 {
4167 auto *enc = static_cast<EncodeVisitor *>(visitor);
4168 auto graph = enc->cg_->GetGraph();
4169 auto runtime = graph->GetRuntime();
4170 auto encoder = enc->GetEncoder();
4171
4172 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathEntrypoint>(inst, EntrypointId::CHECK_CAST);
4173
4174 // Load Component type of Input
4175 encoder->EncodeLdr(tmpReg, false, MemRef(tmpReg, runtime->GetClassComponentTypeOffset(enc->GetArch())));
4176 // Check that src is array class
4177 encoder->EncodeJump(throwLabel, tmpReg, Condition::EQ);
4178 // Load Component type of the instance
4179 auto classReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::REFERENCE);
4180 ScopedTmpReg tmpReg1(encoder, Codegen::ConvertDataType(DataType::REFERENCE, graph->GetArch()));
4181 encoder->EncodeLdr(tmpReg1, false, MemRef(classReg, runtime->GetClassComponentTypeOffset(enc->GetArch())));
4182 // Compare component types
4183 encoder->EncodeJump(slowPath->GetLabel(), tmpReg, tmpReg1, Condition::NE);
4184
4185 slowPath->BindBackLabel(encoder);
4186 }
4187
FillInterfaceClass(GraphVisitor * visitor,Inst * inst)4188 void EncodeVisitor::FillInterfaceClass(GraphVisitor *visitor, Inst *inst)
4189 {
4190 auto enc = static_cast<EncodeVisitor *>(visitor);
4191 auto encoder = enc->GetEncoder();
4192 auto codegen = enc->GetCodegen();
4193 if (codegen->GetArch() == Arch::AARCH32) {
4194 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathEntrypoint>(inst, EntrypointId::CHECK_CAST);
4195 encoder->EncodeJump(slowPath->GetLabel());
4196 slowPath->BindBackLabel(encoder);
4197 } else {
4198 codegen->CreateCheckCastInterfaceCall(inst);
4199 }
4200 }
4201
FillCheckCast(GraphVisitor * visitor,Inst * inst,Reg src,LabelHolder::LabelId endLabel,compiler::ClassType klassType)4202 void EncodeVisitor::FillCheckCast(GraphVisitor *visitor, Inst *inst, Reg src, LabelHolder::LabelId endLabel,
4203 compiler::ClassType klassType)
4204 {
4205 if (klassType == ClassType::INTERFACE_CLASS) {
4206 FillInterfaceClass(visitor, inst);
4207 return;
4208 }
4209 auto *enc = static_cast<EncodeVisitor *>(visitor);
4210 auto graph = enc->cg_->GetGraph();
4211 auto encoder = enc->GetEncoder();
4212 // class_reg - CheckCast class
4213 // tmp_reg - input class
4214 auto classReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::REFERENCE);
4215 ScopedTmpReg tmpReg(encoder, Codegen::ConvertDataType(DataType::REFERENCE, graph->GetArch()));
4216 enc->GetCodegen()->LoadClassFromObject(tmpReg, src);
4217 // There is no exception if the classes are equal
4218 encoder->EncodeJump(endLabel, classReg, tmpReg, Condition::EQ);
4219
4220 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathCheckCast>(inst, EntrypointId::CLASS_CAST_EXCEPTION);
4221 slowPath->SetClassReg(classReg);
4222 auto throwLabel = slowPath->GetLabel();
4223 switch (klassType) {
4224 // The input class should be not primitive type
4225 case ClassType::OBJECT_CLASS: {
4226 FillObjectClass(visitor, tmpReg, throwLabel);
4227 break;
4228 }
4229 case ClassType::OTHER_CLASS: {
4230 FillOtherClass(visitor, inst, tmpReg, throwLabel);
4231 break;
4232 }
4233 // The input class should be array class and component type should be not primitive type
4234 case ClassType::ARRAY_OBJECT_CLASS: {
4235 FillArrayObjectClass(visitor, tmpReg, throwLabel);
4236 break;
4237 }
4238 // Check that components types are equals, else call slow path
4239 case ClassType::ARRAY_CLASS: {
4240 FillArrayClass(visitor, inst, tmpReg, throwLabel);
4241 break;
4242 }
4243 case ClassType::FINAL_CLASS: {
4244 EVENT_CODEGEN_SIMPLIFICATION(events::CodegenSimplificationInst::CHECKCAST,
4245 events::CodegenSimplificationReason::FINAL_CLASS);
4246 encoder->EncodeJump(throwLabel);
4247 break;
4248 }
4249 default: {
4250 UNREACHABLE();
4251 }
4252 }
4253 }
4254
VisitCheckCast(GraphVisitor * visitor,Inst * inst)4255 void EncodeVisitor::VisitCheckCast(GraphVisitor *visitor, Inst *inst)
4256 {
4257 auto *enc = static_cast<EncodeVisitor *>(visitor);
4258 auto method = inst->CastToCheckCast()->GetMethod();
4259 auto typeId = inst->CastToCheckCast()->GetTypeId();
4260 auto encoder = enc->GetEncoder();
4261
4262 auto klassType = inst->CastToCheckCast()->GetClassType();
4263 if (klassType == ClassType::UNRESOLVED_CLASS) {
4264 FillUnresolvedClass(visitor, inst);
4265 return;
4266 }
4267 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // obj
4268 auto endLabel = encoder->CreateLabel();
4269
4270 if (inst->CastToCheckCast()->GetOmitNullCheck()) {
4271 EVENT_CODEGEN_SIMPLIFICATION(events::CodegenSimplificationInst::CHECKCAST,
4272 events::CodegenSimplificationReason::SKIP_NULLCHECK);
4273 } else {
4274 // Compare with nullptr
4275 encoder->EncodeJump(endLabel, src, Condition::EQ);
4276 }
4277
4278 [[maybe_unused]] auto klass = enc->cg_->GetGraph()->GetRuntime()->GetClass(method, typeId);
4279 ASSERT(klass != nullptr);
4280
4281 FillCheckCast(visitor, inst, src, endLabel, klassType);
4282 encoder->BindLabel(endLabel);
4283 }
4284
FillIsInstanceUnresolved(GraphVisitor * visitor,Inst * inst)4285 void EncodeVisitor::FillIsInstanceUnresolved(GraphVisitor *visitor, Inst *inst)
4286 {
4287 auto *enc = static_cast<EncodeVisitor *>(visitor);
4288 auto graph = enc->cg_->GetGraph();
4289 auto encoder = enc->GetEncoder();
4290 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // obj
4291 auto classReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::REFERENCE);
4292 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
4293
4294 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathEntrypoint>(inst, EntrypointId::IS_INSTANCE);
4295 encoder->EncodeJump(slowPath->GetLabel(), classReg, Condition::EQ);
4296 slowPath->CreateBackLabel(encoder);
4297 auto endLabel = slowPath->GetBackLabel();
4298
4299 // Compare with nullptr
4300 auto nextLabel = encoder->CreateLabel();
4301 encoder->EncodeJump(nextLabel, src, Condition::NE);
4302 encoder->EncodeMov(dst, Imm(0));
4303 encoder->EncodeJump(endLabel);
4304 encoder->BindLabel(nextLabel);
4305
4306 // Get instance class
4307 ScopedTmpReg tmpReg(encoder, Codegen::ConvertDataType(DataType::REFERENCE, graph->GetArch()));
4308 enc->GetCodegen()->LoadClassFromObject(tmpReg, src);
4309
4310 // Sets true if the classes are equal
4311 encoder->EncodeJump(slowPath->GetLabel(), tmpReg, classReg, Condition::NE);
4312 encoder->EncodeMov(dst, Imm(1));
4313
4314 slowPath->BindBackLabel(encoder);
4315 }
4316
FillIsInstanceCaseObject(GraphVisitor * visitor,Inst * inst,Reg tmpReg)4317 void EncodeVisitor::FillIsInstanceCaseObject(GraphVisitor *visitor, Inst *inst, Reg tmpReg)
4318 {
4319 auto *enc = static_cast<EncodeVisitor *>(visitor);
4320 auto encoder = enc->GetEncoder();
4321 auto runtime = enc->cg_->GetGraph()->GetRuntime();
4322 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
4323
4324 // ClassType::OBJECT_CLASS
4325 Reg typeReg(tmpReg.GetId(), INT8_TYPE);
4326 // Load type class
4327 encoder->EncodeLdr(typeReg, false, MemRef(tmpReg, runtime->GetClassTypeOffset(enc->GetArch())));
4328 ScopedTmpReg typeMaskReg(encoder, INT8_TYPE);
4329 encoder->EncodeMov(typeMaskReg, Imm(runtime->GetReferenceTypeMask()));
4330 encoder->EncodeCompare(dst, typeMaskReg, typeReg, Condition::EQ);
4331 }
4332
4333 /* Sets true if IsInstance class is a subclass of input class:
4334 ......................
4335 bool Class::IsSubClassOf(const Class *klass) const {
4336 const Class *current = this;
4337 do {
4338 if (current == klass) {
4339 return true;
4340 }
4341 current = current->GetBase();
4342 } while (current != nullptr);
4343 return false;
4344 }
4345 */
4346
FillIsInstanceCaseOther(GraphVisitor * visitor,Inst * inst,Reg tmpReg,LabelHolder::LabelId endLabel)4347 void EncodeVisitor::FillIsInstanceCaseOther(GraphVisitor *visitor, Inst *inst, Reg tmpReg,
4348 LabelHolder::LabelId endLabel)
4349 {
4350 auto *enc = static_cast<EncodeVisitor *>(visitor);
4351 auto encoder = enc->GetEncoder();
4352 auto runtime = enc->cg_->GetGraph()->GetRuntime();
4353 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
4354 auto classReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::REFERENCE);
4355
4356 // ClassType::OTHER_CLASS
4357 auto loopLabel = encoder->CreateLabel();
4358 auto falseLabel = encoder->CreateLabel();
4359
4360 // First compare `current == klass` we make before switch
4361 encoder->BindLabel(loopLabel);
4362 // Load base klass
4363 encoder->EncodeLdr(tmpReg, false, MemRef(tmpReg, runtime->GetClassBaseOffset(enc->GetArch())));
4364 encoder->EncodeJump(falseLabel, tmpReg, Condition::EQ);
4365 encoder->EncodeJump(loopLabel, tmpReg, classReg, Condition::NE);
4366
4367 // Set true result and jump to exit
4368 encoder->EncodeMov(dst, Imm(1));
4369 encoder->EncodeJump(endLabel);
4370
4371 // Set false result and jump to exit
4372 encoder->BindLabel(falseLabel);
4373 encoder->EncodeMov(dst, Imm(0));
4374 }
4375
4376 // Sets true if the Input class is array class and component type is not primitive type
FillIsInstanceCaseArrayObject(GraphVisitor * visitor,Inst * inst,Reg tmpReg,LabelHolder::LabelId endLabel)4377 void EncodeVisitor::FillIsInstanceCaseArrayObject(GraphVisitor *visitor, Inst *inst, Reg tmpReg,
4378 LabelHolder::LabelId endLabel)
4379 {
4380 auto *enc = static_cast<EncodeVisitor *>(visitor);
4381 auto encoder = enc->GetEncoder();
4382 auto graph = enc->cg_->GetGraph();
4383 auto runtime = graph->GetRuntime();
4384 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
4385
4386 // ClassType::ARRAY_OBJECT_CLASS
4387 Reg dstRef(dst.GetId(), Codegen::ConvertDataType(DataType::REFERENCE, graph->GetArch()));
4388 Reg typeReg(tmpReg.GetId(), INT8_TYPE);
4389 // Load Component class
4390 encoder->EncodeLdr(dstRef, false, MemRef(tmpReg, runtime->GetClassComponentTypeOffset(enc->GetArch())));
4391 // Check that src is array class
4392 encoder->EncodeJump(endLabel, dstRef, Condition::EQ);
4393 // Load type of the component class
4394 encoder->EncodeLdr(typeReg, false, MemRef(dstRef, runtime->GetClassTypeOffset(enc->GetArch())));
4395 ScopedTmpReg typeMaskReg(encoder, INT8_TYPE);
4396 encoder->EncodeMov(typeMaskReg, Imm(runtime->GetReferenceTypeMask()));
4397 encoder->EncodeCompare(dst, typeMaskReg, typeReg, Condition::EQ);
4398 }
4399
4400 // Check that components types are equals, else call slow path
FillIsInstanceCaseArrayClass(GraphVisitor * visitor,Inst * inst,Reg tmpReg,LabelHolder::LabelId endLabel)4401 void EncodeVisitor::FillIsInstanceCaseArrayClass(GraphVisitor *visitor, Inst *inst, Reg tmpReg,
4402 LabelHolder::LabelId endLabel)
4403 {
4404 auto *enc = static_cast<EncodeVisitor *>(visitor);
4405 auto encoder = enc->GetEncoder();
4406 auto graph = enc->cg_->GetGraph();
4407 auto runtime = graph->GetRuntime();
4408 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
4409
4410 // ClassType::ARRAY_CLASS
4411 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathEntrypoint>(inst, EntrypointId::IS_INSTANCE);
4412
4413 auto nextLabel1 = encoder->CreateLabel();
4414 // Load Component type of Input
4415 encoder->EncodeLdr(tmpReg, false, MemRef(tmpReg, runtime->GetClassComponentTypeOffset(enc->GetArch())));
4416 // Check that src is array class
4417 encoder->EncodeJump(nextLabel1, tmpReg, Condition::NE);
4418 encoder->EncodeMov(dst, Imm(0));
4419 encoder->EncodeJump(endLabel);
4420 encoder->BindLabel(nextLabel1);
4421 // Load Component type of the instance
4422 auto classReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::REFERENCE);
4423 ScopedTmpReg tmpReg1(encoder, Codegen::ConvertDataType(DataType::REFERENCE, graph->GetArch()));
4424 encoder->EncodeLdr(tmpReg1, false, MemRef(classReg, runtime->GetClassComponentTypeOffset(enc->GetArch())));
4425 // Compare component types
4426 encoder->EncodeJump(slowPath->GetLabel(), tmpReg, tmpReg1, Condition::NE);
4427 encoder->EncodeMov(dst, Imm(1));
4428
4429 slowPath->BindBackLabel(encoder);
4430 }
4431
FillIsInstanceCaseInterface(GraphVisitor * visitor,Inst * inst)4432 void EncodeVisitor::FillIsInstanceCaseInterface(GraphVisitor *visitor, Inst *inst)
4433 {
4434 auto *enc = static_cast<EncodeVisitor *>(visitor);
4435 auto encoder = enc->GetEncoder();
4436 // ClassType::INTERFACE_CLASS
4437 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathEntrypoint>(inst, EntrypointId::IS_INSTANCE);
4438
4439 encoder->EncodeJump(slowPath->GetLabel());
4440
4441 slowPath->BindBackLabel(encoder);
4442 }
4443
FillIsInstance(GraphVisitor * visitor,Inst * inst,Reg tmpReg,LabelHolder::LabelId endLabel)4444 void EncodeVisitor::FillIsInstance(GraphVisitor *visitor, Inst *inst, Reg tmpReg, LabelHolder::LabelId endLabel)
4445 {
4446 auto *enc = static_cast<EncodeVisitor *>(visitor);
4447 auto encoder = enc->GetEncoder();
4448 auto src = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // obj
4449 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
4450
4451 if (inst->CastToIsInstance()->GetOmitNullCheck()) {
4452 EVENT_CODEGEN_SIMPLIFICATION(events::CodegenSimplificationInst::ISINSTANCE,
4453 events::CodegenSimplificationReason::SKIP_NULLCHECK);
4454 } else {
4455 // Compare with nullptr
4456 auto nextLabel = encoder->CreateLabel();
4457 encoder->EncodeJump(nextLabel, src, Condition::NE);
4458 encoder->EncodeMov(dst, Imm(0));
4459 encoder->EncodeJump(endLabel);
4460 encoder->BindLabel(nextLabel);
4461 }
4462
4463 auto classReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::REFERENCE);
4464 enc->GetCodegen()->LoadClassFromObject(tmpReg, src);
4465
4466 // Sets true if the classes are equals
4467 if (inst->CastToIsInstance()->GetClassType() == ClassType::FINAL_CLASS) {
4468 encoder->EncodeCompare(dst, classReg, tmpReg, Condition::EQ);
4469 } else if (dst.GetId() != src.GetId() && dst.GetId() != classReg.GetId()) {
4470 encoder->EncodeCompare(dst, classReg, tmpReg, Condition::EQ);
4471 encoder->EncodeJump(endLabel, dst, Condition::NE);
4472 } else {
4473 auto nextLabel1 = encoder->CreateLabel();
4474 encoder->EncodeJump(nextLabel1, classReg, tmpReg, Condition::NE);
4475 encoder->EncodeMov(dst, Imm(1));
4476 encoder->EncodeJump(endLabel);
4477 encoder->BindLabel(nextLabel1);
4478 }
4479 }
4480
VisitIsInstance(GraphVisitor * visitor,Inst * inst)4481 void EncodeVisitor::VisitIsInstance(GraphVisitor *visitor, Inst *inst)
4482 {
4483 auto *enc = static_cast<EncodeVisitor *>(visitor);
4484 auto graph = enc->cg_->GetGraph();
4485 auto encoder = enc->GetEncoder();
4486 auto klassType = inst->CastToIsInstance()->GetClassType();
4487 if (klassType == ClassType::UNRESOLVED_CLASS) {
4488 FillIsInstanceUnresolved(visitor, inst);
4489 return;
4490 }
4491 // tmp_reg - input class
4492 ScopedTmpReg tmpReg(encoder, Codegen::ConvertDataType(DataType::REFERENCE, graph->GetArch()));
4493 auto endLabel = encoder->CreateLabel();
4494
4495 FillIsInstance(visitor, inst, tmpReg, endLabel);
4496 switch (klassType) {
4497 // Sets true if the Input class is not primitive type
4498 case ClassType::OBJECT_CLASS: {
4499 FillIsInstanceCaseObject(visitor, inst, tmpReg);
4500 break;
4501 }
4502 case ClassType::OTHER_CLASS: {
4503 FillIsInstanceCaseOther(visitor, inst, tmpReg, endLabel);
4504 break;
4505 }
4506 // Sets true if the Input class is array class and component type is not primitive type
4507 case ClassType::ARRAY_OBJECT_CLASS: {
4508 FillIsInstanceCaseArrayObject(visitor, inst, tmpReg, endLabel);
4509 break;
4510 }
4511 // Check that components types are equals, else call slow path
4512 case ClassType::ARRAY_CLASS: {
4513 FillIsInstanceCaseArrayClass(visitor, inst, tmpReg, endLabel);
4514 break;
4515 }
4516 case ClassType::INTERFACE_CLASS: {
4517 FillIsInstanceCaseInterface(visitor, inst);
4518 break;
4519 }
4520 case ClassType::FINAL_CLASS: {
4521 EVENT_CODEGEN_SIMPLIFICATION(events::CodegenSimplificationInst::ISINSTANCE,
4522 events::CodegenSimplificationReason::FINAL_CLASS);
4523 break;
4524 }
4525 default: {
4526 UNREACHABLE();
4527 }
4528 }
4529 encoder->BindLabel(endLabel);
4530 }
4531
CreateMonitorCall(MonitorInst * inst)4532 void Codegen::CreateMonitorCall(MonitorInst *inst)
4533 {
4534 auto src = ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // obj
4535 auto id = inst->IsExit() ? EntrypointId::MONITOR_EXIT_FAST_PATH : EntrypointId::MONITOR_ENTER_FAST_PATH;
4536 CallFastPath(inst, id, INVALID_REGISTER, RegMask::GetZeroMask(), src);
4537 }
4538
CreateMonitorCallOld(MonitorInst * inst)4539 void Codegen::CreateMonitorCallOld(MonitorInst *inst)
4540 {
4541 auto src = ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // obj
4542 auto dst = ConvertRegister(inst->GetDstReg(), inst->GetType());
4543 auto id = inst->IsExit() ? EntrypointId::UNLOCK_OBJECT : EntrypointId::LOCK_OBJECT;
4544 CallRuntime(inst, id, dst, RegMask::GetZeroMask(), src);
4545 }
4546
CreateCheckCastInterfaceCall(Inst * inst)4547 void Codegen::CreateCheckCastInterfaceCall(Inst *inst)
4548 {
4549 auto obj = ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE);
4550 auto interface = ConvertRegister(inst->GetSrcReg(SECOND_OPERAND), DataType::REFERENCE);
4551 CallFastPath(inst, EntrypointId::CHECK_CAST_INTERFACE, INVALID_REGISTER, RegMask::GetZeroMask(), obj, interface);
4552 }
4553
VisitMonitor(GraphVisitor * visitor,Inst * inst)4554 void EncodeVisitor::VisitMonitor(GraphVisitor *visitor, Inst *inst)
4555 {
4556 auto *enc = static_cast<EncodeVisitor *>(visitor);
4557 if (enc->GetCodegen()->GetArch() == Arch::AARCH32) {
4558 enc->GetCodegen()->CreateMonitorCallOld(inst->CastToMonitor());
4559 } else {
4560 enc->GetCodegen()->CreateMonitorCall(inst->CastToMonitor());
4561 }
4562 }
4563
TryInsertImplicitNullCheck(Inst * inst,size_t prevOffset)4564 void Codegen::TryInsertImplicitNullCheck(Inst *inst, size_t prevOffset)
4565 {
4566 if (!IsSuitableForImplicitNullCheck(inst)) {
4567 return;
4568 }
4569 if (!inst->CanThrow()) {
4570 return;
4571 }
4572
4573 auto nullcheck = inst->GetInput(0).GetInst();
4574 ASSERT(nullcheck->GetOpcode() == Opcode::NullCheck && nullcheck->CastToNullCheck()->IsImplicit());
4575 auto currOffset = GetEncoder()->GetCursorOffset();
4576 ASSERT(currOffset > prevOffset);
4577 GetCodeBuilder()->AddImplicitNullCheck(currOffset, currOffset - prevOffset);
4578 CreateStackMap(nullcheck, inst);
4579 }
4580
CreateFloatIsInf(IntrinsicInst * inst,Reg dst,SRCREGS src)4581 void Codegen::CreateFloatIsInf([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
4582 {
4583 GetEncoder()->EncodeIsInf(dst, src[0]);
4584 }
4585
CreateFloatIsInteger(IntrinsicInst * inst,Reg dst,SRCREGS src)4586 void Codegen::CreateFloatIsInteger([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
4587 {
4588 GetEncoder()->EncodeIsInteger(dst, src[0]);
4589 }
4590
CreateFloatIsSafeInteger(IntrinsicInst * inst,Reg dst,SRCREGS src)4591 void Codegen::CreateFloatIsSafeInteger([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
4592 {
4593 GetEncoder()->EncodeIsSafeInteger(dst, src[0]);
4594 }
4595
CreateStringEquals(IntrinsicInst * inst,Reg dst,SRCREGS src)4596 void Codegen::CreateStringEquals([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
4597 {
4598 auto entrypointId = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::STRING_EQUALS_COMPRESSED
4599 : EntrypointId::STRING_EQUALS;
4600 CallFastPath(inst, entrypointId, dst, {}, src[0], src[1U]);
4601 }
4602
CreateMathCeil(IntrinsicInst * inst,Reg dst,SRCREGS src)4603 void Codegen::CreateMathCeil([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
4604 {
4605 GetEncoder()->EncodeCeil(dst, src[0]);
4606 }
4607
CreateMathFloor(IntrinsicInst * inst,Reg dst,SRCREGS src)4608 void Codegen::CreateMathFloor([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
4609 {
4610 GetEncoder()->EncodeFloor(dst, src[0]);
4611 }
4612
CreateCountLeadingZeroBits(IntrinsicInst * inst,Reg dst,SRCREGS src)4613 void Codegen::CreateCountLeadingZeroBits([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
4614 {
4615 GetEncoder()->EncodeCountLeadingZeroBits(Reg(dst.GetId(), src[0].GetType()), src[0]);
4616 }
4617
CreateStringBuilderChar(IntrinsicInst * inst,Reg dst,SRCREGS src)4618 void Codegen::CreateStringBuilderChar(IntrinsicInst *inst, [[maybe_unused]] Reg dst, SRCREGS src)
4619 {
4620 CallFastPath(inst, EntrypointId::STRING_BUILDER_CHAR, dst, RegMask::GetZeroMask(), src[0], src[1U]);
4621 }
4622
CreateStringBuilderBool(IntrinsicInst * inst,Reg dst,SRCREGS src)4623 void Codegen::CreateStringBuilderBool(IntrinsicInst *inst, [[maybe_unused]] Reg dst, SRCREGS src)
4624 {
4625 CallFastPath(inst, EntrypointId::STRING_BUILDER_BOOL, dst, RegMask::GetZeroMask(), src[0], src[1U]);
4626 }
4627
CreateStringBuilderString(IntrinsicInst * inst,Reg dst,SRCREGS src)4628 void Codegen::CreateStringBuilderString(IntrinsicInst *inst, [[maybe_unused]] Reg dst, SRCREGS src)
4629 {
4630 auto entrypointId = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::STRING_BUILDER_STRING_COMPRESSED
4631 : EntrypointId::STRING_BUILDER_STRING;
4632
4633 CallFastPath(inst, entrypointId, dst, RegMask::GetZeroMask(), src[0], src[1U]);
4634 }
4635
CallFastCreateStringFromCharArrayTlab(Inst * inst,Reg dst,Reg offset,Reg count,Reg array,std::variant<Reg,TypedImm> klass)4636 void Codegen::CallFastCreateStringFromCharArrayTlab(Inst *inst, Reg dst, Reg offset, Reg count, Reg array,
4637 std::variant<Reg, TypedImm> klass)
4638 {
4639 if (GetRegfile()->GetZeroReg().GetId() == offset.GetId()) {
4640 auto entryId = GetRuntime()->IsCompressedStringsEnabled()
4641 ? EntrypointId::CREATE_STRING_FROM_ZERO_BASED_CHAR_ARRAY_TLAB_COMPRESSED
4642 : EntrypointId::CREATE_STRING_FROM_ZERO_BASED_CHAR_ARRAY_TLAB;
4643 if (std::holds_alternative<TypedImm>(klass)) {
4644 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), count, array, std::get<TypedImm>(klass));
4645 } else {
4646 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), count, array, std::get<Reg>(klass));
4647 }
4648 } else {
4649 auto entryId = GetRuntime()->IsCompressedStringsEnabled()
4650 ? EntrypointId::CREATE_STRING_FROM_CHAR_ARRAY_TLAB_COMPRESSED
4651 : EntrypointId::CREATE_STRING_FROM_CHAR_ARRAY_TLAB;
4652 if (std::holds_alternative<TypedImm>(klass)) {
4653 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), offset, count, array, std::get<TypedImm>(klass));
4654 } else {
4655 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), offset, count, array, std::get<Reg>(klass));
4656 }
4657 }
4658 }
4659
CreateStringFromCharArrayTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)4660 void Codegen::CreateStringFromCharArrayTlab(IntrinsicInst *inst, Reg dst, SRCREGS src)
4661 {
4662 auto runtime = GetGraph()->GetRuntime();
4663 auto offset = src[FIRST_OPERAND];
4664 auto count = src[SECOND_OPERAND];
4665 auto array = src[THIRD_OPERAND];
4666 if (GetGraph()->IsAotMode()) {
4667 ScopedTmpReg klassReg(GetEncoder());
4668 GetEncoder()->EncodeLdr(klassReg, false,
4669 MemRef(ThreadReg(), runtime->GetStringClassPointerTlsOffset(GetArch())));
4670 CallFastCreateStringFromCharArrayTlab(inst, dst, offset, count, array, klassReg);
4671 } else {
4672 auto klassImm = TypedImm(reinterpret_cast<uintptr_t>(runtime->GetStringClass(GetGraph()->GetMethod())));
4673 CallFastCreateStringFromCharArrayTlab(inst, dst, offset, count, array, klassImm);
4674 }
4675 }
4676
CreateStringFromStringTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)4677 void Codegen::CreateStringFromStringTlab(IntrinsicInst *inst, Reg dst, SRCREGS src)
4678 {
4679 auto entryId = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::CREATE_STRING_FROM_STRING_TLAB_COMPRESSED
4680 : EntrypointId::CREATE_STRING_FROM_STRING_TLAB;
4681 auto srcStr = src[FIRST_OPERAND];
4682 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), srcStr);
4683 }
4684
CreateStringSubstringTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)4685 void Codegen::CreateStringSubstringTlab([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
4686 {
4687 auto entrypointId = GetRuntime()->IsCompressedStringsEnabled()
4688 ? EntrypointId::SUB_STRING_FROM_STRING_TLAB_COMPRESSED
4689 : EntrypointId::SUB_STRING_FROM_STRING_TLAB;
4690 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND]);
4691 }
4692
CreateStringGetCharsTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)4693 void Codegen::CreateStringGetCharsTlab([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
4694 {
4695 auto entrypointId = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::STRING_GET_CHARS_TLAB_COMPRESSED
4696 : EntrypointId::STRING_GET_CHARS_TLAB;
4697 auto runtime = GetGraph()->GetRuntime();
4698 if (GetGraph()->IsAotMode()) {
4699 ScopedTmpReg klassReg(GetEncoder());
4700 GetEncoder()->EncodeLdr(klassReg, false,
4701 MemRef(ThreadReg(), runtime->GetArrayU16ClassPointerTlsOffset(GetArch())));
4702 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
4703 klassReg);
4704 } else {
4705 auto klassImm = TypedImm(reinterpret_cast<uintptr_t>(runtime->GetArrayU16Class(GetGraph()->GetMethod())));
4706 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
4707 klassImm);
4708 }
4709 }
4710
CreateStringHashCode(IntrinsicInst * inst,Reg dst,SRCREGS src)4711 void Codegen::CreateStringHashCode([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
4712 {
4713 auto entrypoint = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::STRING_HASH_CODE_COMPRESSED
4714 : EntrypointId::STRING_HASH_CODE;
4715 auto strReg = src[FIRST_OPERAND];
4716 auto mref = MemRef(strReg, panda::coretypes::String::GetHashcodeOffset());
4717 auto slowPath = CreateSlowPath<SlowPathStringHashCode>(inst, entrypoint);
4718 slowPath->SetDstReg(dst);
4719 slowPath->SetSrcReg(strReg);
4720 if (dst.GetId() != strReg.GetId()) {
4721 GetEncoder()->EncodeLdr(ConvertRegister(dst.GetId(), DataType::INT32), false, mref);
4722 GetEncoder()->EncodeJump(slowPath->GetLabel(), dst, Condition::EQ);
4723 } else {
4724 ScopedTmpReg hashReg(GetEncoder(), INT32_TYPE);
4725 GetEncoder()->EncodeLdr(hashReg, false, mref);
4726 GetEncoder()->EncodeJump(slowPath->GetLabel(), hashReg, Condition::EQ);
4727 GetEncoder()->EncodeMov(dst, hashReg);
4728 }
4729 slowPath->BindBackLabel(GetEncoder());
4730 }
4731
4732 #include "intrinsics_codegen.inl"
4733
CreateBuiltinIntrinsic(IntrinsicInst * inst)4734 void Codegen::CreateBuiltinIntrinsic(IntrinsicInst *inst)
4735 {
4736 Reg dst = INVALID_REGISTER;
4737 SRCREGS src;
4738
4739 if (!inst->NoDest()) {
4740 dst = ConvertRegister(inst->GetDstReg(), inst->GetType());
4741 }
4742
4743 for (size_t i = 0; i < inst->GetInputsCount(); ++i) {
4744 if (inst->GetInput(i).GetInst()->IsSaveState()) {
4745 continue;
4746 }
4747 auto location = inst->GetLocation(i);
4748 auto type = inst->GetInputType(i);
4749 src[i] = ConvertRegister(location.GetValue(), type);
4750 }
4751 FillBuiltin(inst, src, dst);
4752 }
4753
VisitIntrinsic(GraphVisitor * visitor,Inst * inst)4754 void EncodeVisitor::VisitIntrinsic(GraphVisitor *visitor, Inst *inst)
4755 {
4756 auto *enc = static_cast<EncodeVisitor *>(visitor);
4757 auto codegen = enc->GetCodegen();
4758 auto intrinsic = inst->CastToIntrinsic();
4759 auto id = intrinsic->GetIntrinsicId();
4760 auto arch = codegen->GetGraph()->GetArch();
4761 auto runtime = codegen->GetGraph()->GetRuntime();
4762 if (EncodesBuiltin(runtime, id, arch) || IsIrtocIntrinsic(id)) {
4763 codegen->CreateBuiltinIntrinsic(intrinsic);
4764 return;
4765 }
4766 codegen->CreateCallIntrinsic(intrinsic);
4767 }
4768
VisitBoundsCheckI(GraphVisitor * visitor,Inst * inst)4769 void EncodeVisitor::VisitBoundsCheckI(GraphVisitor *visitor, Inst *inst)
4770 {
4771 auto *enc = static_cast<EncodeVisitor *>(visitor);
4772 auto lenReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), inst->GetInputType(0));
4773
4774 ASSERT(inst->GetInput(1).GetInst()->GetOpcode() == Opcode::SaveState ||
4775 inst->GetInput(1).GetInst()->GetOpcode() == Opcode::SaveStateDeoptimize);
4776 EntrypointId entrypoint = inst->CastToBoundsCheckI()->IsArray()
4777 ? EntrypointId::ARRAY_INDEX_OUT_OF_BOUNDS_EXCEPTION
4778 : EntrypointId::STRING_INDEX_OUT_OF_BOUNDS_EXCEPTION;
4779 LabelHolder::LabelId label;
4780 if (inst->CanDeoptimize()) {
4781 label = enc->GetCodegen()->CreateSlowPath<SlowPathDeoptimize>(inst, DeoptimizeType::BOUNDS_CHECK)->GetLabel();
4782 } else {
4783 label = enc->GetCodegen()->CreateSlowPath<SlowPathCheck>(inst, entrypoint)->GetLabel();
4784 }
4785
4786 auto value = inst->CastToBoundsCheckI()->GetImm();
4787 if (enc->GetEncoder()->CanEncodeImmAddSubCmp(value, WORD_SIZE, false)) {
4788 enc->GetEncoder()->EncodeJump(label, lenReg, Imm(value), Condition::LS);
4789 } else {
4790 ScopedTmpReg tmp(enc->GetEncoder(), lenReg.GetType());
4791 enc->GetEncoder()->EncodeMov(tmp, Imm(value));
4792 enc->GetEncoder()->EncodeJump(label, lenReg, tmp, Condition::LS);
4793 }
4794 }
4795
VisitStoreArrayI(GraphVisitor * visitor,Inst * inst)4796 void EncodeVisitor::VisitStoreArrayI(GraphVisitor *visitor, Inst *inst)
4797 {
4798 auto *enc = static_cast<EncodeVisitor *>(visitor);
4799 auto arrayReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE);
4800 auto type = inst->GetType();
4801 auto value = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
4802
4803 auto index = inst->CastToStoreArrayI()->GetImm();
4804 int64_t offset = enc->cg_->GetGraph()->GetRuntime()->GetArrayDataOffset(enc->GetCodegen()->GetArch()) +
4805 (index << DataType::ShiftByType(type, enc->GetCodegen()->GetArch()));
4806 if (!enc->GetCodegen()->OffsetFitReferenceTypeSize(offset)) {
4807 // such code should not be executed
4808 enc->GetEncoder()->EncodeAbort();
4809 return;
4810 }
4811 auto mem = MemRef(arrayReg, offset);
4812 if (inst->CastToStoreArrayI()->GetNeedBarrier()) {
4813 enc->GetCodegen()->CreatePreWRB(inst, mem, MakeMask(arrayReg.GetId(), value.GetId()));
4814 }
4815 auto prevOffset = enc->GetEncoder()->GetCursorOffset();
4816 enc->GetEncoder()->EncodeStr(value, mem);
4817 enc->GetCodegen()->TryInsertImplicitNullCheck(inst, prevOffset);
4818 if (inst->CastToStoreArrayI()->GetNeedBarrier()) {
4819 enc->GetCodegen()->CreatePostWRB(inst, mem, value, INVALID_REGISTER);
4820 }
4821 }
4822
VisitLoadArrayI(GraphVisitor * visitor,Inst * inst)4823 void EncodeVisitor::VisitLoadArrayI(GraphVisitor *visitor, Inst *inst)
4824 {
4825 auto *enc = static_cast<EncodeVisitor *>(visitor);
4826 auto instLoadArrayI = inst->CastToLoadArrayI();
4827 auto runtime = enc->cg_->GetGraph()->GetRuntime();
4828 ASSERT(instLoadArrayI->IsArray() || !runtime->IsCompressedStringsEnabled());
4829 if (instLoadArrayI->GetNeedBarrier()) {
4830 // Consider inserting barriers for GC
4831 }
4832 auto arrayReg = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0));
4833 uint32_t index = instLoadArrayI->GetImm();
4834 auto type = inst->GetType();
4835 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type);
4836 int64_t dataOffset = instLoadArrayI->IsArray() ? runtime->GetArrayDataOffset(enc->GetArch())
4837 : runtime->GetStringDataOffset(enc->GetArch());
4838 uint32_t shift = DataType::ShiftByType(type, enc->GetArch());
4839 int64_t offset = dataOffset + (index << shift);
4840 auto mem = MemRef(arrayReg, offset);
4841 auto encoder = enc->GetEncoder();
4842 auto arch = enc->GetArch();
4843 ScopedTmpReg scopedTmp(encoder, Codegen::ConvertDataType(DataType::GetIntTypeForReference(arch), arch));
4844 auto prevOffset = enc->GetEncoder()->GetCursorOffset();
4845 encoder->EncodeLdr(dst, IsTypeSigned(type), mem);
4846 enc->GetCodegen()->TryInsertImplicitNullCheck(inst, prevOffset);
4847 }
4848
VisitLoadCompressedStringCharI(GraphVisitor * visitor,Inst * inst)4849 void EncodeVisitor::VisitLoadCompressedStringCharI(GraphVisitor *visitor, Inst *inst)
4850 {
4851 auto *enc = static_cast<EncodeVisitor *>(visitor);
4852 auto runtime = enc->cg_->GetGraph()->GetRuntime();
4853 auto type = inst->GetType();
4854 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0)); // array
4855 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::INT32); // length
4856 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), type); // load value
4857 int32_t offset = runtime->GetStringDataOffset(enc->GetArch());
4858 auto encoder = enc->GetEncoder();
4859 auto arch = encoder->GetArch();
4860 int32_t shift = DataType::ShiftByType(type, arch);
4861 auto index = inst->CastToLoadCompressedStringCharI()->GetImm();
4862 ASSERT(encoder->CanEncodeCompressedStringCharAt());
4863 auto mask = runtime->GetStringCompressionMask();
4864 if (mask != 1) {
4865 UNREACHABLE(); // mask is hardcoded in JCL, but verify it just in case it's changed
4866 }
4867 enc->GetEncoder()->EncodeCompressedStringCharAtI(dst, src0, src1, offset, index, shift);
4868 }
4869
VisitMultiArray(GraphVisitor * visitor,Inst * inst)4870 void EncodeVisitor::VisitMultiArray(GraphVisitor *visitor, Inst *inst)
4871 {
4872 auto *enc = static_cast<EncodeVisitor *>(visitor);
4873 auto codegen = enc->GetCodegen();
4874
4875 auto arrayInst = inst->CastToMultiArray();
4876 codegen->CreateMultiArrayCall(arrayInst);
4877 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
4878 enc->GetEncoder()->EncodeMemoryBarrier(memory_order::RELEASE);
4879 }
4880 }
4881
VisitInitEmptyString(GraphVisitor * visitor,Inst * inst)4882 void EncodeVisitor::VisitInitEmptyString(GraphVisitor *visitor, Inst *inst)
4883 {
4884 auto codegen = static_cast<EncodeVisitor *>(visitor)->GetCodegen();
4885 auto dst = codegen->ConvertRegister(inst->GetDstReg(), inst->GetType());
4886 codegen->CallRuntime(inst, EntrypointId::CREATE_EMPTY_STRING, dst, RegMask::GetZeroMask());
4887 }
4888
VisitInitString(GraphVisitor * visitor,Inst * inst)4889 void EncodeVisitor::VisitInitString(GraphVisitor *visitor, Inst *inst)
4890 {
4891 auto cg = static_cast<EncodeVisitor *>(visitor)->GetCodegen();
4892 auto initStr = inst->CastToInitString();
4893
4894 auto dst = cg->ConvertRegister(initStr->GetDstReg(), initStr->GetType());
4895 auto ctorArg = cg->ConvertRegister(initStr->GetSrcReg(0), initStr->GetInputType(0));
4896
4897 if (cg->GetArch() == Arch::AARCH32) {
4898 auto entryId =
4899 initStr->IsFromString() ? EntrypointId::CREATE_STRING_FROM_STRING : EntrypointId::CREATE_STRING_FROM_CHARS;
4900 cg->CallRuntime(initStr, entryId, dst, RegMask::GetZeroMask(), ctorArg);
4901 return;
4902 }
4903
4904 if (initStr->IsFromString()) {
4905 auto entryId = cg->GetRuntime()->IsCompressedStringsEnabled()
4906 ? compiler::RuntimeInterface::EntrypointId::CREATE_STRING_FROM_STRING_TLAB_COMPRESSED
4907 : compiler::RuntimeInterface::EntrypointId::CREATE_STRING_FROM_STRING_TLAB;
4908 cg->CallFastPath(initStr, entryId, dst, RegMask::GetZeroMask(), ctorArg);
4909 } else {
4910 auto enc = cg->GetEncoder();
4911 auto runtime = cg->GetGraph()->GetRuntime();
4912 auto mem = MemRef(ctorArg, static_cast<int64_t>(runtime->GetArrayLengthOffset(cg->GetArch())));
4913 ScopedTmpReg lengthReg(enc);
4914 enc->EncodeLdr(lengthReg, IsTypeSigned(initStr->GetType()), mem);
4915 if (cg->GetGraph()->IsAotMode()) {
4916 auto klassOffs = runtime->GetStringClassPointerTlsOffset(cg->GetArch());
4917 ScopedTmpReg klassReg(enc);
4918 enc->EncodeLdr(klassReg, false, MemRef(cg->ThreadReg(), klassOffs));
4919 cg->CallFastCreateStringFromCharArrayTlab(initStr, dst, cg->GetRegfile()->GetZeroReg(), lengthReg, ctorArg,
4920 klassReg);
4921 } else {
4922 auto klassPtr = runtime->GetStringClass(cg->GetGraph()->GetMethod());
4923 auto klassImm = TypedImm(reinterpret_cast<uintptr_t>(klassPtr));
4924 cg->CallFastCreateStringFromCharArrayTlab(initStr, dst, cg->GetRegfile()->GetZeroReg(), lengthReg, ctorArg,
4925 klassImm);
4926 }
4927 }
4928 }
4929
VisitCallLaunchStatic(GraphVisitor * visitor,Inst * inst)4930 void EncodeVisitor::VisitCallLaunchStatic(GraphVisitor *visitor, Inst *inst)
4931 {
4932 auto *enc = static_cast<EncodeVisitor *>(visitor);
4933 enc->GetCodegen()->CreateLaunchCall(inst->CastToCallLaunchStatic());
4934 }
4935
VisitCallLaunchVirtual(GraphVisitor * visitor,Inst * inst)4936 void EncodeVisitor::VisitCallLaunchVirtual(GraphVisitor *visitor, [[maybe_unused]] Inst *inst)
4937 {
4938 auto *enc = static_cast<EncodeVisitor *>(visitor);
4939 enc->GetCodegen()->CreateLaunchCall(inst->CastToCallLaunchVirtual());
4940 }
4941
VisitCallResolvedLaunchStatic(GraphVisitor * visitor,Inst * inst)4942 void EncodeVisitor::VisitCallResolvedLaunchStatic(GraphVisitor *visitor, Inst *inst)
4943 {
4944 auto *enc = static_cast<EncodeVisitor *>(visitor);
4945 enc->GetCodegen()->CreateLaunchCall(inst->CastToCallResolvedLaunchStatic());
4946 }
4947
VisitCallResolvedLaunchVirtual(GraphVisitor * visitor,Inst * inst)4948 void EncodeVisitor::VisitCallResolvedLaunchVirtual(GraphVisitor *visitor, Inst *inst)
4949 {
4950 auto *enc = static_cast<EncodeVisitor *>(visitor);
4951 enc->GetCodegen()->CreateLaunchCall(inst->CastToCallResolvedLaunchVirtual());
4952 }
4953
VisitResolveStatic(GraphVisitor * visitor,Inst * inst)4954 void EncodeVisitor::VisitResolveStatic(GraphVisitor *visitor, Inst *inst)
4955 {
4956 auto *enc = static_cast<EncodeVisitor *>(visitor);
4957 enc->GetCodegen()->EmitResolveStatic(inst->CastToResolveStatic());
4958 }
4959
VisitCallResolvedStatic(GraphVisitor * visitor,Inst * inst)4960 void EncodeVisitor::VisitCallResolvedStatic(GraphVisitor *visitor, Inst *inst)
4961 {
4962 auto *enc = static_cast<EncodeVisitor *>(visitor);
4963 enc->GetCodegen()->EmitCallResolvedStatic(inst->CastToCallResolvedStatic());
4964 }
4965
VisitCallStatic(GraphVisitor * visitor,Inst * inst)4966 void EncodeVisitor::VisitCallStatic(GraphVisitor *visitor, Inst *inst)
4967 {
4968 auto *enc = static_cast<EncodeVisitor *>(visitor);
4969 enc->GetCodegen()->EmitCallStatic(inst->CastToCallStatic());
4970 }
4971
VisitCallVirtual(GraphVisitor * visitor,Inst * inst)4972 void EncodeVisitor::VisitCallVirtual(GraphVisitor *visitor, [[maybe_unused]] Inst *inst)
4973 {
4974 auto *enc = static_cast<EncodeVisitor *>(visitor);
4975 enc->GetCodegen()->EmitCallVirtual(inst->CastToCallVirtual());
4976 }
4977
VisitResolveVirtual(GraphVisitor * visitor,Inst * inst)4978 void EncodeVisitor::VisitResolveVirtual(GraphVisitor *visitor, Inst *inst)
4979 {
4980 auto *enc = static_cast<EncodeVisitor *>(visitor);
4981 enc->GetCodegen()->EmitResolveVirtual(inst->CastToResolveVirtual());
4982 }
4983
VisitCallResolvedVirtual(GraphVisitor * visitor,Inst * inst)4984 void EncodeVisitor::VisitCallResolvedVirtual(GraphVisitor *visitor, Inst *inst)
4985 {
4986 auto *enc = static_cast<EncodeVisitor *>(visitor);
4987 enc->GetCodegen()->EmitCallResolvedVirtual(inst->CastToCallResolvedVirtual());
4988 }
4989
VisitCallDynamic(GraphVisitor * visitor,Inst * inst)4990 void EncodeVisitor::VisitCallDynamic(GraphVisitor *visitor, Inst *inst)
4991 {
4992 auto *enc = static_cast<EncodeVisitor *>(visitor);
4993 enc->GetCodegen()->EmitCallDynamic(inst->CastToCallDynamic());
4994 }
4995
VisitLoadConstantPool(GraphVisitor * visitor,Inst * inst)4996 void EncodeVisitor::VisitLoadConstantPool(GraphVisitor *visitor, Inst *inst)
4997 {
4998 if (TryLoadConstantPoolGen(inst, static_cast<EncodeVisitor *>(visitor))) {
4999 return;
5000 }
5001 UNREACHABLE();
5002 }
5003
VisitLoadLexicalEnv(GraphVisitor * visitor,Inst * inst)5004 void EncodeVisitor::VisitLoadLexicalEnv(GraphVisitor *visitor, Inst *inst)
5005 {
5006 if (TryLoadLexicalEnvGen(inst, static_cast<EncodeVisitor *>(visitor))) {
5007 return;
5008 }
5009 UNREACHABLE();
5010 }
5011
VisitLoadFromConstantPool(GraphVisitor * visitor,Inst * inst)5012 void EncodeVisitor::VisitLoadFromConstantPool([[maybe_unused]] GraphVisitor *visitor, [[maybe_unused]] Inst *inst)
5013 {
5014 UNREACHABLE();
5015 }
5016
VisitSafePoint(GraphVisitor * visitor,Inst * inst)5017 void EncodeVisitor::VisitSafePoint(GraphVisitor *visitor, Inst *inst)
5018 {
5019 auto *enc = static_cast<EncodeVisitor *>(visitor);
5020 auto codegen = enc->GetCodegen();
5021 auto graph = codegen->GetGraph();
5022 auto encoder = enc->GetEncoder();
5023 int64_t flagAddrOffset = graph->GetRuntime()->GetFlagAddrOffset(codegen->GetArch());
5024 ScopedTmpRegU16 tmp(encoder);
5025
5026 // TMP <= Flag
5027 auto mem = MemRef(codegen->ThreadReg(), flagAddrOffset);
5028 encoder->EncodeLdr(tmp, false, mem);
5029
5030 // check value and jump to call GC
5031 auto slowPath = codegen->CreateSlowPath<SlowPathEntrypoint>(inst, EntrypointId::SAFEPOINT);
5032
5033 encoder->EncodeJump(slowPath->GetLabel(), tmp, Condition::NE);
5034
5035 slowPath->BindBackLabel(encoder);
5036 }
5037
VisitSelect(GraphVisitor * visitor,Inst * inst)5038 void EncodeVisitor::VisitSelect(GraphVisitor *visitor, Inst *inst)
5039 {
5040 auto *enc = static_cast<EncodeVisitor *>(visitor);
5041 auto dstType = inst->GetType();
5042 auto cmpType = inst->CastToSelect()->GetOperandsType();
5043
5044 constexpr int32_t IMM_2 = 2;
5045 constexpr int32_t IMM_3 = 3;
5046 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), dstType);
5047 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), dstType);
5048 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), dstType);
5049 auto src2 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(IMM_2), cmpType);
5050 auto src3 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(IMM_3), cmpType);
5051 auto cc = enc->GetCodegen()->ConvertCc(inst->CastToSelect()->GetCc());
5052 if (IsTestCc(cc)) {
5053 enc->GetEncoder()->EncodeSelectTest(dst, src0, src1, src2, src3, cc);
5054 } else {
5055 enc->GetEncoder()->EncodeSelect(dst, src0, src1, src2, src3, cc);
5056 }
5057 }
5058
VisitSelectImm(GraphVisitor * visitor,Inst * inst)5059 void EncodeVisitor::VisitSelectImm(GraphVisitor *visitor, Inst *inst)
5060 {
5061 auto *enc = static_cast<EncodeVisitor *>(visitor);
5062 auto dstType = inst->GetType();
5063 auto cmpType = inst->CastToSelectImm()->GetOperandsType();
5064
5065 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), dstType);
5066 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), dstType);
5067 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), dstType);
5068 constexpr int32_t IMM_2 = 2;
5069 auto src2 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(IMM_2), cmpType);
5070 auto imm = Imm(inst->CastToSelectImm()->GetImm());
5071 auto cc = enc->GetCodegen()->ConvertCc(inst->CastToSelectImm()->GetCc());
5072 if (IsTestCc(cc)) {
5073 enc->GetEncoder()->EncodeSelectTest(dst, src0, src1, src2, imm, cc);
5074 } else {
5075 enc->GetEncoder()->EncodeSelect(dst, src0, src1, src2, imm, cc);
5076 }
5077 }
5078
VisitIf(GraphVisitor * visitor,Inst * inst)5079 void EncodeVisitor::VisitIf(GraphVisitor *visitor, Inst *inst)
5080 {
5081 auto *enc = static_cast<EncodeVisitor *>(visitor);
5082
5083 auto bb = inst->GetBasicBlock();
5084 auto label = bb->GetTrueSuccessor()->GetId();
5085
5086 auto type = inst->CastToIf()->GetOperandsType();
5087 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
5088 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
5089 auto cc = enc->GetCodegen()->ConvertCc(inst->CastToIf()->GetCc());
5090 if (IsTestCc(cc)) {
5091 enc->GetEncoder()->EncodeJumpTest(label, src0, src1, cc);
5092 } else {
5093 enc->GetEncoder()->EncodeJump(label, src0, src1, cc);
5094 }
5095 }
5096
VisitIfImm(GraphVisitor * visitor,Inst * inst)5097 void EncodeVisitor::VisitIfImm(GraphVisitor *visitor, Inst *inst)
5098 {
5099 auto *enc = static_cast<EncodeVisitor *>(visitor);
5100
5101 auto bb = inst->GetBasicBlock();
5102 auto label = bb->GetTrueSuccessor()->GetId();
5103
5104 auto type = inst->CastToIfImm()->GetOperandsType();
5105 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
5106 auto imm = Imm(inst->CastToIfImm()->GetImm());
5107 auto cc = enc->GetCodegen()->ConvertCc(inst->CastToIfImm()->GetCc());
5108 if (IsTestCc(cc)) {
5109 enc->GetEncoder()->EncodeJumpTest(label, src0, imm, cc);
5110 } else {
5111 enc->GetEncoder()->EncodeJump(label, src0, imm, cc);
5112 }
5113 }
5114
VisitAddOverflow(GraphVisitor * visitor,Inst * inst)5115 void EncodeVisitor::VisitAddOverflow(GraphVisitor *visitor, Inst *inst)
5116 {
5117 auto *enc = static_cast<EncodeVisitor *>(visitor);
5118
5119 auto bb = inst->GetBasicBlock();
5120 auto label = bb->GetTrueSuccessor()->GetId();
5121
5122 auto type = inst->CastToAddOverflow()->GetOperandsType();
5123 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
5124 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
5125 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
5126 auto cc = enc->GetCodegen()->ConvertCcOverflow(inst->CastToAddOverflow()->GetCc());
5127 enc->GetEncoder()->EncodeAddOverflow(label, dst, src0, src1, cc);
5128 }
5129
VisitAddOverflowCheck(GraphVisitor * visitor,Inst * inst)5130 void EncodeVisitor::VisitAddOverflowCheck(GraphVisitor *visitor, Inst *inst)
5131 {
5132 ASSERT(DataType::IsTypeNumeric(inst->GetInput(0).GetInst()->GetType()));
5133 ASSERT(DataType::IsTypeNumeric(inst->GetInput(1).GetInst()->GetType()));
5134 auto *enc = static_cast<EncodeVisitor *>(visitor);
5135 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathDeoptimize>(inst, DeoptimizeType::OVERFLOW);
5136 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
5137 auto src0 = Reg(inst->GetSrcReg(0), INT32_TYPE);
5138 auto src1 = Reg(inst->GetSrcReg(1), INT32_TYPE);
5139 enc->GetEncoder()->EncodeAddOverflow(slowPath->GetLabel(), dst, src0, src1, Condition::VS);
5140 }
5141
VisitSubOverflow(GraphVisitor * visitor,Inst * inst)5142 void EncodeVisitor::VisitSubOverflow(GraphVisitor *visitor, Inst *inst)
5143 {
5144 auto *enc = static_cast<EncodeVisitor *>(visitor);
5145
5146 auto bb = inst->GetBasicBlock();
5147 auto label = bb->GetTrueSuccessor()->GetId();
5148
5149 auto type = inst->CastToSubOverflow()->GetOperandsType();
5150 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
5151 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), type);
5152 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type);
5153 auto cc = enc->GetCodegen()->ConvertCcOverflow(inst->CastToSubOverflow()->GetCc());
5154 enc->GetEncoder()->EncodeSubOverflow(label, dst, src0, src1, cc);
5155 }
5156
VisitSubOverflowCheck(GraphVisitor * visitor,Inst * inst)5157 void EncodeVisitor::VisitSubOverflowCheck(GraphVisitor *visitor, Inst *inst)
5158 {
5159 ASSERT(DataType::IsTypeNumeric(inst->GetInput(0).GetInst()->GetType()));
5160 ASSERT(DataType::IsTypeNumeric(inst->GetInput(1).GetInst()->GetType()));
5161 auto *enc = static_cast<EncodeVisitor *>(visitor);
5162 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathDeoptimize>(inst, DeoptimizeType::OVERFLOW);
5163 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
5164 auto src0 = Reg(inst->GetSrcReg(0), INT32_TYPE);
5165 auto src1 = Reg(inst->GetSrcReg(1), INT32_TYPE);
5166 enc->GetEncoder()->EncodeSubOverflow(slowPath->GetLabel(), dst, src0, src1, Condition::VS);
5167 }
5168
VisitNegOverflowAndZeroCheck(GraphVisitor * visitor,Inst * inst)5169 void EncodeVisitor::VisitNegOverflowAndZeroCheck(GraphVisitor *visitor, Inst *inst)
5170 {
5171 ASSERT(DataType::IsTypeNumeric(inst->GetInput(0).GetInst()->GetType()));
5172 auto *enc = static_cast<EncodeVisitor *>(visitor);
5173 auto slowPath = enc->GetCodegen()->CreateSlowPath<SlowPathDeoptimize>(inst, DeoptimizeType::OVERFLOW);
5174 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
5175 auto src = Reg(inst->GetSrcReg(0), INT32_TYPE);
5176 enc->GetEncoder()->EncodeNegOverflowAndZero(slowPath->GetLabel(), dst, src);
5177 }
5178
VisitLoadArrayPair(GraphVisitor * visitor,Inst * inst)5179 void EncodeVisitor::VisitLoadArrayPair(GraphVisitor *visitor, Inst *inst)
5180 {
5181 auto *enc = static_cast<EncodeVisitor *>(visitor);
5182 if (inst->CastToLoadArrayPair()->GetNeedBarrier()) {
5183 // Consider inserting barriers for GC
5184 }
5185 auto type = inst->GetType();
5186 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // array
5187 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::INT32); // index
5188 auto dst0 = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(0), type); // first value
5189 auto dst1 = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(1), type); // second value
5190 uint64_t index = inst->CastToLoadArrayPair()->GetImm();
5191 int64_t offset = enc->cg_->GetGraph()->GetRuntime()->GetArrayDataOffset(enc->GetCodegen()->GetArch()) +
5192 (index << DataType::ShiftByType(type, enc->GetCodegen()->GetArch()));
5193 ScopedTmpReg tmp(enc->GetEncoder());
5194
5195 int32_t scale = DataType::ShiftByType(type, enc->GetCodegen()->GetArch());
5196 enc->GetEncoder()->EncodeAdd(tmp, src0, Shift(src1, scale));
5197 auto mem = MemRef(tmp, offset);
5198 auto prevOffset = enc->GetEncoder()->GetCursorOffset();
5199 enc->GetEncoder()->EncodeLdp(dst0, dst1, IsTypeSigned(type), mem);
5200 enc->GetCodegen()->TryInsertImplicitNullCheck(inst, prevOffset);
5201 }
5202
VisitLoadArrayPairI(GraphVisitor * visitor,Inst * inst)5203 void EncodeVisitor::VisitLoadArrayPairI(GraphVisitor *visitor, Inst *inst)
5204 {
5205 auto *enc = static_cast<EncodeVisitor *>(visitor);
5206 if (inst->CastToLoadArrayPairI()->GetNeedBarrier()) {
5207 // Consider inserting barriers for GC
5208 }
5209 auto type = inst->GetType();
5210 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // array
5211 auto dst0 = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(0), type); // first value
5212 auto dst1 = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(1), type); // second value
5213 uint64_t index = inst->CastToLoadArrayPairI()->GetImm();
5214 int64_t offset = enc->cg_->GetGraph()->GetRuntime()->GetArrayDataOffset(enc->GetCodegen()->GetArch()) +
5215 (index << DataType::ShiftByType(type, enc->GetCodegen()->GetArch()));
5216 auto mem = MemRef(src0, offset);
5217
5218 auto prevOffset = enc->GetEncoder()->GetCursorOffset();
5219 enc->GetEncoder()->EncodeLdp(dst0, dst1, IsTypeSigned(type), mem);
5220 enc->GetCodegen()->TryInsertImplicitNullCheck(inst, prevOffset);
5221 }
5222
5223 /**
5224 * It is a pseudo instruction that is needed to separate multiple outputs from a single instruction in SSA such as
5225 * LoadArrayPair and LoadArrayPairI. No codegeneration is required.
5226 */
VisitLoadPairPart(GraphVisitor * visitor,Inst * inst)5227 void EncodeVisitor::VisitLoadPairPart([[maybe_unused]] GraphVisitor *visitor, [[maybe_unused]] Inst *inst) {}
5228
VisitStoreArrayPair(GraphVisitor * visitor,Inst * inst)5229 void EncodeVisitor::VisitStoreArrayPair(GraphVisitor *visitor, Inst *inst)
5230 {
5231 auto *enc = static_cast<EncodeVisitor *>(visitor);
5232 auto type = inst->GetType();
5233 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // array
5234 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), DataType::INT32); // index
5235 constexpr int32_t IMM_2 = 2;
5236 auto src2 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(IMM_2), type); // first value
5237 constexpr int32_t IMM_3 = 3;
5238 auto src3 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(IMM_3), type); // second value
5239 uint64_t index = inst->CastToStoreArrayPair()->GetImm();
5240 int64_t offset = enc->cg_->GetGraph()->GetRuntime()->GetArrayDataOffset(enc->GetCodegen()->GetArch()) +
5241 (index << DataType::ShiftByType(type, enc->GetCodegen()->GetArch()));
5242 int32_t scale = DataType::ShiftByType(type, enc->GetCodegen()->GetArch());
5243
5244 auto tmp = enc->GetCodegen()->ConvertInstTmpReg(inst, DataType::REFERENCE);
5245 enc->GetEncoder()->EncodeAdd(tmp, src0, Shift(src1, scale));
5246 auto mem = MemRef(tmp, offset);
5247 if (inst->CastToStoreArrayPair()->GetNeedBarrier()) {
5248 enc->GetCodegen()->CreatePreWRB(inst, mem, MakeMask(src0.GetId(), src2.GetId(), src3.GetId()), true);
5249 }
5250 auto prevOffset = enc->GetEncoder()->GetCursorOffset();
5251 enc->GetEncoder()->EncodeStp(src2, src3, mem);
5252 enc->GetCodegen()->TryInsertImplicitNullCheck(inst, prevOffset);
5253
5254 if (inst->CastToStoreArrayPair()->GetNeedBarrier()) {
5255 auto tmpOffset = enc->GetCodegen()->ConvertInstTmpReg(inst, DataType::GetIntTypeForReference(enc->GetArch()));
5256 enc->GetEncoder()->EncodeShl(tmpOffset, src1, Imm(scale));
5257 enc->GetEncoder()->EncodeAdd(tmpOffset, tmpOffset, Imm(offset));
5258 auto mem1 = MemRef(src0, tmpOffset, 0);
5259 enc->GetCodegen()->CreatePostWRB(inst, mem1, src2, src3);
5260 }
5261 }
5262
VisitStoreArrayPairI(GraphVisitor * visitor,Inst * inst)5263 void EncodeVisitor::VisitStoreArrayPairI(GraphVisitor *visitor, Inst *inst)
5264 {
5265 auto *enc = static_cast<EncodeVisitor *>(visitor);
5266 auto type = inst->GetType();
5267 auto src0 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // array
5268 auto src1 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(1), type); // first value
5269 constexpr int32_t IMM_2 = 2;
5270 auto src2 = enc->GetCodegen()->ConvertRegister(inst->GetSrcReg(IMM_2), type); // second value
5271
5272 auto index = inst->CastToStoreArrayPairI()->GetImm();
5273 int64_t offset = enc->cg_->GetGraph()->GetRuntime()->GetArrayDataOffset(enc->GetCodegen()->GetArch()) +
5274 (index << DataType::ShiftByType(type, enc->GetCodegen()->GetArch()));
5275 if (!enc->GetCodegen()->OffsetFitReferenceTypeSize(offset)) {
5276 // such code should not be executed
5277 enc->GetEncoder()->EncodeAbort();
5278 return;
5279 }
5280 auto mem = MemRef(src0, offset);
5281 if (inst->CastToStoreArrayPairI()->GetNeedBarrier()) {
5282 enc->GetCodegen()->CreatePreWRB(inst, mem, MakeMask(src0.GetId(), src1.GetId(), src2.GetId()), true);
5283 }
5284 auto prevOffset = enc->GetEncoder()->GetCursorOffset();
5285 enc->GetEncoder()->EncodeStp(src1, src2, mem);
5286 enc->GetCodegen()->TryInsertImplicitNullCheck(inst, prevOffset);
5287 if (inst->CastToStoreArrayPairI()->GetNeedBarrier()) {
5288 enc->GetCodegen()->CreatePostWRB(inst, mem, src1, src2);
5289 }
5290 }
5291
VisitNOP(GraphVisitor * visitor,Inst * inst)5292 void EncodeVisitor::VisitNOP([[maybe_unused]] GraphVisitor *visitor, [[maybe_unused]] Inst *inst)
5293 {
5294 #ifndef NDEBUG
5295 COMPILER_LOG(DEBUG, CODEGEN) << "The NOP wasn't removed before " << *inst;
5296 #endif
5297 }
5298
VisitThrow(GraphVisitor * visitor,Inst * inst)5299 void EncodeVisitor::VisitThrow(GraphVisitor *visitor, [[maybe_unused]] Inst *inst)
5300 {
5301 auto codegen = static_cast<EncodeVisitor *>(visitor)->GetCodegen();
5302 SlowPathCheck slowPath(codegen->GetEncoder()->CreateLabel(), inst, EntrypointId::THROW_EXCEPTION);
5303 slowPath.Generate(codegen);
5304 }
5305
VisitDeoptimize(GraphVisitor * visitor,Inst * inst)5306 void EncodeVisitor::VisitDeoptimize(GraphVisitor *visitor, Inst *inst)
5307 {
5308 auto codegen = static_cast<EncodeVisitor *>(visitor)->GetCodegen();
5309 ASSERT(inst->GetSaveState() != nullptr);
5310
5311 SlowPathDeoptimize slowPath(codegen->GetEncoder()->CreateLabel(), inst,
5312 inst->CastToDeoptimize()->GetDeoptimizeType());
5313 slowPath.Generate(codegen);
5314 }
5315
VisitIsMustDeoptimize(GraphVisitor * visitor,Inst * inst)5316 void EncodeVisitor::VisitIsMustDeoptimize(GraphVisitor *visitor, Inst *inst)
5317 {
5318 auto *codegen = static_cast<EncodeVisitor *>(visitor)->GetCodegen();
5319 auto *enc = static_cast<EncodeVisitor *>(visitor)->GetEncoder();
5320 auto dst = codegen->ConvertRegister(inst->GetDstReg(), inst->GetType());
5321 auto offset = CFrameFlags::GetOffsetFromSpInBytes(codegen->GetFrameLayout());
5322
5323 enc->EncodeLdr(dst, false, MemRef(codegen->SpReg(), offset));
5324 enc->EncodeAnd(dst, dst, Imm(1));
5325 }
5326
VisitGetInstanceClass(GraphVisitor * visitor,Inst * inst)5327 void EncodeVisitor::VisitGetInstanceClass(GraphVisitor *visitor, Inst *inst)
5328 {
5329 auto *codegen = static_cast<EncodeVisitor *>(visitor)->GetCodegen();
5330 auto dst = codegen->ConvertRegister(inst->GetDstReg(), inst->GetType());
5331 auto objReg = codegen->ConvertRegister(inst->GetSrcReg(0), inst->GetType());
5332 ASSERT(objReg.IsValid());
5333 codegen->LoadClassFromObject(dst, objReg);
5334 }
5335
VisitLoadImmediate(GraphVisitor * visitor,Inst * inst)5336 void EncodeVisitor::VisitLoadImmediate(GraphVisitor *visitor, Inst *inst)
5337 {
5338 auto codegen = static_cast<EncodeVisitor *>(visitor)->GetCodegen();
5339 auto dst = codegen->ConvertRegister(inst->GetDstReg(), inst->GetType());
5340 auto loadImm = inst->CastToLoadImmediate();
5341 if (loadImm->GetObjectType() != LoadImmediateInst::ObjectType::PANDA_FILE_OFFSET) {
5342 codegen->GetEncoder()->EncodeMov(dst, Imm(reinterpret_cast<uintptr_t>(loadImm->GetObject())));
5343 } else {
5344 auto runtime = codegen->GetGraph()->GetRuntime();
5345 auto pfOffset = runtime->GetPandaFileOffset(codegen->GetGraph()->GetArch());
5346 codegen->LoadMethod(dst);
5347 // load pointer to panda file
5348 codegen->GetEncoder()->EncodeLdr(dst, false, MemRef(dst, pfOffset));
5349 codegen->GetEncoder()->EncodeLdr(dst, false, MemRef(dst, cross_values::GetFileBaseOffset(codegen->GetArch())));
5350 // add pointer to pf with offset to str
5351 codegen->GetEncoder()->EncodeAdd(dst, dst, Imm(loadImm->GetPandaFileOffset()));
5352 }
5353 }
5354
VisitFunctionImmediate(GraphVisitor * visitor,Inst * inst)5355 void EncodeVisitor::VisitFunctionImmediate(GraphVisitor *visitor, Inst *inst)
5356 {
5357 auto *codegen = static_cast<EncodeVisitor *>(visitor)->GetCodegen();
5358 auto dst = codegen->ConvertRegister(inst->GetDstReg(), inst->GetType());
5359
5360 codegen->GetEncoder()->EncodeMov(dst, Imm(inst->CastToFunctionImmediate()->GetFunctionPtr()));
5361 codegen->GetEncoder()->EncodeLdr(dst, false, MemRef(dst, 0U));
5362 }
5363
VisitLoadObjFromConst(GraphVisitor * visitor,Inst * inst)5364 void EncodeVisitor::VisitLoadObjFromConst(GraphVisitor *visitor, Inst *inst)
5365 {
5366 auto *codegen = static_cast<EncodeVisitor *>(visitor)->GetCodegen();
5367 auto dst = codegen->ConvertRegister(inst->GetDstReg(), inst->GetType());
5368
5369 Reg dstPointer(dst.As(TypeInfo::FromDataType(DataType::POINTER, codegen->GetArch())));
5370 codegen->GetEncoder()->EncodeMov(dstPointer, Imm(inst->CastToLoadObjFromConst()->GetObjPtr()));
5371 codegen->GetEncoder()->EncodeLdr(dst, false, MemRef(dstPointer, 0U));
5372 }
5373
VisitRegDef(GraphVisitor * visitor,Inst * inst)5374 void EncodeVisitor::VisitRegDef([[maybe_unused]] GraphVisitor *visitor, [[maybe_unused]] Inst *inst) {}
5375
VisitLiveIn(GraphVisitor * visitor,Inst * inst)5376 void EncodeVisitor::VisitLiveIn([[maybe_unused]] GraphVisitor *visitor, [[maybe_unused]] Inst *inst) {}
5377
VisitLiveOut(GraphVisitor * visitor,Inst * inst)5378 void EncodeVisitor::VisitLiveOut([[maybe_unused]] GraphVisitor *visitor, [[maybe_unused]] Inst *inst)
5379 {
5380 auto enc = static_cast<EncodeVisitor *>(visitor);
5381 auto codegen = enc->GetCodegen();
5382
5383 codegen->AddLiveOut(inst->GetBasicBlock(), inst->GetDstReg());
5384
5385 auto dstReg = codegen->ConvertRegister(inst->GetDstReg(), inst->GetType());
5386 if (codegen->GetTarget().GetTempRegsMask().Test(dstReg.GetId()) &&
5387 enc->GetEncoder()->IsScratchRegisterReleased(dstReg)) {
5388 enc->GetEncoder()->AcquireScratchRegister(dstReg);
5389 }
5390
5391 if (inst->GetLocation(0) != inst->GetDstLocation()) {
5392 auto *src = inst->GetInput(0).GetInst();
5393 enc->GetEncoder()->EncodeMov(dstReg, codegen->ConvertRegister(inst->GetSrcReg(0), src->GetType()));
5394 }
5395 }
5396
VisitCompareAnyType(GraphVisitor * visitor,Inst * inst)5397 void EncodeVisitor::VisitCompareAnyType(GraphVisitor *visitor, Inst *inst)
5398 {
5399 auto enc = static_cast<EncodeVisitor *>(visitor);
5400 const auto *cati = inst->CastToCompareAnyType();
5401
5402 if (cati->GetInputType(0) != DataType::Type::ANY) {
5403 enc->GetEncoder()->EncodeAbort();
5404 UNREACHABLE();
5405 return;
5406 }
5407
5408 if (TryCompareAnyTypePluginGen(cati, enc)) {
5409 return;
5410 }
5411
5412 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), inst->GetType());
5413 if (cati->GetAnyType() == AnyBaseType::UNDEFINED_TYPE) {
5414 enc->GetEncoder()->EncodeMov(dst, Imm(true));
5415 } else {
5416 enc->GetEncoder()->EncodeMov(dst, Imm(false));
5417 }
5418 }
5419
VisitGetAnyTypeName(GraphVisitor * visitor,Inst * inst)5420 void EncodeVisitor::VisitGetAnyTypeName(GraphVisitor *visitor, Inst *inst)
5421 {
5422 auto enc = static_cast<EncodeVisitor *>(visitor);
5423 const auto *cati = inst->CastToGetAnyTypeName();
5424
5425 if (TryGetAnyTypeNamePluginGen(cati, enc)) {
5426 return;
5427 }
5428
5429 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), DataType::Type::BOOL);
5430 enc->GetEncoder()->EncodeMov(dst, Imm(0));
5431 }
5432
VisitCastAnyTypeValue(GraphVisitor * visitor,Inst * inst)5433 void EncodeVisitor::VisitCastAnyTypeValue(GraphVisitor *visitor, Inst *inst)
5434 {
5435 auto enc = static_cast<EncodeVisitor *>(visitor);
5436 const auto *cati = inst->CastToCastAnyTypeValue();
5437
5438 if (cati->GetInputType(0) != DataType::Type::ANY) {
5439 enc->GetEncoder()->EncodeAbort();
5440 UNREACHABLE();
5441 return;
5442 }
5443
5444 if (TryCastAnyTypeValuePluginGen(cati, enc)) {
5445 return;
5446 }
5447
5448 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), DataType::Type::BOOL);
5449 enc->GetEncoder()->EncodeMov(dst, Imm(0));
5450 }
5451
VisitCastValueToAnyType(GraphVisitor * visitor,Inst * inst)5452 void EncodeVisitor::VisitCastValueToAnyType(GraphVisitor *visitor, Inst *inst)
5453 {
5454 auto enc = static_cast<EncodeVisitor *>(visitor);
5455 const auto *cvai = inst->CastToCastValueToAnyType();
5456
5457 /* // NOTE(vpukhov): AnyConst
5458 if (cvai->GetInputType(0) == DataType::Type::ANY) {
5459 enc->GetEncoder()->EncodeAbort();
5460 UNREACHABLE();
5461 return;
5462 }
5463 */
5464
5465 if (TryCastValueToAnyTypePluginGen(cvai, enc)) {
5466 return;
5467 }
5468
5469 auto dst = enc->GetCodegen()->ConvertRegister(inst->GetDstReg(), DataType::Type::BOOL);
5470 enc->GetEncoder()->EncodeMov(dst, Imm(0));
5471 }
5472
VisitAnyTypeCheck(GraphVisitor * visitor,Inst * inst)5473 void EncodeVisitor::VisitAnyTypeCheck(GraphVisitor *visitor, Inst *inst)
5474 {
5475 auto enc = static_cast<EncodeVisitor *>(visitor);
5476 auto *checkInst = inst->CastToAnyTypeCheck();
5477
5478 if (checkInst->GetInputType(0) != DataType::Type::ANY) {
5479 enc->GetEncoder()->EncodeAbort();
5480 UNREACHABLE();
5481 }
5482 // Empty check
5483 if (checkInst->GetAnyType() == AnyBaseType::UNDEFINED_TYPE) {
5484 return;
5485 }
5486
5487 if (TryAnyTypeCheckPluginGen(checkInst, enc)) {
5488 return;
5489 }
5490 UNREACHABLE();
5491 }
5492
VisitHclassCheck(GraphVisitor * visitor,Inst * inst)5493 void EncodeVisitor::VisitHclassCheck(GraphVisitor *visitor, Inst *inst)
5494 {
5495 auto enc = static_cast<EncodeVisitor *>(visitor);
5496 auto *checkInst = inst->CastToHclassCheck();
5497
5498 if (checkInst->GetInputType(0) != DataType::Type::REFERENCE) {
5499 enc->GetEncoder()->EncodeAbort();
5500 UNREACHABLE();
5501 }
5502
5503 if (TryHclassCheckPluginGen(checkInst, enc)) {
5504 return;
5505 }
5506 UNREACHABLE();
5507 }
5508
VisitObjByIndexCheck(GraphVisitor * visitor,Inst * inst)5509 void EncodeVisitor::VisitObjByIndexCheck(GraphVisitor *visitor, Inst *inst)
5510 {
5511 auto enc = static_cast<EncodeVisitor *>(visitor);
5512 const auto *checkInst = inst->CastToObjByIndexCheck();
5513
5514 if (checkInst->GetInputType(0) != DataType::Type::ANY) {
5515 enc->GetEncoder()->EncodeAbort();
5516 UNREACHABLE();
5517 }
5518 auto id = enc->GetCodegen()->CreateSlowPath<SlowPathDeoptimize>(inst, DeoptimizeType::ANY_TYPE_CHECK)->GetLabel();
5519 if (TryObjByIndexCheckPluginGen(checkInst, enc, id)) {
5520 return;
5521 }
5522 UNREACHABLE();
5523 }
5524
GetNeedBarrierProperty(const Inst * inst)5525 static bool GetNeedBarrierProperty(const Inst *inst)
5526 {
5527 Opcode op = inst->GetOpcode();
5528 if (op == Opcode::LoadObject) {
5529 return inst->CastToLoadObject()->GetNeedBarrier();
5530 }
5531 if (op == Opcode::StoreObject) {
5532 return inst->CastToStoreObject()->GetNeedBarrier();
5533 }
5534 if (op == Opcode::LoadArray) {
5535 return inst->CastToLoadArray()->GetNeedBarrier();
5536 }
5537 if (op == Opcode::StoreArray) {
5538 return inst->CastToStoreArray()->GetNeedBarrier();
5539 }
5540 if (op == Opcode::LoadArrayI) {
5541 return inst->CastToLoadArrayI()->GetNeedBarrier();
5542 }
5543 if (op == Opcode::StoreArrayI) {
5544 return inst->CastToStoreArrayI()->GetNeedBarrier();
5545 }
5546 if (op == Opcode::LoadArrayPair) {
5547 return inst->CastToLoadArrayPair()->GetNeedBarrier();
5548 }
5549 if (op == Opcode::StoreArrayPair) {
5550 return inst->CastToStoreArrayPair()->GetNeedBarrier();
5551 }
5552 if (op == Opcode::LoadArrayPairI) {
5553 return inst->CastToLoadArrayPairI()->GetNeedBarrier();
5554 }
5555 if (op == Opcode::StoreArrayPairI) {
5556 return inst->CastToStoreArrayPairI()->GetNeedBarrier();
5557 }
5558 if (op == Opcode::LoadStatic) {
5559 return inst->CastToLoadStatic()->GetNeedBarrier();
5560 }
5561 if (op == Opcode::StoreStatic) {
5562 return inst->CastToStoreStatic()->GetNeedBarrier();
5563 }
5564 return false;
5565 }
5566
5567 /**
5568 * Returns true if codegen emits call(s) to some library function(s)
5569 * while processing the instruction.
5570 */
InstEncodedWithLibCall(const Inst * inst,Arch arch)5571 bool Codegen::InstEncodedWithLibCall(const Inst *inst, Arch arch)
5572 {
5573 ASSERT(inst != nullptr);
5574 Opcode op = inst->GetOpcode();
5575 if (op == Opcode::Mod) {
5576 auto dstType = inst->GetType();
5577 if (arch == Arch::AARCH64 || arch == Arch::X86_64) {
5578 return dstType == DataType::FLOAT32 || dstType == DataType::FLOAT64;
5579 }
5580 return arch == Arch::AARCH32;
5581 }
5582 if (op == Opcode::Div && arch == Arch::AARCH32) {
5583 auto dstType = inst->GetType();
5584 return dstType == DataType::INT64 || dstType == DataType::UINT64;
5585 }
5586 if (op == Opcode::Cast && arch == Arch::AARCH32) {
5587 auto dstType = inst->GetType();
5588 auto srcType = inst->GetInputType(0);
5589 if (dstType == DataType::FLOAT32 || dstType == DataType::FLOAT64) {
5590 return srcType == DataType::INT64 || srcType == DataType::UINT64;
5591 }
5592 if (srcType == DataType::FLOAT32 || srcType == DataType::FLOAT64) {
5593 return dstType == DataType::INT64 || dstType == DataType::UINT64;
5594 }
5595 return false;
5596 }
5597
5598 return GetNeedBarrierProperty(inst);
5599 }
5600
ConvertInstTmpReg(const Inst * inst,DataType::Type type) const5601 Reg Codegen::ConvertInstTmpReg(const Inst *inst, DataType::Type type) const
5602 {
5603 ASSERT(inst->GetTmpLocation().IsFixedRegister());
5604 return Reg(inst->GetTmpLocation().GetValue(), ConvertDataType(type, GetArch()));
5605 }
5606
ConvertInstTmpReg(const Inst * inst) const5607 Reg Codegen::ConvertInstTmpReg(const Inst *inst) const
5608 {
5609 return ConvertInstTmpReg(inst, Is64BitsArch(GetArch()) ? DataType::INT64 : DataType::INT32);
5610 }
5611
5612 } // namespace panda::compiler
5613