1 /*
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 /*
17 Codegen Hi-Level implementation
18 */
19 #include "operands.h"
20 #include "codegen.h"
21 #include "encode_visitor.h"
22 #include "compiler_options.h"
23 #include "relocations.h"
24 #include "include/compiler_interface.h"
25 #include "ir-dyn-base-types.h"
26 #include "runtime/include/coretypes/string.h"
27 #include "compiler/optimizer/ir/analysis.h"
28 #include "compiler/optimizer/ir/locations.h"
29 #include "compiler/optimizer/analysis/liveness_analyzer.h"
30 #include "optimizer/code_generator/method_properties.h"
31 #include "events/events.h"
32 #include "libpandabase/utils/tsan_interface.h"
33 #include "libpandabase/utils/utils.h"
34 #include <algorithm>
35 #include <iomanip>
36
37 namespace ark::compiler {
38
39 class OsrEntryStub {
FixIntervals(Codegen * codegen,Encoder * encoder)40 void FixIntervals(Codegen *codegen, Encoder *encoder)
41 {
42 auto &la = codegen->GetGraph()->GetAnalysis<LivenessAnalyzer>();
43 la.EnumerateLiveIntervalsForInst(saveState_, [this, codegen, encoder](const auto &li) {
44 auto inst = li->GetInst();
45 auto location = li->GetLocation();
46 // Skip live registers that are already in the input list of the OsrSaveState
47 const auto &ssInputs = saveState_->GetInputs();
48 if (std::find_if(ssInputs.begin(), ssInputs.end(),
49 [inst](auto &input) { return input.GetInst() == inst; }) != ssInputs.end()) {
50 return;
51 }
52 // Only constants allowed
53 switch (inst->GetOpcode()) {
54 case Opcode::LoadAndInitClass: {
55 auto klass = reinterpret_cast<uintptr_t>(inst->CastToLoadAndInitClass()->GetClass());
56 encoder->EncodeMov(codegen->ConvertRegister(inst->GetDstReg(), inst->GetType()),
57 Imm(reinterpret_cast<uintptr_t>(klass)));
58 break;
59 }
60 case Opcode::Constant: {
61 if (location.IsFixedRegister()) {
62 EncodeConstantMove(li, encoder);
63 } else if (location.IsStack()) {
64 auto slot = location.GetValue();
65 encoder->EncodeSti(
66 bit_cast<int64_t>(inst->CastToConstant()->GetRawValue()), DOUBLE_WORD_SIZE_BYTES,
67 MemRef(codegen->SpReg(), codegen->GetFrameLayout().GetSpillOffsetFromSpInBytes(slot)));
68 } else {
69 ASSERT(location.IsConstant());
70 }
71 break;
72 }
73 // NOTE (ekudriashov): UnresolvedLoadAndInitClass
74 default:
75 break;
76 }
77 });
78 }
79
EncodeConstantMove(const LifeIntervals * li,Encoder * encoder)80 static void EncodeConstantMove(const LifeIntervals *li, Encoder *encoder)
81 {
82 auto inst = li->GetInst();
83 switch (li->GetType()) {
84 case DataType::FLOAT64:
85 encoder->EncodeMov(Reg(li->GetReg(), FLOAT64_TYPE),
86 Imm(bit_cast<double>(inst->CastToConstant()->GetDoubleValue())));
87 break;
88 case DataType::FLOAT32:
89 encoder->EncodeMov(Reg(li->GetReg(), FLOAT32_TYPE),
90 Imm(bit_cast<float>(inst->CastToConstant()->GetFloatValue())));
91 break;
92 case DataType::UINT32:
93 encoder->EncodeMov(Reg(li->GetReg(), INT32_TYPE), Imm(inst->CastToConstant()->GetRawValue()));
94 break;
95 case DataType::UINT64:
96 encoder->EncodeMov(Reg(li->GetReg(), INT64_TYPE), Imm(inst->CastToConstant()->GetRawValue()));
97 break;
98 default:
99 UNREACHABLE();
100 }
101 }
102
103 public:
OsrEntryStub(Codegen * codegen,SaveStateInst * inst)104 OsrEntryStub(Codegen *codegen, SaveStateInst *inst) : label_(codegen->GetEncoder()->CreateLabel()), saveState_(inst)
105 {
106 }
107
108 DEFAULT_MOVE_SEMANTIC(OsrEntryStub);
109 DEFAULT_COPY_SEMANTIC(OsrEntryStub);
110 ~OsrEntryStub() = default;
111
Generate(Codegen * codegen)112 void Generate(Codegen *codegen)
113 {
114 auto encoder = codegen->GetEncoder();
115 auto lr = codegen->GetTarget().GetLinkReg();
116 auto fl = codegen->GetFrameLayout();
117 codegen->CreateStackMap(saveState_->CastToSaveStateOsr());
118 ssize_t slot = CFrameLayout::LOCALS_START_SLOT + CFrameLayout::GetLocalsCount() - 1U;
119 encoder->EncodeStp(
120 codegen->FpReg(), lr,
121 MemRef(codegen->FpReg(),
122 -fl.GetOffset<CFrameLayout::OffsetOrigin::FP, CFrameLayout::OffsetUnit::BYTES>(slot)));
123
124 FixIntervals(codegen, encoder);
125 encoder->EncodeJump(label_);
126 }
127
GetInst()128 SaveStateInst *GetInst()
129 {
130 return saveState_;
131 }
132
GetLabel()133 auto &GetLabel()
134 {
135 return label_;
136 }
137
138 private:
139 LabelHolder::LabelId label_;
140 SaveStateInst *saveState_ {nullptr};
141 };
142
Codegen(Graph * graph)143 Codegen::Codegen(Graph *graph)
144 : Optimization(graph),
145 allocator_(graph->GetAllocator()),
146 localAllocator_(graph->GetLocalAllocator()),
147 codeBuilder_(allocator_->New<CodeInfoBuilder>(graph->GetArch(), allocator_)),
148 slowPaths_(graph->GetLocalAllocator()->Adapter()),
149 slowPathsMap_(graph->GetLocalAllocator()->Adapter()),
150 frameLayout_(CFrameLayout(graph->GetArch(), graph->GetStackSlotsCount())),
151 osrEntries_(graph->GetLocalAllocator()->Adapter()),
152 vregIndices_(GetAllocator()->Adapter()),
153 runtime_(graph->GetRuntime()),
154 target_(graph->GetArch()),
155 liveOuts_(graph->GetLocalAllocator()->Adapter()),
156 disasm_(this),
157 spillFillsResolver_(graph)
158 {
159 graph->SetCodeBuilder(codeBuilder_);
160 regfile_ = graph->GetRegisters();
161 if (regfile_ != nullptr) {
162 ASSERT(regfile_->IsValid());
163 ArenaVector<Reg> regsUsage(allocator_->Adapter());
164 Convert(®sUsage, graph->GetUsedRegs<DataType::INT64>(), INT64_TYPE);
165 Convert(®sUsage, graph->GetUsedRegs<DataType::FLOAT64>(), FLOAT64_TYPE);
166 regfile_->SetUsedRegs(regsUsage);
167 #ifndef NDEBUG
168 COMPILER_LOG(DEBUG, CODEGEN) << "Regalloc used registers scalar " << graph->GetUsedRegs<DataType::INT64>();
169 COMPILER_LOG(DEBUG, CODEGEN) << "Regalloc used registers vector " << graph->GetUsedRegs<DataType::FLOAT64>();
170 #endif
171 }
172
173 enc_ = graph->GetEncoder();
174 ASSERT(enc_ != nullptr && enc_->IsValid());
175 enc_->SetRegfile(regfile_);
176 if (enc_->InitMasm()) {
177 enc_->SetFrameLayout(GetFrameLayout());
178 }
179
180 callconv_ = graph->GetCallingConvention();
181 if (callconv_ != nullptr) {
182 ASSERT(callconv_->IsValid());
183 if (callconv_->GetEncoder() == nullptr) {
184 callconv_->SetEncoder(enc_);
185 }
186 }
187
188 auto method = graph->GetMethod();
189 // workaround for test
190 if (method != nullptr) {
191 methodId_ = graph->GetRuntime()->GetMethodId(method);
192 }
193 GetDisasm()->Init();
194 GetDisasm()->SetEncoder(GetEncoder());
195 }
196
GetPassName() const197 const char *Codegen::GetPassName() const
198 {
199 return "Codegen";
200 }
201
AbortIfFailed() const202 bool Codegen::AbortIfFailed() const
203 {
204 return true;
205 }
206
CreateFrameInfo()207 void Codegen::CreateFrameInfo()
208 {
209 // Create FrameInfo for CFrame
210 auto &fl = GetFrameLayout();
211 auto frame = GetGraph()->GetLocalAllocator()->New<FrameInfo>(
212 FrameInfo::PositionedCallers::Encode(true) | FrameInfo::PositionedCallees::Encode(true) |
213 FrameInfo::CallersRelativeFp::Encode(false) | FrameInfo::CalleesRelativeFp::Encode(true));
214 frame->SetFrameSize(fl.GetFrameSize<CFrameLayout::OffsetUnit::BYTES>());
215 frame->SetSpillsCount(fl.GetSpillsCount());
216
217 frame->SetCallersOffset(fl.GetOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::SLOTS>(
218 fl.GetStackStartSlot() + fl.GetCallerLastSlot(false)));
219 frame->SetFpCallersOffset(fl.GetOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::SLOTS>(
220 fl.GetStackStartSlot() + fl.GetCallerLastSlot(true)));
221 frame->SetCalleesOffset(-fl.GetOffset<CFrameLayout::OffsetOrigin::FP, CFrameLayout::OffsetUnit::SLOTS>(
222 fl.GetStackStartSlot() + fl.GetCalleeLastSlot(false)));
223 frame->SetFpCalleesOffset(-fl.GetOffset<CFrameLayout::OffsetOrigin::FP, CFrameLayout::OffsetUnit::SLOTS>(
224 fl.GetStackStartSlot() + fl.GetCalleeLastSlot(true)));
225
226 frame->SetSetupFrame(true);
227 frame->SetSaveFrameAndLinkRegs(true);
228 frame->SetSaveUnusedCalleeRegs(!GetGraph()->GetMethodProperties().GetCompactPrologueAllowed());
229 frame->SetAdjustSpReg(true);
230 frame->SetHasFloatRegs(GetGraph()->HasFloatRegs());
231
232 GetCodeBuilder()->SetHasFloatRegs(GetGraph()->HasFloatRegs());
233
234 SetFrameInfo(frame);
235 }
236
FillOnlyParameters(RegMask * liveRegs,uint32_t numParams,bool isFastpath) const237 void Codegen::FillOnlyParameters(RegMask *liveRegs, uint32_t numParams, bool isFastpath) const
238 {
239 ASSERT(numParams <= 6U);
240 if (GetArch() == Arch::AARCH64 && !isFastpath) {
241 numParams = AlignUp(numParams, 2U);
242 }
243 *liveRegs &= GetTarget().GetParamRegsMask(numParams);
244 }
245
Convert(ArenaVector<Reg> * regsUsage,const ArenaVector<bool> * mask,TypeInfo typeInfo)246 void Codegen::Convert(ArenaVector<Reg> *regsUsage, const ArenaVector<bool> *mask, TypeInfo typeInfo)
247 {
248 ASSERT(regsUsage != nullptr);
249 // There are no used registers
250 if (mask == nullptr) {
251 return;
252 }
253 ASSERT(mask->size() == MAX_NUM_REGS);
254 for (uint32_t i = 0; i < MAX_NUM_REGS; ++i) {
255 if ((*mask)[i]) {
256 regsUsage->emplace_back(i, typeInfo);
257 }
258 }
259 }
260
IntrinsicSlowPathEntry(IntrinsicInst * inst)261 void Codegen::IntrinsicSlowPathEntry([[maybe_unused]] IntrinsicInst *inst)
262 {
263 GetEncoder()->SetFalseResult();
264 }
IntrinsicCallRuntimeSaveAll(IntrinsicInst * inst)265 void Codegen::IntrinsicCallRuntimeSaveAll([[maybe_unused]] IntrinsicInst *inst)
266 {
267 GetEncoder()->SetFalseResult();
268 }
IntrinsicSaveRegisters(IntrinsicInst * inst)269 void Codegen::IntrinsicSaveRegisters([[maybe_unused]] IntrinsicInst *inst)
270 {
271 GetEncoder()->SetFalseResult();
272 }
IntrinsicRestoreRegisters(IntrinsicInst * inst)273 void Codegen::IntrinsicRestoreRegisters([[maybe_unused]] IntrinsicInst *inst)
274 {
275 GetEncoder()->SetFalseResult();
276 }
IntrinsicTailCall(IntrinsicInst * inst)277 void Codegen::IntrinsicTailCall([[maybe_unused]] IntrinsicInst *inst)
278 {
279 GetEncoder()->SetFalseResult();
280 }
IntrinsicSaveTlabStatsSafe(IntrinsicInst * inst,Reg src1,Reg src2,Reg tmp)281 void Codegen::IntrinsicSaveTlabStatsSafe([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg src1,
282 [[maybe_unused]] Reg src2, [[maybe_unused]] Reg tmp)
283 {
284 GetEncoder()->SetFalseResult();
285 }
286
EmitAtomicByteOr(Reg addr,Reg value)287 void Codegen::EmitAtomicByteOr(Reg addr, Reg value)
288 {
289 bool fastEncoding = true;
290 if (GetArch() == Arch::AARCH64 && !g_options.IsCpuFeatureEnabled(CpuFeature::ATOMICS)) {
291 fastEncoding = false;
292 }
293 GetEncoder()->EncodeAtomicByteOr(addr, value, fastEncoding);
294 }
295
296 #ifdef INTRINSIC_SLOW_PATH_ENTRY_ENABLED
297 // NOLINTNEXTLINE(readability-function-size)
CreateIrtocIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)298 void Codegen::CreateIrtocIntrinsic(IntrinsicInst *inst, [[maybe_unused]] Reg dst, [[maybe_unused]] SRCREGS src)
299 {
300 switch (inst->GetIntrinsicId()) {
301 case RuntimeInterface::IntrinsicId::INTRINSIC_SLOW_PATH_ENTRY:
302 IntrinsicSlowPathEntry(inst);
303 break;
304 case RuntimeInterface::IntrinsicId::INTRINSIC_UNREACHABLE:
305 GetEncoder()->EncodeAbort();
306 break;
307 case RuntimeInterface::IntrinsicId::INTRINSIC_SAVE_REGISTERS_EP:
308 IntrinsicSaveRegisters(inst);
309 break;
310 case RuntimeInterface::IntrinsicId::INTRINSIC_RESTORE_REGISTERS_EP:
311 IntrinsicRestoreRegisters(inst);
312 break;
313 case RuntimeInterface::IntrinsicId::INTRINSIC_TAIL_CALL:
314 IntrinsicTailCall(inst);
315 break;
316 case RuntimeInterface::IntrinsicId::INTRINSIC_INTERPRETER_RETURN:
317 GetCallingConvention()->GenerateNativeEpilogue(*GetFrameInfo(), []() {});
318 break;
319 case RuntimeInterface::IntrinsicId::INTRINSIC_LOAD_ACQUIRE_MARK_WORD_EXCLUSIVE:
320 ASSERT(GetRuntime()->GetObjMarkWordOffset(GetArch()) == 0);
321 GetEncoder()->EncodeLdrExclusive(dst, src[0], true);
322 break;
323 case RuntimeInterface::IntrinsicId::INTRINSIC_STORE_RELEASE_MARK_WORD_EXCLUSIVE:
324 ASSERT(GetRuntime()->GetObjMarkWordOffset(GetArch()) == 0);
325 GetEncoder()->EncodeStrExclusive(dst, src[SECOND_OPERAND], src[0], true);
326 break;
327 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPARE_AND_SET_MARK_WORD:
328 ASSERT(GetRuntime()->GetObjMarkWordOffset(GetArch()) == 0);
329 GetEncoder()->EncodeCompareAndSwap(dst, src[0], src[SECOND_OPERAND], src[THIRD_OPERAND]);
330 break;
331 case RuntimeInterface::IntrinsicId::INTRINSIC_DATA_MEMORY_BARRIER_FULL:
332 GetEncoder()->EncodeMemoryBarrier(memory_order::FULL);
333 break;
334 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPRESS_EIGHT_UTF16_TO_UTF8_CHARS_USING_SIMD:
335 GetEncoder()->EncodeCompressEightUtf16ToUtf8CharsUsingSimd(src[FIRST_OPERAND], src[SECOND_OPERAND]);
336 break;
337 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPRESS_SIXTEEN_UTF16_TO_UTF8_CHARS_USING_SIMD:
338 GetEncoder()->EncodeCompressSixteenUtf16ToUtf8CharsUsingSimd(src[FIRST_OPERAND], src[SECOND_OPERAND]);
339 break;
340 case RuntimeInterface::IntrinsicId::INTRINSIC_WRITE_TLAB_STATS_SAFE:
341 IntrinsicSaveTlabStatsSafe(inst, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND]);
342 break;
343 case RuntimeInterface::IntrinsicId::INTRINSIC_REVERSE_BYTES_U64:
344 case RuntimeInterface::IntrinsicId::INTRINSIC_REVERSE_BYTES_U32:
345 GetEncoder()->EncodeReverseBytes(dst, src[0]);
346 break;
347 case RuntimeInterface::IntrinsicId::INTRINSIC_REVERSE_HALF_WORDS:
348 GetEncoder()->EncodeReverseHalfWords(dst, src[0]);
349 break;
350 case RuntimeInterface::IntrinsicId::INTRINSIC_EXPAND_U8_TO_U16:
351 GetEncoder()->EncodeUnsignedExtendBytesToShorts(dst, src[0]);
352 break;
353 case RuntimeInterface::IntrinsicId::INTRINSIC_ATOMIC_BYTE_OR:
354 EmitAtomicByteOr(src[FIRST_OPERAND], src[SECOND_OPERAND]);
355 break;
356 default:
357 UNREACHABLE();
358 break;
359 }
360 }
361 #endif
362
BeginMethod()363 bool Codegen::BeginMethod()
364 {
365 // Do not try to encode too large graph
366 auto estimatedCodeSize = GetGraph()->EstimateCodeSize();
367 if (estimatedCodeSize > g_options.GetCompilerMaxGenCodeSize()) {
368 COMPILER_LOG(WARNING, CODEGEN) << "Code is too large - estimated code size: " << estimatedCodeSize;
369 return false;
370 }
371 // Limit encoder to match estimated to actual code size.
372 // After this - encoder aborted, if allocated too much size.
373 GetEncoder()->SetMaxAllocatedBytes(g_options.GetCompilerMaxGenCodeSize());
374
375 if (!IsCompressedStringsEnabled()) {
376 #ifndef NDEBUG
377 LOG(FATAL, COMPILER) << "String compression must be enabled";
378 #else
379 LOG(ERROR, COMPILER) << "String compression must be enabled";
380 #endif
381 return false;
382 }
383
384 if (GetGraph()->IsAotMode()) {
385 GetEncoder()->SetCodeOffset(GetGraph()->GetAotData()->GetCodeOffset() + CodeInfo::GetCodeOffset(GetArch()));
386 } else {
387 GetEncoder()->SetCodeOffset(0);
388 }
389
390 codeBuilder_->BeginMethod(GetFrameLayout().GetFrameSize<CFrameLayout::OffsetUnit::BYTES>(),
391 GetGraph()->GetVRegsCount() + GetGraph()->GetEnvCount());
392
393 GetEncoder()->BindLabel(GetLabelEntry());
394 SetStartCodeOffset(GetEncoder()->GetCursorOffset());
395
396 GeneratePrologue();
397
398 return GetEncoder()->GetResult();
399 }
400
GeneratePrologue()401 void Codegen::GeneratePrologue()
402 {
403 SCOPED_DISASM_STR(this, "Method Prologue");
404
405 GetCallingConvention()->GeneratePrologue(*frameInfo_);
406
407 if (!GetGraph()->GetMode().IsNative()) {
408 GetEncoder()->EncodeSti(1, 1, MemRef(ThreadReg(), GetRuntime()->GetTlsFrameKindOffset(GetArch())));
409 }
410 if (!GetGraph()->GetMode().IsNative()) {
411 // Create stack overflow check
412 GetEncoder()->EncodeStackOverflowCheck(-GetRuntime()->GetStackOverflowCheckOffset());
413 // Create empty stackmap for the stack overflow check
414 SaveStateInst ss(Opcode::SaveState, 0, nullptr, nullptr, 0);
415 GetCodeBuilder()->BeginStackMap(0, 0, &ss, false);
416 GetCodeBuilder()->EndStackMap();
417 }
418
419 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
420 if (GetGraph()->IsAotMode()) {
421 SCOPED_DISASM_STR(this, "LoadMethod for trace");
422 ScopedTmpReg method_reg(GetEncoder());
423 LoadMethod(method_reg);
424 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_ENTER)), method_reg,
425 Imm(static_cast<size_t>(events::MethodEnterKind::COMPILED)));
426 } else {
427 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
428 Imm(reinterpret_cast<size_t>(GetGraph()->GetMethod())),
429 Imm(static_cast<size_t>(events::MethodEnterKind::COMPILED)));
430 }
431 #endif
432 }
433
GenerateEpilogue()434 void Codegen::GenerateEpilogue()
435 {
436 SCOPED_DISASM_STR(this, "Method Epilogue");
437
438 #if defined(EVENT_METHOD_EXIT_ENABLED) && EVENT_METHOD_EXIT_ENABLED != 0
439 GetCallingConvention()->GenerateEpilogue(*frame_info_, [this]() {
440 if (GetGraph()->IsAotMode()) {
441 ScopedTmpReg method_reg(GetEncoder());
442 LoadMethod(method_reg);
443 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_EXIT)), method_reg,
444 Imm(static_cast<size_t>(events::MethodExitKind::COMPILED)));
445 } else {
446 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_EXIT)),
447 Imm(reinterpret_cast<size_t>(GetGraph()->GetMethod())),
448 Imm(static_cast<size_t>(events::MethodExitKind::COMPILED)));
449 }
450 });
451 #else
452 GetCallingConvention()->GenerateEpilogue(*frameInfo_, []() {});
453 #endif
454 }
455
VisitGraph()456 bool Codegen::VisitGraph()
457 {
458 EncodeVisitor visitor(this);
459 visitor_ = &visitor;
460
461 const auto &blocks = GetGraph()->GetBlocksLinearOrder();
462
463 for (auto bb : blocks) {
464 GetEncoder()->BindLabel(bb->GetId());
465 for (auto inst : bb->AllInsts()) {
466 SCOPED_DISASM_INST(this, inst);
467
468 #ifdef PANDA_COMPILER_DEBUG_INFO
469 auto debugInfo = inst->GetDebugInfo();
470 if (GetGraph()->IsLineDebugInfoEnabled() && debugInfo != nullptr) {
471 debugInfo->SetOffset(GetEncoder()->GetCursorOffset());
472 }
473 #endif
474 visitor.VisitInstruction(inst);
475 if (!visitor.GetResult()) {
476 COMPILER_LOG(DEBUG, CODEGEN)
477 << "Can't encode instruction: " << GetOpcodeString(inst->GetOpcode()) << *inst;
478 break;
479 }
480 }
481
482 if (bb->NeedsJump()) {
483 EmitJump(bb);
484 }
485
486 if (!visitor.GetResult()) {
487 return false;
488 }
489 }
490
491 auto estimatedSlowPathsSize = slowPaths_.size() * SLOW_PATH_SIZE;
492 auto estimatedCodeSize = GetGraph()->EstimateCodeSize<true>() + estimatedSlowPathsSize;
493 if (estimatedCodeSize > g_options.GetCompilerMaxGenCodeSize()) {
494 COMPILER_LOG(WARNING, CODEGEN) << "Code is too large - code size: " << GetGraph()->EstimateCodeSize<true>()
495 << ", slow paths' code estimation: " << estimatedSlowPathsSize;
496 return false;
497 }
498
499 EmitSlowPaths();
500 visitor_ = nullptr;
501
502 return true;
503 }
504
EmitJump(const BasicBlock * bb)505 void Codegen::EmitJump(const BasicBlock *bb)
506 {
507 BasicBlock *sucBb = nullptr;
508
509 if (bb->GetLastInst() == nullptr) {
510 ASSERT(bb->IsEmpty());
511 sucBb = bb->GetSuccsBlocks()[0];
512 } else {
513 switch (bb->GetLastInst()->GetOpcode()) {
514 case Opcode::If:
515 case Opcode::IfImm:
516 ASSERT(bb->GetSuccsBlocks().size() == MAX_SUCCS_NUM);
517 sucBb = bb->GetFalseSuccessor();
518 break;
519 default:
520 sucBb = bb->GetSuccsBlocks()[0];
521 break;
522 }
523 }
524 SCOPED_DISASM_STR(this, std::string("Jump from BB_") + std::to_string(bb->GetId()) + " to BB_" +
525 std::to_string(sucBb->GetId()));
526
527 auto label = sucBb->GetId();
528 GetEncoder()->EncodeJump(label);
529 }
530
EndMethod()531 void Codegen::EndMethod()
532 {
533 for (auto &osrStub : osrEntries_) {
534 SCOPED_DISASM_STR(this,
535 std::string("Osr stub for OsrStateStump ") + std::to_string(osrStub->GetInst()->GetId()));
536 osrStub->Generate(this);
537 }
538
539 GetEncoder()->Finalize();
540 }
541
542 // Allocates memory, copies generated code to it, sets the code to the graph's codegen data. Also this function
543 // encodes the code info and sets it to the graph.
CopyToCodeCache()544 bool Codegen::CopyToCodeCache()
545 {
546 auto codeEntry = reinterpret_cast<char *>(GetEncoder()->GetLabelAddress(GetLabelEntry()));
547 auto codeSize = GetEncoder()->GetCursorOffset();
548 bool saveAllCalleeRegisters = !GetGraph()->GetMethodProperties().GetCompactPrologueAllowed();
549
550 auto code = reinterpret_cast<uint8_t *>(GetAllocator()->Alloc(codeSize));
551 if (code == nullptr) {
552 return false;
553 }
554 std::copy_n(codeEntry, codeSize, code);
555 GetGraph()->SetCode(EncodeDataType(code, codeSize));
556
557 RegMask calleeRegs;
558 VRegMask calleeVregs;
559 GetRegfile()->FillUsedCalleeSavedRegisters(&calleeRegs, &calleeVregs, saveAllCalleeRegisters);
560 constexpr size_t MAX_NUM_REGISTERS = 32;
561 static_assert(MAX_NUM_REGS <= MAX_NUM_REGISTERS && MAX_NUM_VREGS <= MAX_NUM_REGISTERS);
562 codeBuilder_->SetSavedCalleeRegsMask(static_cast<uint32_t>(calleeRegs.to_ulong()),
563 static_cast<uint32_t>(calleeVregs.to_ulong()));
564
565 ArenaVector<uint8_t> codeInfoData(GetGraph()->GetAllocator()->Adapter());
566 codeBuilder_->Encode(&codeInfoData);
567 GetGraph()->SetCodeInfo(Span(codeInfoData));
568
569 return true;
570 }
571
IssueDisasm()572 void Codegen::IssueDisasm()
573 {
574 if (GetGraph()->GetMode().SupportManagedCode() && g_options.IsCompilerDisasmDumpCodeInfo()) {
575 GetDisasm()->PrintCodeInfo(this);
576 }
577 GetDisasm()->PrintCodeStatistics(this);
578 auto &stream = GetDisasm()->GetStream();
579 for (auto &item : GetDisasm()->GetItems()) {
580 if (std::holds_alternative<CodeItem>(item)) {
581 auto &code = std::get<CodeItem>(item);
582 for (size_t pc = code.GetPosition(); pc < code.GetCursorOffset();) {
583 stream << GetDisasm()->GetIndent(code.GetDepth());
584 auto newPc = GetEncoder()->DisasmInstr(stream, pc, 0);
585 stream << "\n";
586 pc = newPc;
587 }
588 } else if (std::holds_alternative<std::string>(item)) {
589 stream << (std::get<std::string>(item));
590 }
591 }
592 }
593
RunImpl()594 bool Codegen::RunImpl()
595 {
596 Initialize();
597
598 auto encoder = GetEncoder();
599 encoder->GetLabels()->CreateLabels(GetGraph()->GetVectorBlocks().size());
600 labelEntry_ = encoder->CreateLabel();
601 labelExit_ = encoder->CreateLabel();
602
603 #ifndef NDEBUG
604 if (g_options.IsCompilerNonOptimizing()) {
605 // In case of non-optimizing compiler lowering pass is not run but low-level instructions may
606 // also appear on codegen so, to satisfy GraphChecker, the flag should be raised.
607 GetGraph()->SetLowLevelInstructionsEnabled();
608 }
609 #endif // NDEBUG
610
611 if ((GetCallingConvention() == nullptr) || (GetEncoder() == nullptr)) {
612 return false;
613 }
614
615 if (GetCallingConvention()->IsDynCallMode()) {
616 auto numExpectedArgs = GetRuntime()->GetMethodTotalArgumentsCount(GetGraph()->GetMethod());
617 if (numExpectedArgs > GetRuntime()->GetDynamicNumFixedArgs()) {
618 CallConvDynInfo dynInfo(numExpectedArgs,
619 GetRuntime()->GetEntrypointTlsOffset(
620 GetArch(), RuntimeInterface::EntrypointId::EXPAND_COMPILED_CODE_ARGS_DYN));
621 GetCallingConvention()->SetDynInfo(dynInfo);
622 frameInfo_->SetSaveFrameAndLinkRegs(true);
623 } else {
624 GetCallingConvention()->SetDynInfo(CallConvDynInfo());
625 }
626 }
627
628 if (!GetEncoder()->GetResult()) {
629 return false;
630 }
631
632 // Remove registers from the temp registers, if they are in the regalloc mask, i.e. available for regalloc.
633 auto usedRegs = ~GetGraph()->GetArchUsedRegs();
634 auto forbiddenTemps = usedRegs & GetTarget().GetTempRegsMask();
635 if (forbiddenTemps.Any()) {
636 for (size_t i = forbiddenTemps.GetMinRegister(); i <= forbiddenTemps.GetMaxRegister(); i++) {
637 if (forbiddenTemps[i] && encoder->IsScratchRegisterReleased(Reg(i, INT64_TYPE))) {
638 encoder->AcquireScratchRegister(Reg(i, INT64_TYPE));
639 }
640 }
641 }
642
643 return Finalize();
644 }
645
Initialize()646 void Codegen::Initialize()
647 {
648 CreateFrameInfo();
649
650 GetRegfile()->SetCalleeSaved(GetRegfile()->GetCalleeSaved());
651
652 if (!GetGraph()->SupportManagedCode()) {
653 for (auto inst : GetGraph()->GetStartBlock()->AllInsts()) {
654 if (inst->GetOpcode() == Opcode::LiveIn && GetTarget().GetTempRegsMask().Test(inst->GetDstReg())) {
655 GetEncoder()->AcquireScratchRegister(Reg(inst->GetDstReg(), INT64_TYPE));
656 }
657 }
658 }
659
660 bool hasCalls = false;
661
662 for (auto bb : GetGraph()->GetBlocksLinearOrder()) {
663 // Calls may be in the middle of method
664 for (auto inst : bb->Insts()) {
665 // For throw instruction need jump2runtime same way
666 if (inst->IsCall() || inst->GetOpcode() == Opcode::Throw) {
667 hasCalls = true;
668 break;
669 }
670 }
671 if (hasCalls) {
672 break;
673 }
674 }
675
676 /* Convert Graph::GetUsedRegs(), which is std::vector<bool>, to simple
677 * RegMask and save it in the Codegen. These masks are used to determine
678 * which registers we need to save in prologue.
679 *
680 * NB! It's related to IRTOC specific prologue generation (see CodegenFastPath etc.).
681 * Arch specific CallingConvention::GenerateProlog() relies on reg usage information
682 * prepared in the Codegen constructor (before Initialize() is called).
683 */
684 auto fillMask = [](RegMask *mask, auto *vector) {
685 if (vector == nullptr) {
686 return;
687 }
688 ASSERT(mask->size() >= vector->size());
689 mask->reset();
690 for (size_t i = 0; i < mask->size(); i++) {
691 if ((*vector)[i]) {
692 mask->set(i);
693 }
694 }
695 };
696 fillMask(&usedRegs_, GetGraph()->GetUsedRegs<DataType::INT64>());
697 fillMask(&usedVregs_, GetGraph()->GetUsedRegs<DataType::FLOAT64>());
698 usedVregs_ &= GetTarget().GetAvailableVRegsMask();
699 usedRegs_ &= GetTarget().GetAvailableRegsMask();
700 usedRegs_ &= ~GetGraph()->GetArchUsedRegs();
701 usedVregs_ &= ~GetGraph()->GetArchUsedVRegs();
702
703 /* Remove used registers from Encoder's scratch registers */
704 RegMask usedTemps = usedRegs_ & GetTarget().GetTempRegsMask();
705 if (usedTemps.any()) {
706 for (size_t i = 0; i < usedTemps.size(); i++) {
707 if (usedTemps[i]) {
708 GetEncoder()->AcquireScratchRegister(Reg(i, INT64_TYPE));
709 }
710 }
711 }
712 }
713
Finalize()714 bool Codegen::Finalize()
715 {
716 if (GetDisasm()->IsEnabled()) {
717 GetDisasm()->PrintMethodEntry(this);
718 }
719
720 if (!BeginMethod()) {
721 return false;
722 }
723
724 if (!VisitGraph()) {
725 return false;
726 }
727 EndMethod();
728
729 if (!CopyToCodeCache()) {
730 return false;
731 }
732
733 if (GetDisasm()->IsEnabled()) {
734 IssueDisasm();
735 }
736
737 return true;
738 }
739
ConvertRegister(Register r,DataType::Type type)740 Reg Codegen::ConvertRegister(Register r, DataType::Type type)
741 {
742 switch (type) {
743 case DataType::BOOL:
744 case DataType::UINT8:
745 case DataType::INT8: {
746 return Reg(r, INT8_TYPE);
747 }
748 case DataType::UINT16:
749 case DataType::INT16: {
750 return Reg(r, INT16_TYPE);
751 }
752 case DataType::UINT32:
753 case DataType::INT32: {
754 return Reg(r, INT32_TYPE);
755 }
756 case DataType::UINT64:
757 case DataType::INT64:
758 case DataType::ANY: {
759 return Reg(r, INT64_TYPE);
760 }
761 case DataType::FLOAT32: {
762 return Reg(r, FLOAT32_TYPE);
763 }
764 case DataType::FLOAT64: {
765 return Reg(r, FLOAT64_TYPE);
766 }
767 case DataType::REFERENCE: {
768 return ConvertRegister(r, DataType::GetIntTypeForReference(GetArch()));
769 }
770 case DataType::POINTER: {
771 return Reg(r, ConvertDataType(DataType::POINTER, GetArch()));
772 }
773 default:
774 // Invalid converted register
775 return INVALID_REGISTER;
776 }
777 }
778
779 // Panda don't support types less then 32, so we need sign or zero extended to 32
ConvertImmWithExtend(uint64_t imm,DataType::Type type)780 Imm Codegen::ConvertImmWithExtend(uint64_t imm, DataType::Type type)
781 {
782 switch (type) {
783 case DataType::BOOL:
784 case DataType::UINT8:
785 return Imm(static_cast<uint32_t>(static_cast<uint8_t>(imm)));
786 case DataType::INT8:
787 return Imm(static_cast<int32_t>(bit_cast<int8_t, uint8_t>(imm)));
788 case DataType::UINT16:
789 return Imm(static_cast<uint32_t>(static_cast<uint16_t>(imm)));
790 case DataType::INT16:
791 return Imm(static_cast<int32_t>(bit_cast<int16_t, uint16_t>(imm)));
792 // NOLINTNEXTLINE(bugprone-branch-clone)
793 case DataType::UINT32:
794 return Imm(bit_cast<int32_t, uint32_t>(imm));
795 case DataType::INT32:
796 return Imm(bit_cast<int32_t, uint32_t>(imm));
797 // NOLINTNEXTLINE(bugprone-branch-clone)
798 case DataType::UINT64:
799 return Imm(bit_cast<int64_t, uint64_t>(imm));
800 case DataType::INT64:
801 return Imm(bit_cast<int64_t, uint64_t>(imm));
802 case DataType::FLOAT32:
803 return Imm(bit_cast<float, uint32_t>(static_cast<uint32_t>(imm)));
804 case DataType::FLOAT64:
805 return Imm(bit_cast<double, uint64_t>(imm));
806 case DataType::ANY:
807 return Imm(bit_cast<uint64_t, uint64_t>(imm));
808 case DataType::REFERENCE:
809 if (imm == 0) {
810 return Imm(0);
811 }
812 [[fallthrough]]; /* fall-through */
813 default:
814 // Invalid converted immediate
815 UNREACHABLE();
816 }
817 return Imm(0);
818 }
819
ConvertCc(ConditionCode cc)820 Condition Codegen::ConvertCc(ConditionCode cc)
821 {
822 switch (cc) {
823 case CC_EQ:
824 return Condition::EQ;
825 case CC_NE:
826 return Condition::NE;
827 case CC_LT:
828 return Condition::LT;
829 case CC_LE:
830 return Condition::LE;
831 case CC_GT:
832 return Condition::GT;
833 case CC_GE:
834 return Condition::GE;
835 case CC_B:
836 return Condition::LO;
837 case CC_BE:
838 return Condition::LS;
839 case CC_A:
840 return Condition::HI;
841 case CC_AE:
842 return Condition::HS;
843 case CC_TST_EQ:
844 return Condition::TST_EQ;
845 case CC_TST_NE:
846 return Condition::TST_NE;
847 default:
848 UNREACHABLE();
849 return Condition::EQ;
850 }
851 return Condition::EQ;
852 }
853
ConvertCcOverflow(ConditionCode cc)854 Condition Codegen::ConvertCcOverflow(ConditionCode cc)
855 {
856 switch (cc) {
857 case CC_EQ:
858 return Condition::VS;
859 case CC_NE:
860 return Condition::VC;
861 default:
862 UNREACHABLE();
863 return Condition::VS;
864 }
865 return Condition::VS;
866 }
867
EmitSlowPaths()868 void Codegen::EmitSlowPaths()
869 {
870 for (auto slowPath : slowPaths_) {
871 slowPath->Generate(this);
872 }
873 }
874
CreateStackMap(Inst * inst,Inst * user)875 void Codegen::CreateStackMap(Inst *inst, Inst *user)
876 {
877 SaveStateInst *saveState = nullptr;
878 if (inst->IsSaveState()) {
879 saveState = static_cast<SaveStateInst *>(inst);
880 } else {
881 saveState = inst->GetSaveState();
882 }
883 ASSERT(saveState != nullptr);
884
885 bool requireVregMap = inst->RequireRegMap();
886 uint32_t outerBpc = inst->GetPc();
887 for (auto callInst = saveState->GetCallerInst(); callInst != nullptr;
888 callInst = callInst->GetSaveState()->GetCallerInst()) {
889 outerBpc = callInst->GetPc();
890 }
891 codeBuilder_->BeginStackMap(outerBpc, GetEncoder()->GetCursorOffset(), saveState, requireVregMap);
892 if (user == nullptr) {
893 user = inst;
894 if (inst == saveState && inst->HasUsers()) {
895 auto users = inst->GetUsers();
896 auto it = std::find_if(users.begin(), users.end(),
897 [](auto &u) { return u.GetInst()->GetOpcode() != Opcode::ReturnInlined; });
898 user = it->GetInst();
899 }
900 }
901 CreateStackMapRec(saveState, requireVregMap, user);
902
903 codeBuilder_->EndStackMap();
904 if (GetDisasm()->IsEnabled()) {
905 GetDisasm()->PrintStackMap(this);
906 }
907 }
908
CreateStackMapRec(SaveStateInst * saveState,bool requireVregMap,Inst * targetSite)909 void Codegen::CreateStackMapRec(SaveStateInst *saveState, bool requireVregMap, Inst *targetSite)
910 {
911 bool hasInlineInfo = saveState->GetCallerInst() != nullptr;
912 size_t vregsCount = 0;
913 if (requireVregMap) {
914 auto runtime = GetRuntime();
915 if (auto caller = saveState->GetCallerInst()) {
916 vregsCount = runtime->GetMethodRegistersCount(caller->GetCallMethod()) +
917 runtime->GetMethodArgumentsCount(caller->GetCallMethod());
918 } else {
919 vregsCount = runtime->GetMethodRegistersCount(saveState->GetMethod()) +
920 runtime->GetMethodArgumentsCount(saveState->GetMethod());
921 }
922 ASSERT(!GetGraph()->IsBytecodeOptimizer());
923 // 1 for acc, number of env regs for dynamic method
924 vregsCount += 1U + GetGraph()->GetEnvCount();
925 #ifndef NDEBUG
926 ASSERT_PRINT(!saveState->GetInputsWereDeleted(), "Some vregs were deleted from the save state");
927 #endif
928 }
929
930 if (auto callInst = saveState->GetCallerInst()) {
931 CreateStackMapRec(callInst->GetSaveState(), requireVregMap, targetSite);
932 auto method = GetGraph()->IsAotMode() ? nullptr : callInst->GetCallMethod();
933 codeBuilder_->BeginInlineInfo(method, GetRuntime()->GetMethodId(callInst->GetCallMethod()), saveState->GetPc(),
934 vregsCount);
935 }
936
937 if (requireVregMap) {
938 CreateVRegMap(saveState, vregsCount, targetSite);
939 }
940
941 if (hasInlineInfo) {
942 codeBuilder_->EndInlineInfo();
943 }
944 }
945
CreateVRegMap(SaveStateInst * saveState,size_t vregsCount,Inst * targetSite)946 void Codegen::CreateVRegMap(SaveStateInst *saveState, size_t vregsCount, Inst *targetSite)
947 {
948 vregIndices_.clear();
949 vregIndices_.resize(vregsCount, {-1, -1});
950 FillVregIndices(saveState);
951
952 ASSERT(GetGraph()->IsAnalysisValid<LivenessAnalyzer>());
953 auto &la = GetGraph()->GetAnalysis<LivenessAnalyzer>();
954 auto targetLifeNumber = la.GetInstLifeIntervals(targetSite)->GetBegin();
955
956 for (auto &inputIndex : vregIndices_) {
957 if (inputIndex.first == -1 && inputIndex.second == -1) {
958 codeBuilder_->AddVReg(VRegInfo());
959 continue;
960 }
961 if (inputIndex.second != -1) {
962 auto imm = saveState->GetImmediate(inputIndex.second);
963 codeBuilder_->AddConstant(imm.value, IrTypeToMetainfoType(imm.type), imm.vregType);
964 continue;
965 }
966 ASSERT(inputIndex.first != -1);
967 auto vreg = saveState->GetVirtualRegister(inputIndex.first);
968 auto inputInst = saveState->GetDataFlowInput(inputIndex.first);
969 auto interval = la.GetInstLifeIntervals(inputInst)->FindSiblingAt(targetLifeNumber);
970 ASSERT(interval != nullptr);
971 CreateVreg(interval->GetLocation(), inputInst, vreg);
972 }
973 }
974
CreateVreg(const Location & location,Inst * inst,const VirtualRegister & vreg)975 void Codegen::CreateVreg(const Location &location, Inst *inst, const VirtualRegister &vreg)
976 {
977 switch (location.GetKind()) {
978 case LocationType::FP_REGISTER:
979 case LocationType::REGISTER: {
980 CreateVRegForRegister(location, inst, vreg);
981 break;
982 }
983 case LocationType::STACK_PARAMETER: {
984 auto slot = location.GetValue();
985 codeBuilder_->AddVReg(VRegInfo(GetFrameLayout().GetStackArgsStartSlot() - slot - CFrameSlots::Start(),
986 VRegInfo::Location::SLOT, IrTypeToMetainfoType(inst->GetType()),
987 vreg.GetVRegType()));
988 break;
989 }
990 case LocationType::STACK: {
991 auto slot = location.GetValue();
992 if (!Is64BitsArch(GetArch())) {
993 slot = ((location.GetValue() << 1U) + 1);
994 }
995 codeBuilder_->AddVReg(VRegInfo(GetFrameLayout().GetFirstSpillSlot() + slot, VRegInfo::Location::SLOT,
996 IrTypeToMetainfoType(inst->GetType()), vreg.GetVRegType()));
997 break;
998 }
999 case LocationType::IMMEDIATE: {
1000 ASSERT(inst->IsConst());
1001 auto type = inst->GetType();
1002 // There are no int64 type for dynamic methods.
1003 if (GetGraph()->IsDynamicMethod() && type == DataType::INT64) {
1004 type = DataType::INT32;
1005 }
1006 codeBuilder_->AddConstant(inst->CastToConstant()->GetRawValue(), IrTypeToMetainfoType(type),
1007 vreg.GetVRegType());
1008 break;
1009 }
1010 default:
1011 // Reg-to-reg spill fill must not occurs within SaveState
1012 UNREACHABLE();
1013 }
1014 }
1015
FillVregIndices(SaveStateInst * saveState)1016 void Codegen::FillVregIndices(SaveStateInst *saveState)
1017 {
1018 for (size_t i = 0; i < saveState->GetInputsCount(); ++i) {
1019 size_t vregIndex = saveState->GetVirtualRegister(i).Value();
1020 if (vregIndex == VirtualRegister::BRIDGE) {
1021 continue;
1022 }
1023 ASSERT(vregIndex < vregIndices_.size());
1024 vregIndices_[vregIndex].first = i;
1025 }
1026 for (size_t i = 0; i < saveState->GetImmediatesCount(); i++) {
1027 auto vregImm = saveState->GetImmediate(i);
1028 if (vregImm.vreg == VirtualRegister::BRIDGE) {
1029 continue;
1030 }
1031 ASSERT(vregImm.vreg < vregIndices_.size());
1032 ASSERT(vregIndices_[vregImm.vreg].first == -1);
1033 vregIndices_[vregImm.vreg].second = i;
1034 }
1035 }
1036
CreateVRegForRegister(const Location & location,Inst * inst,const VirtualRegister & vreg)1037 void Codegen::CreateVRegForRegister(const Location &location, Inst *inst, const VirtualRegister &vreg)
1038 {
1039 bool isOsr = GetGraph()->IsOsrMode();
1040 bool isFp = (location.GetKind() == LocationType::FP_REGISTER);
1041 auto regNum = location.GetValue();
1042 auto reg = Reg(regNum, isFp ? FLOAT64_TYPE : INT64_TYPE);
1043 if (!isOsr && GetRegfile()->GetZeroReg() == reg) {
1044 codeBuilder_->AddConstant(0, IrTypeToMetainfoType(inst->GetType()), vreg.GetVRegType());
1045 } else if (isOsr || GetRegfile()->IsCalleeRegister(reg)) {
1046 if (isFp) {
1047 ASSERT(inst->GetType() != DataType::REFERENCE);
1048 ASSERT(isOsr || regNum >= GetFirstCalleeReg(GetArch(), true));
1049 codeBuilder_->AddVReg(VRegInfo(regNum, VRegInfo::Location::FP_REGISTER,
1050 IrTypeToMetainfoType(inst->GetType()), vreg.GetVRegType()));
1051 } else {
1052 ASSERT(isOsr || regNum >= GetFirstCalleeReg(GetArch(), false));
1053 codeBuilder_->AddVReg(VRegInfo(regNum, VRegInfo::Location::REGISTER, IrTypeToMetainfoType(inst->GetType()),
1054 vreg.GetVRegType()));
1055 }
1056 } else {
1057 ASSERT(regNum >= GetFirstCallerReg(GetArch(), isFp));
1058 auto lastSlot = GetFrameLayout().GetCallerLastSlot(isFp);
1059 RegMask mask(GetCallerRegsMask(GetArch(), isFp));
1060 regNum = mask.GetDistanceFromTail(regNum);
1061 codeBuilder_->AddVReg(VRegInfo(lastSlot - regNum, VRegInfo::Location::SLOT,
1062 IrTypeToMetainfoType(inst->GetType()), vreg.GetVRegType()));
1063 }
1064 }
1065
CreateOsrEntry(SaveStateInst * saveState)1066 void Codegen::CreateOsrEntry(SaveStateInst *saveState)
1067 {
1068 auto &stub = osrEntries_.emplace_back(GetAllocator()->New<OsrEntryStub>(this, saveState));
1069 GetEncoder()->BindLabel(stub->GetLabel());
1070 }
1071
CallIntrinsic(Inst * inst,RuntimeInterface::IntrinsicId id)1072 void Codegen::CallIntrinsic(Inst *inst, RuntimeInterface::IntrinsicId id)
1073 {
1074 SCOPED_DISASM_STR(this, "CallIntrinsic");
1075 // Call intrinsics isn't supported for IrToc, because we don't know address of intrinsics during IRToc compilation.
1076 // We store adresses of intrinsics in aot file in AOT mode.
1077 ASSERT(GetGraph()->SupportManagedCode());
1078 if (GetGraph()->IsAotMode()) {
1079 auto aotData = GetGraph()->GetAotData();
1080 intptr_t offset = aotData->GetEntrypointOffset(GetEncoder()->GetCursorOffset(), static_cast<int32_t>(id));
1081 GetEncoder()->MakeCallAot(offset);
1082 } else {
1083 GetEncoder()->MakeCall(reinterpret_cast<const void *>(GetRuntime()->GetIntrinsicAddress(
1084 inst->IsRuntimeCall(), GetRuntime()->GetMethodSourceLanguage(GetGraph()->GetMethod()), id)));
1085 }
1086 }
1087
EmitCallRuntimeCode(Inst * inst,std::variant<EntrypointId,Reg> entrypoint)1088 bool Codegen::EmitCallRuntimeCode(Inst *inst, std::variant<EntrypointId, Reg> entrypoint)
1089 {
1090 auto encoder = GetEncoder();
1091 if (std::holds_alternative<Reg>(entrypoint)) {
1092 auto reg = std::get<Reg>(entrypoint);
1093 ASSERT(reg.IsValid());
1094 encoder->MakeCall(reg);
1095 } else {
1096 auto id = std::get<EntrypointId>(entrypoint);
1097 MemRef entry(ThreadReg(), GetRuntime()->GetEntrypointTlsOffset(GetArch(), id));
1098 encoder->MakeCall(entry);
1099 }
1100
1101 SaveStateInst *saveState =
1102 (inst == nullptr || inst->IsSaveState()) ? static_cast<SaveStateInst *>(inst) : inst->GetSaveState();
1103 // StackMap should follow the call as the bridge function expects retaddr points to the stackmap
1104 if (saveState != nullptr) {
1105 CreateStackMap(inst);
1106 }
1107
1108 if (std::holds_alternative<EntrypointId>(entrypoint) &&
1109 GetRuntime()->IsEntrypointNoreturn(std::get<EntrypointId>(entrypoint))) {
1110 if constexpr (DEBUG_BUILD) { // NOLINT
1111 encoder->EncodeAbort();
1112 }
1113 return false;
1114 }
1115 ASSERT(saveState == nullptr || inst->IsRuntimeCall());
1116
1117 return true;
1118 }
1119
SaveRegistersForImplicitRuntime(Inst * inst,RegMask * paramsMask,RegMask * mask)1120 void Codegen::SaveRegistersForImplicitRuntime(Inst *inst, RegMask *paramsMask, RegMask *mask)
1121 {
1122 auto &rootsMask = inst->GetSaveState()->GetRootsRegsMask();
1123 for (auto i = 0U; i < inst->GetInputsCount(); i++) {
1124 auto input = inst->GetDataFlowInput(i);
1125 if (!input->IsMovableObject()) {
1126 continue;
1127 }
1128 auto location = inst->GetLocation(i);
1129 if (location.IsRegister() && location.IsRegisterValid()) {
1130 auto val = location.GetValue();
1131 auto reg = Reg(val, INT64_TYPE);
1132 GetEncoder()->SetRegister(mask, nullptr, reg, true);
1133 if (DataType::IsReference(inst->GetInputType(i)) && !rootsMask.test(val)) {
1134 paramsMask->set(val);
1135 rootsMask.set(val);
1136 }
1137 }
1138 }
1139 }
1140
CreateCheckForTLABWithConstSize(Inst * inst,Reg regTlabStart,Reg regTlabSize,size_t size,LabelHolder::LabelId label)1141 void Codegen::CreateCheckForTLABWithConstSize([[maybe_unused]] Inst *inst, Reg regTlabStart, Reg regTlabSize,
1142 size_t size, LabelHolder::LabelId label)
1143 {
1144 SCOPED_DISASM_STR(this, "CreateCheckForTLABWithConstSize");
1145 auto encoder = GetEncoder();
1146 if (encoder->CanEncodeImmAddSubCmp(size, WORD_SIZE, false)) {
1147 encoder->EncodeJump(label, regTlabSize, Imm(size), Condition::LO);
1148 // Change pointer to start free memory
1149 encoder->EncodeAdd(regTlabSize, regTlabStart, Imm(size));
1150 } else {
1151 ScopedTmpReg sizeReg(encoder);
1152 encoder->EncodeMov(sizeReg, Imm(size));
1153 encoder->EncodeJump(label, regTlabSize, sizeReg, Condition::LO);
1154 encoder->EncodeAdd(regTlabSize, regTlabStart, sizeReg);
1155 }
1156 }
1157
CreateDebugRuntimeCallsForNewObject(Inst * inst,Reg regTlabStart,size_t allocSize,RegMask preserved)1158 void Codegen::CreateDebugRuntimeCallsForNewObject(Inst *inst, [[maybe_unused]] Reg regTlabStart, size_t allocSize,
1159 RegMask preserved)
1160 {
1161 #if defined(PANDA_ASAN_ON) || defined(PANDA_TSAN_ON)
1162 CallRuntime(inst, EntrypointId::ANNOTATE_SANITIZERS, INVALID_REGISTER, preserved, regTlabStart,
1163 TypedImm(allocSize));
1164 #endif
1165 if (GetRuntime()->IsTrackTlabAlloc()) {
1166 CallRuntime(inst, EntrypointId::WRITE_TLAB_STATS, INVALID_REGISTER, preserved, regTlabStart,
1167 TypedImm(allocSize));
1168 }
1169 }
1170
CreateNewObjCall(NewObjectInst * newObj)1171 void Codegen::CreateNewObjCall(NewObjectInst *newObj)
1172 {
1173 auto dst = ConvertRegister(newObj->GetDstReg(), newObj->GetType());
1174 auto src = ConvertRegister(newObj->GetSrcReg(0), newObj->GetInputType(0));
1175 auto initClass = newObj->GetInput(0).GetInst();
1176 auto srcClass = ConvertRegister(newObj->GetSrcReg(0), DataType::POINTER);
1177 auto runtime = GetRuntime();
1178
1179 auto maxTlabSize = runtime->GetTLABMaxSize();
1180 if (maxTlabSize == 0 ||
1181 (initClass->GetOpcode() != Opcode::LoadAndInitClass && initClass->GetOpcode() != Opcode::LoadImmediate)) {
1182 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1183 return;
1184 }
1185 RuntimeInterface::ClassPtr klass;
1186 if (initClass->GetOpcode() == Opcode::LoadAndInitClass) {
1187 klass = initClass->CastToLoadAndInitClass()->GetClass();
1188 } else {
1189 ASSERT(initClass->GetOpcode() == Opcode::LoadImmediate);
1190 klass = initClass->CastToLoadImmediate()->GetClass();
1191 }
1192 if (klass == nullptr || !runtime->CanUseTlabForClass(klass)) {
1193 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1194 return;
1195 }
1196 auto classSize = runtime->GetClassSize(klass);
1197 auto alignment = runtime->GetTLABAlignment();
1198
1199 classSize = (classSize & ~(alignment - 1U)) + ((classSize % alignment) != 0U ? alignment : 0U);
1200 if (classSize > maxTlabSize) {
1201 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1202 return;
1203 }
1204 CallFastPath(newObj, EntrypointId::ALLOCATE_OBJECT_TLAB, dst, RegMask::GetZeroMask(), srcClass,
1205 TypedImm(classSize));
1206 }
1207
CreateNewObjCallOld(NewObjectInst * newObj)1208 void Codegen::CreateNewObjCallOld(NewObjectInst *newObj)
1209 {
1210 auto dst = ConvertRegister(newObj->GetDstReg(), newObj->GetType());
1211 auto src = ConvertRegister(newObj->GetSrcReg(0), newObj->GetInputType(0));
1212 auto initClass = newObj->GetInput(0).GetInst();
1213 auto runtime = GetRuntime();
1214 auto maxTlabSize = runtime->GetTLABMaxSize();
1215 auto encoder = GetEncoder();
1216
1217 if (maxTlabSize == 0 ||
1218 (initClass->GetOpcode() != Opcode::LoadAndInitClass && initClass->GetOpcode() != Opcode::LoadImmediate)) {
1219 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1220 return;
1221 }
1222 RuntimeInterface::ClassPtr klass;
1223 if (initClass->GetOpcode() == Opcode::LoadAndInitClass) {
1224 klass = initClass->CastToLoadAndInitClass()->GetClass();
1225 } else {
1226 ASSERT(initClass->GetOpcode() == Opcode::LoadImmediate);
1227 klass = initClass->CastToLoadImmediate()->GetClass();
1228 }
1229 if (klass == nullptr || !runtime->CanUseTlabForClass(klass)) {
1230 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1231 return;
1232 }
1233 auto classSize = runtime->GetClassSize(klass);
1234 auto alignment = runtime->GetTLABAlignment();
1235
1236 classSize = (classSize & ~(alignment - 1U)) + ((classSize % alignment) != 0U ? alignment : 0U);
1237 if (classSize > maxTlabSize) {
1238 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1239 return;
1240 }
1241 ScopedLiveTmpReg regTlabStart(encoder);
1242 ScopedLiveTmpReg regTlabSize(encoder);
1243
1244 auto slowPath = CreateSlowPath<SlowPathEntrypoint>(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS);
1245 CreateLoadTLABInformation(regTlabStart, regTlabSize);
1246 CreateCheckForTLABWithConstSize(newObj, regTlabStart, regTlabSize, classSize, slowPath->GetLabel());
1247
1248 RegMask preservedRegs;
1249 encoder->SetRegister(&preservedRegs, nullptr, src);
1250 CreateDebugRuntimeCallsForNewObject(newObj, regTlabStart, reinterpret_cast<size_t>(classSize), preservedRegs);
1251
1252 ScopedTmpReg tlabReg(encoder);
1253 // Load pointer to tlab
1254 encoder->EncodeLdr(tlabReg, false, MemRef(ThreadReg(), runtime->GetCurrentTLABOffset(GetArch())));
1255
1256 // Store pointer to the class
1257 encoder->EncodeStr(src, MemRef(regTlabStart, runtime->GetObjClassOffset(GetArch())));
1258 encoder->EncodeMov(dst, regTlabStart);
1259 regTlabStart.Release();
1260 // Store new pointer to start free memory in TLAB
1261 encoder->EncodeStrRelease(regTlabSize, MemRef(tlabReg, runtime->GetTLABFreePointerOffset(GetArch())));
1262 slowPath->BindBackLabel(encoder);
1263 }
1264
LoadClassFromObject(Reg classReg,Reg objReg)1265 void Codegen::LoadClassFromObject(Reg classReg, Reg objReg)
1266 {
1267 Reg reg = ConvertRegister(classReg.GetId(), DataType::REFERENCE);
1268 GetEncoder()->EncodeLdr(reg, false, MemRef(objReg, GetRuntime()->GetObjClassOffset(GetArch())));
1269 }
1270
CreateMultiArrayCall(CallInst * callInst)1271 void Codegen::CreateMultiArrayCall(CallInst *callInst)
1272 {
1273 SCOPED_DISASM_STR(this, "Create Call for MultiArray");
1274
1275 auto dstReg = ConvertRegister(callInst->GetDstReg(), callInst->GetType());
1276 auto numArgs = callInst->GetInputsCount() - 2U; // first is class, last is save_state
1277
1278 ScopedTmpReg classReg(GetEncoder());
1279 auto classType = ConvertDataType(DataType::REFERENCE, GetArch());
1280 Reg classOrig = classReg.GetReg().As(classType);
1281 auto location = callInst->GetLocation(0);
1282 ASSERT(location.IsFixedRegister() && location.IsRegisterValid());
1283
1284 GetEncoder()->EncodeMov(classOrig, ConvertRegister(location.GetValue(), DataType::INT32));
1285 CallRuntime(callInst, EntrypointId::CREATE_MULTI_ARRAY, dstReg, RegMask::GetZeroMask(), classReg, TypedImm(numArgs),
1286 SpReg());
1287 if (callInst->GetFlag(inst_flags::MEM_BARRIER)) {
1288 GetEncoder()->EncodeMemoryBarrier(memory_order::RELEASE);
1289 }
1290 }
1291
CreateJumpToClassResolverPltShared(Inst * inst,Reg tmpReg,RuntimeInterface::EntrypointId id)1292 void Codegen::CreateJumpToClassResolverPltShared(Inst *inst, Reg tmpReg, RuntimeInterface::EntrypointId id)
1293 {
1294 auto encoder = GetEncoder();
1295 auto graph = GetGraph();
1296 auto aotData = graph->GetAotData();
1297 auto offset = aotData->GetSharedSlowPathOffset(id, encoder->GetCursorOffset());
1298 if (offset == 0 || !encoder->CanMakeCallByOffset(offset)) {
1299 SlowPathShared *slowPath;
1300 auto search = slowPathsMap_.find(id);
1301 if (search != slowPathsMap_.end()) {
1302 slowPath = search->second;
1303 ASSERT(slowPath->GetTmpReg().GetId() == tmpReg.GetId());
1304 } else {
1305 slowPath = CreateSlowPath<SlowPathShared>(inst, id);
1306 slowPath->SetTmpReg(tmpReg);
1307 slowPathsMap_[id] = slowPath;
1308 }
1309 encoder->MakeCall(slowPath->GetLabel());
1310 } else {
1311 encoder->MakeCallByOffset(offset);
1312 }
1313 CreateStackMap(inst);
1314 }
1315
CreateLoadClassFromPLT(Inst * inst,Reg tmpReg,Reg dst,size_t classId)1316 void Codegen::CreateLoadClassFromPLT(Inst *inst, Reg tmpReg, Reg dst, size_t classId)
1317 {
1318 auto encoder = GetEncoder();
1319 auto graph = GetGraph();
1320 auto aotData = graph->GetAotData();
1321 intptr_t offset = aotData->GetClassSlotOffset(encoder->GetCursorOffset(), classId, false);
1322 auto label = encoder->CreateLabel();
1323 ASSERT(tmpReg.GetId() != dst.GetId());
1324 ASSERT(inst->IsRuntimeCall());
1325 encoder->MakeLoadAotTableAddr(offset, tmpReg, dst);
1326 encoder->EncodeJump(label, dst, Condition::NE);
1327
1328 // PLT Class Resolver has special calling convention:
1329 // First encoder temporary (tmp_reg) works as parameter and return value
1330 CHECK_EQ(tmpReg.GetId(), encoder->GetTarget().GetTempRegsMask().GetMinRegister());
1331
1332 CreateJumpToClassResolverPltShared(inst, tmpReg, EntrypointId::CLASS_RESOLVER);
1333
1334 encoder->EncodeMov(dst, tmpReg);
1335 encoder->BindLabel(label);
1336 }
1337
CreateLoadTLABInformation(Reg regTlabStart,Reg regTlabSize)1338 void Codegen::CreateLoadTLABInformation(Reg regTlabStart, Reg regTlabSize)
1339 {
1340 SCOPED_DISASM_STR(this, "LoadTLABInformation");
1341 auto runtime = GetRuntime();
1342 // Load pointer to tlab
1343 GetEncoder()->EncodeLdr(regTlabSize, false, MemRef(ThreadReg(), runtime->GetCurrentTLABOffset(GetArch())));
1344 // Load pointer to start free memory in TLAB
1345 GetEncoder()->EncodeLdr(regTlabStart, false, MemRef(regTlabSize, runtime->GetTLABFreePointerOffset(GetArch())));
1346 // Load pointer to end free memory in TLAB
1347 GetEncoder()->EncodeLdr(regTlabSize, false, MemRef(regTlabSize, runtime->GetTLABEndPointerOffset(GetArch())));
1348 // Calculate size of the free memory
1349 GetEncoder()->EncodeSub(regTlabSize, regTlabSize, regTlabStart);
1350 }
1351
1352 // The function alignment up the value from alignment_reg using tmp_reg.
CreateAlignmentValue(Reg alignmentReg,Reg tmpReg,size_t alignment)1353 void Codegen::CreateAlignmentValue(Reg alignmentReg, Reg tmpReg, size_t alignment)
1354 {
1355 auto andVal = ~(alignment - 1U);
1356 // zeroed lower bits
1357 GetEncoder()->EncodeAnd(tmpReg, alignmentReg, Imm(andVal));
1358 GetEncoder()->EncodeSub(alignmentReg, alignmentReg, tmpReg);
1359
1360 auto endLabel = GetEncoder()->CreateLabel();
1361
1362 // if zeroed value is different, add alignment
1363 GetEncoder()->EncodeJump(endLabel, alignmentReg, Condition::EQ);
1364 GetEncoder()->EncodeAdd(tmpReg, tmpReg, Imm(alignment));
1365
1366 GetEncoder()->BindLabel(endLabel);
1367 GetEncoder()->EncodeMov(alignmentReg, tmpReg);
1368 }
1369
GetObjectReg(Codegen * codegen,Inst * inst)1370 Reg GetObjectReg(Codegen *codegen, Inst *inst)
1371 {
1372 ASSERT(inst->IsCall() || inst->GetOpcode() == Opcode::ResolveVirtual);
1373 auto location = inst->GetLocation(0);
1374 ASSERT(location.IsFixedRegister() && location.IsRegisterValid());
1375 auto objReg = codegen->ConvertRegister(location.GetValue(), inst->GetInputType(0));
1376 ASSERT(objReg != INVALID_REGISTER);
1377 return objReg;
1378 }
1379
CreateCallIntrinsic(IntrinsicInst * inst)1380 void Codegen::CreateCallIntrinsic(IntrinsicInst *inst)
1381 {
1382 if (inst->HasArgumentsOnStack()) {
1383 // Since for some intrinsics(f.e. TimedWaitNanos) we need SaveState instruction and
1384 // more than two arguments, we need to place arguments on the stack, but in same time we need to
1385 // create boundary frame
1386 LOG(WARNING, COMPILER) << "Intrinsics with arguments on stack are not supported";
1387 GetEncoder()->SetFalseResult();
1388 return;
1389 }
1390
1391 ASSERT(!HasLiveCallerSavedRegs(inst));
1392 if (inst->HasImms() && GetGraph()->SupportManagedCode()) {
1393 EncodeImms(inst->GetImms(), (inst->HasIdInput() || inst->IsMethodFirstInput()));
1394 }
1395
1396 size_t explicitArgs;
1397 if (IsStackRangeIntrinsic(inst->GetIntrinsicId(), &explicitArgs)) {
1398 auto paramInfo = GetCallingConvention()->GetParameterInfo(explicitArgs);
1399 auto rangePtrReg =
1400 ConvertRegister(paramInfo->GetNextLocation(DataType::POINTER).GetRegister(), DataType::POINTER);
1401 if (inst->GetInputsCount() > explicitArgs + (inst->RequireState() ? 1U : 0U)) {
1402 auto rangeSpOffs = GetStackOffset(inst->GetLocation(explicitArgs));
1403 GetEncoder()->EncodeAdd(rangePtrReg, GetTarget().GetStackReg(), Imm(rangeSpOffs));
1404 }
1405 }
1406 if (inst->IsMethodFirstInput()) {
1407 Reg param0 = GetTarget().GetParamReg(0);
1408 if (GetGraph()->IsAotMode()) {
1409 LoadMethod(param0);
1410 } else {
1411 GetEncoder()->EncodeMov(param0, Imm(reinterpret_cast<uintptr_t>(inst->GetMethod())));
1412 }
1413 }
1414 CallIntrinsic(inst, inst->GetIntrinsicId());
1415
1416 if (inst->GetSaveState() != nullptr) {
1417 // StackMap should follow the call as the bridge function expects retaddr points to the stackmap
1418 CreateStackMap(inst);
1419 }
1420
1421 if (inst->GetType() != DataType::VOID) {
1422 auto arch = GetArch();
1423 auto returnType = inst->GetType();
1424 auto dstReg = ConvertRegister(inst->GetDstReg(), inst->GetType());
1425 auto returnReg = GetTarget().GetReturnReg(dstReg.GetType());
1426 // We must:
1427 // sign extended INT8 and INT16 to INT32
1428 // zero extended UINT8 and UINT16 to UINT32
1429 if (DataType::ShiftByType(returnType, arch) < DataType::ShiftByType(DataType::INT32, arch)) {
1430 bool isSigned = DataType::IsTypeSigned(returnType);
1431 GetEncoder()->EncodeCast(dstReg, isSigned, Reg(returnReg.GetId(), INT32_TYPE), isSigned);
1432 } else {
1433 GetEncoder()->EncodeMov(dstReg, returnReg);
1434 }
1435 }
1436 }
1437
IntfInlineCachePass(ResolveVirtualInst * resolver,Reg methodReg,Reg tmpReg,Reg objReg)1438 void Codegen::IntfInlineCachePass(ResolveVirtualInst *resolver, Reg methodReg, Reg tmpReg, Reg objReg)
1439 {
1440 // Cache structure:（offset addr)/(class addr) 32bit/32bit
1441 // -----------------------------------------------
1442 // (.aot_got)
1443 // ...
1444 // cache:offset/class <----------|
1445 // ... |
1446 // (.text) |
1447 // interface call start |
1448 // call runtime irtoc function
1449 // if call class == cache.class <------|
1450 // use cache.offset(method) <------|
1451 // else |
1452 // call RESOLVE_VIRTUAL_CALL_AOT |
1453 // save method‘s offset to cache >--|
1454 // return to (.text)
1455 // call method
1456 // -----------------------------------------------
1457 auto aotData = GetGraph()->GetAotData();
1458 uint64_t intfInlineCacheIndex = aotData->GetIntfInlineCacheIndex();
1459 // NOTE(liyiming): do LoadMethod in irtoc to reduce use tmp reg
1460 if (objReg.GetId() != tmpReg.GetId()) {
1461 auto regTmp64 = tmpReg.As(INT64_TYPE);
1462 uint64_t offset = aotData->GetInfInlineCacheSlotOffset(GetEncoder()->GetCursorOffset(), intfInlineCacheIndex);
1463 GetEncoder()->MakeLoadAotTableAddr(offset, regTmp64, INVALID_REGISTER);
1464 LoadMethod(methodReg);
1465 CallFastPath(resolver, EntrypointId::INTF_INLINE_CACHE, methodReg, {}, methodReg, objReg,
1466 TypedImm(resolver->GetCallMethodId()), regTmp64);
1467 } else {
1468 // we don't have tmp reg here, so use x3 directly
1469 Reg reg3 = Reg(3U, INT64_TYPE);
1470 ScopedTmpRegF64 vtmp(GetEncoder());
1471 GetEncoder()->EncodeMov(vtmp, reg3);
1472 uint64_t offset = aotData->GetInfInlineCacheSlotOffset(GetEncoder()->GetCursorOffset(), intfInlineCacheIndex);
1473 GetEncoder()->MakeLoadAotTableAddr(offset, reg3, INVALID_REGISTER);
1474 LoadMethod(methodReg);
1475 CallFastPath(resolver, EntrypointId::INTF_INLINE_CACHE, methodReg, {}, methodReg, objReg,
1476 TypedImm(resolver->GetCallMethodId()), reg3);
1477 GetEncoder()->EncodeMov(reg3, vtmp);
1478 }
1479
1480 intfInlineCacheIndex++;
1481 aotData->SetIntfInlineCacheIndex(intfInlineCacheIndex);
1482 }
1483
1484 /**
1485 * Returns a pointer to a caller of a virtual/static resolver.
1486 *
1487 * InstBuilder uses UnresolvedTypesInterface::AddTableSlot(method_ptr, ...) to allocate cache slots
1488 * for unresolved methods. The codegen uses UnresolvedTypesInterface::GetTableSlot(method_ptr, ...)
1489 * to get corresponding slots for unresolved methods.
1490 *
1491 * Since there is no valid method_ptr for an unresolved method at the point of IR construction
1492 * InstBuilder::BuildCallInst uses the pointer to the caller (GetGraph()->GetMethod()) as an argument
1493 * of UnresolvedTypesInterface::AddTableSlot. So the codegen has to deduce the pointer to the caller
1494 * and pass it to UnresolvedTypesInterface::GetTableSlot. If the caller was not inlined then
1495 * GetGraph()->GetMethod() works just fine, otherwise it is done by means of SaveState,
1496 * which is capable of remebering the corresponding caller instruction.
1497 *
1498 * To clarify the details let's consider the following sequence of calls:
1499 *
1500 * main() -> foo() -> bar()
1501 *
1502 * Suppose that foo() is a static method and bar() is an unresolved virtual method.
1503 *
1504 * 1) foo() is going to be inlined
1505 *
1506 * To inline foo() the inlining pass creates its IR by means of IRBuilder(..., caller_inst),
1507 * where caller_inst (the third argument) represents "CallStatic foo" instruction.
1508 * The caller_inst gets remembered in the corresponding SaveState created for bar's resolver
1509 * (see InstBuilder::CreateSaveState for the details).
1510 * At the same time "CallStatic foo" instruction contains a valid method pointer to foo()
1511 * (it can not be nullptr because we do not inline Unresolved).
1512 *
1513 * So we go through the following chain: bar's resolver->SaveState->caller_inst->pointer to the method.
1514 *
1515 * 2) foo() is not inlined
1516 *
1517 * In this case SaveState.caller_inst_ == nullptr (because foo() is not inlined).
1518 * Thus we just use plain GetGraph()->GetMethod().
1519 */
1520 template <typename T>
GetCallerOfUnresolvedMethod(T * resolver)1521 RuntimeInterface::MethodPtr Codegen::GetCallerOfUnresolvedMethod(T *resolver)
1522 {
1523 ASSERT(resolver->GetCallMethod() == nullptr);
1524 auto saveState = resolver->GetSaveState();
1525 ASSERT(saveState != nullptr);
1526 auto caller = saveState->GetCallerInst();
1527 auto method = (caller == nullptr ? GetGraph()->GetMethod() : caller->GetCallMethod());
1528 ASSERT(method != nullptr);
1529 return method;
1530 }
1531
EmitResolveUnknownVirtual(ResolveVirtualInst * resolver,Reg methodReg)1532 void Codegen::EmitResolveUnknownVirtual(ResolveVirtualInst *resolver, Reg methodReg)
1533 {
1534 SCOPED_DISASM_STR(this, "Create runtime call to resolve an unknown virtual method");
1535 ASSERT(resolver->GetOpcode() == Opcode::ResolveVirtual);
1536 auto method = GetCallerOfUnresolvedMethod(resolver);
1537 ScopedTmpReg tmpReg(GetEncoder(), ConvertDataType(DataType::REFERENCE, GetArch()));
1538 Reg objReg = GetObjectReg(this, resolver);
1539 if (GetGraph()->IsAotMode()) {
1540 LoadMethod(methodReg);
1541 CallRuntime(resolver, EntrypointId::RESOLVE_VIRTUAL_CALL_AOT, methodReg, {}, methodReg, objReg,
1542 TypedImm(resolver->GetCallMethodId()), TypedImm(0));
1543 } else {
1544 auto runtime = GetRuntime();
1545 auto utypes = runtime->GetUnresolvedTypes();
1546 auto skind = UnresolvedTypesInterface::SlotKind::VIRTUAL_METHOD;
1547 // Try to load vtable index
1548 auto methodSlotAddr = utypes->GetTableSlot(method, resolver->GetCallMethodId(), skind);
1549 GetEncoder()->EncodeMov(methodReg, Imm(methodSlotAddr));
1550 GetEncoder()->EncodeLdr(methodReg, false, MemRef(methodReg));
1551 // 0 means the virtual call is uninitialized or it is an interface call
1552 auto entrypoint = EntrypointId::RESOLVE_UNKNOWN_VIRTUAL_CALL;
1553 auto slowPath = CreateSlowPath<SlowPathUnresolved>(resolver, entrypoint);
1554 slowPath->SetUnresolvedType(method, resolver->GetCallMethodId());
1555 slowPath->SetDstReg(methodReg);
1556 slowPath->SetArgReg(objReg);
1557 slowPath->SetSlotAddr(methodSlotAddr);
1558 GetEncoder()->EncodeJump(slowPath->GetLabel(), methodReg, Condition::EQ);
1559 // Load klass into tmp_reg
1560 LoadClassFromObject(tmpReg, objReg);
1561 // Load from VTable, address = (klass + (index << shift)) + vtable_offset
1562 auto tmpReg64 = Reg(tmpReg.GetReg().GetId(), INT64_TYPE);
1563 GetEncoder()->EncodeAdd(tmpReg64, tmpReg, Shift(methodReg, GetVtableShift()));
1564 GetEncoder()->EncodeLdr(methodReg, false,
1565 MemRef(tmpReg64, runtime->GetVTableOffset(GetArch()) - (1U << GetVtableShift())));
1566 slowPath->BindBackLabel(GetEncoder());
1567 }
1568 }
1569
EmitResolveVirtualAot(ResolveVirtualInst * resolver,Reg methodReg)1570 void Codegen::EmitResolveVirtualAot(ResolveVirtualInst *resolver, Reg methodReg)
1571 {
1572 SCOPED_DISASM_STR(this, "AOT ResolveVirtual using PLT-GOT");
1573 ASSERT(resolver->IsRuntimeCall());
1574 ScopedTmpReg tmpReg(GetEncoder(), ConvertDataType(DataType::REFERENCE, GetArch()));
1575 auto methodReg64 = Reg(methodReg.GetId(), INT64_TYPE);
1576 auto tmpReg64 = Reg(tmpReg.GetReg().GetId(), INT64_TYPE);
1577 auto aotData = GetGraph()->GetAotData();
1578 intptr_t offset = aotData->GetVirtIndexSlotOffset(GetEncoder()->GetCursorOffset(), resolver->GetCallMethodId());
1579 GetEncoder()->MakeLoadAotTableAddr(offset, tmpReg64, methodReg64);
1580 auto label = GetEncoder()->CreateLabel();
1581 GetEncoder()->EncodeJump(label, methodReg, Condition::NE);
1582 GetEncoder()->EncodeMov(methodReg64, tmpReg64);
1583 // PLT CallVirtual Resolver has a very special calling convention:
1584 // First encoder temporary (method_reg) works as a parameter and return value
1585 CHECK_EQ(methodReg64.GetId(), GetTarget().GetTempRegsMask().GetMinRegister());
1586 MemRef entry(ThreadReg(), GetRuntime()->GetEntrypointTlsOffset(GetArch(), EntrypointId::CALL_VIRTUAL_RESOLVER));
1587 GetEncoder()->MakeCall(entry);
1588 // Need a stackmap to build correct boundary frame
1589 CreateStackMap(resolver);
1590 GetEncoder()->BindLabel(label);
1591 // Load class into method_reg
1592 Reg objReg = GetObjectReg(this, resolver);
1593 LoadClassFromObject(tmpReg, objReg);
1594 // Load from VTable, address = (klass + (index << shift)) + vtable_offset
1595 GetEncoder()->EncodeAdd(methodReg, tmpReg64, Shift(methodReg, GetVtableShift()));
1596 GetEncoder()->EncodeLdr(methodReg, false,
1597 MemRef(methodReg, GetRuntime()->GetVTableOffset(GetArch()) - (1U << GetVtableShift())));
1598 }
1599
EmitResolveVirtual(ResolveVirtualInst * resolver)1600 void Codegen::EmitResolveVirtual(ResolveVirtualInst *resolver)
1601 {
1602 auto methodReg = ConvertRegister(resolver->GetDstReg(), resolver->GetType());
1603 auto objectReg = ConvertRegister(resolver->GetSrcReg(0), DataType::REFERENCE);
1604 ScopedTmpReg tmpMethodReg(GetEncoder());
1605 if (resolver->GetCallMethod() == nullptr) {
1606 EmitResolveUnknownVirtual(resolver, tmpMethodReg);
1607 GetEncoder()->EncodeMov(methodReg, tmpMethodReg);
1608 } else if (GetRuntime()->IsInterfaceMethod(resolver->GetCallMethod())) {
1609 SCOPED_DISASM_STR(this, "Create runtime call to resolve a known virtual call");
1610 if (GetGraph()->IsAotMode()) {
1611 if (GetArch() == Arch::AARCH64) {
1612 ScopedTmpReg tmpReg(GetEncoder(), ConvertDataType(DataType::REFERENCE, GetArch()));
1613 IntfInlineCachePass(resolver, tmpMethodReg, tmpReg, objectReg);
1614 } else {
1615 LoadMethod(tmpMethodReg);
1616 CallRuntime(resolver, EntrypointId::RESOLVE_VIRTUAL_CALL_AOT, tmpMethodReg, {}, tmpMethodReg, objectReg,
1617 TypedImm(resolver->GetCallMethodId()), TypedImm(0));
1618 }
1619 } else {
1620 CallRuntime(resolver, EntrypointId::RESOLVE_VIRTUAL_CALL, tmpMethodReg, {},
1621 TypedImm(reinterpret_cast<size_t>(resolver->GetCallMethod())), objectReg);
1622 }
1623 GetEncoder()->EncodeMov(methodReg, tmpMethodReg);
1624 } else if (GetGraph()->IsAotNoChaMode()) {
1625 // ResolveVirtualAot requires method_reg to be the first tmp register.
1626 EmitResolveVirtualAot(resolver, tmpMethodReg);
1627 GetEncoder()->EncodeMov(methodReg, tmpMethodReg);
1628 } else {
1629 UNREACHABLE();
1630 }
1631 }
1632
EmitCallResolvedVirtual(CallInst * call)1633 void Codegen::EmitCallResolvedVirtual(CallInst *call)
1634 {
1635 SCOPED_DISASM_STR(this, "Create a call to resolved virtual method");
1636 ASSERT(call->GetOpcode() == Opcode::CallResolvedVirtual);
1637 if (call->GetSaveState() != nullptr && call->IsInlined()) {
1638 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
1639 if (!GetGraph()->IsAotMode()) {
1640 InsertTrace({Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
1641 Imm(reinterpret_cast<size_t>(call->GetCallMethod())),
1642 Imm(static_cast<size_t>(events::MethodEnterKind::INLINED))});
1643 }
1644 #endif
1645 return;
1646 }
1647 ASSERT(!HasLiveCallerSavedRegs(call));
1648 ASSERT(call->GetCallMethod() == nullptr || GetGraph()->GetRuntime()->IsInterfaceMethod(call->GetCallMethod()) ||
1649 GetGraph()->IsAotNoChaMode());
1650 Reg methodReg = ConvertRegister(call->GetSrcReg(0), DataType::POINTER);
1651 Reg param0 = GetTarget().GetParamReg(0);
1652 // Set method
1653 GetEncoder()->EncodeMov(param0, methodReg);
1654 GetEncoder()->MakeCall(MemRef(param0, GetRuntime()->GetCompiledEntryPointOffset(GetArch())));
1655 FinalizeCall(call);
1656 }
1657
EmitCallVirtual(CallInst * call)1658 void Codegen::EmitCallVirtual(CallInst *call)
1659 {
1660 SCOPED_DISASM_STR(this, "Create a call to virtual method");
1661 ASSERT(call->GetOpcode() == Opcode::CallVirtual);
1662 if (call->GetSaveState() != nullptr && call->IsInlined()) {
1663 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
1664 if (!GetGraph()->IsAotMode()) {
1665 InsertTrace({Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
1666 Imm(reinterpret_cast<size_t>(call->GetCallMethod())),
1667 Imm(static_cast<size_t>(events::MethodEnterKind::INLINED))});
1668 }
1669 #endif
1670 return;
1671 }
1672 auto runtime = GetRuntime();
1673 auto method = call->GetCallMethod();
1674 ASSERT(!HasLiveCallerSavedRegs(call));
1675 ASSERT(!call->IsUnresolved() && !runtime->IsInterfaceMethod(method) && !GetGraph()->IsAotNoChaMode());
1676 Reg methodReg = GetTarget().GetParamReg(0);
1677 ASSERT(!RegisterKeepCallArgument(call, methodReg));
1678 LoadClassFromObject(methodReg, GetObjectReg(this, call));
1679 // Get index
1680 auto vtableIndex = runtime->GetVTableIndex(method);
1681 // Load from VTable, address = klass + ((index << shift) + vtable_offset)
1682 auto totalOffset = runtime->GetVTableOffset(GetArch()) + (vtableIndex << GetVtableShift());
1683 // Class ref was loaded to method_reg
1684 GetEncoder()->EncodeLdr(methodReg, false, MemRef(methodReg, totalOffset));
1685 // Set method
1686 GetEncoder()->MakeCall(MemRef(methodReg, runtime->GetCompiledEntryPointOffset(GetArch())));
1687 FinalizeCall(call);
1688 }
1689
EnsureParamsFitIn32Bit(std::initializer_list<std::variant<Reg,TypedImm>> params)1690 bool Codegen::EnsureParamsFitIn32Bit(std::initializer_list<std::variant<Reg, TypedImm>> params)
1691 {
1692 for (auto ¶m : params) {
1693 if (std::holds_alternative<Reg>(param)) {
1694 auto reg = std::get<Reg>(param);
1695 if (reg.GetSize() > WORD_SIZE) {
1696 return false;
1697 }
1698 } else {
1699 auto immType = std::get<TypedImm>(param).GetType();
1700 if (immType.GetSize() > WORD_SIZE) {
1701 return false;
1702 }
1703 }
1704 }
1705 return true;
1706 }
1707
EmitResolveStatic(ResolveStaticInst * resolver)1708 void Codegen::EmitResolveStatic(ResolveStaticInst *resolver)
1709 {
1710 auto methodReg = ConvertRegister(resolver->GetDstReg(), resolver->GetType());
1711 if (GetGraph()->IsAotMode()) {
1712 LoadMethod(methodReg);
1713 CallRuntime(resolver, EntrypointId::GET_UNKNOWN_CALLEE_METHOD, methodReg, RegMask::GetZeroMask(), methodReg,
1714 TypedImm(resolver->GetCallMethodId()), TypedImm(0));
1715 return;
1716 }
1717 auto method = GetCallerOfUnresolvedMethod(resolver);
1718 ScopedTmpReg tmp(GetEncoder());
1719 auto utypes = GetRuntime()->GetUnresolvedTypes();
1720 auto skind = UnresolvedTypesInterface::SlotKind::METHOD;
1721 auto methodAddr = utypes->GetTableSlot(method, resolver->GetCallMethodId(), skind);
1722 GetEncoder()->EncodeMov(tmp.GetReg(), Imm(methodAddr));
1723 GetEncoder()->EncodeLdr(tmp.GetReg(), false, MemRef(tmp.GetReg()));
1724 auto slowPath = CreateSlowPath<SlowPathUnresolved>(resolver, EntrypointId::GET_UNKNOWN_CALLEE_METHOD);
1725 slowPath->SetUnresolvedType(method, resolver->GetCallMethodId());
1726 slowPath->SetDstReg(tmp.GetReg());
1727 slowPath->SetSlotAddr(methodAddr);
1728 GetEncoder()->EncodeJump(slowPath->GetLabel(), tmp.GetReg(), Condition::EQ);
1729 slowPath->BindBackLabel(GetEncoder());
1730 GetEncoder()->EncodeMov(methodReg, tmp.GetReg());
1731 }
1732
EmitCallResolvedStatic(CallInst * call)1733 void Codegen::EmitCallResolvedStatic(CallInst *call)
1734 {
1735 ASSERT(call->GetOpcode() == Opcode::CallResolvedStatic && call->GetCallMethod() == nullptr);
1736 Reg methodReg = ConvertRegister(call->GetSrcReg(0), DataType::POINTER);
1737 Reg param0 = GetTarget().GetParamReg(0);
1738 GetEncoder()->EncodeMov(param0, methodReg);
1739 GetEncoder()->MakeCall(MemRef(param0, GetRuntime()->GetCompiledEntryPointOffset(GetArch())));
1740 FinalizeCall(call);
1741 }
1742
EmitCallStatic(CallInst * call)1743 void Codegen::EmitCallStatic(CallInst *call)
1744 {
1745 ASSERT(call->IsStaticCall() && !call->IsUnresolved());
1746 if (call->GetSaveState() != nullptr && call->IsInlined()) {
1747 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
1748 if (!GetGraph()->IsAotMode()) {
1749 InsertTrace({Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
1750 Imm(reinterpret_cast<size_t>(call->GetCallMethod())),
1751 Imm(static_cast<size_t>(events::MethodEnterKind::INLINED))});
1752 }
1753 #endif
1754 return;
1755 }
1756 SCOPED_DISASM_STR(this, "Create a call to static");
1757 ASSERT(!HasLiveCallerSavedRegs(call));
1758 // Now MakeCallByOffset is not supported in Arch32Encoder (support ADR instruction)
1759 Reg param0 = GetTarget().GetParamReg(0);
1760 if (call->GetCallMethod() == GetGraph()->GetMethod() && GetArch() != Arch::AARCH32 && !GetGraph()->IsOsrMode() &&
1761 !GetGraph()->GetMethodProperties().GetHasDeopt()) {
1762 if (GetGraph()->IsAotMode()) {
1763 LoadMethod(param0);
1764 } else {
1765 GetEncoder()->EncodeMov(param0, Imm(reinterpret_cast<size_t>(GetGraph()->GetMethod())));
1766 }
1767 GetEncoder()->MakeCallByOffset(GetStartCodeOffset() - GetEncoder()->GetCursorOffset());
1768 } else {
1769 if (GetGraph()->IsAotMode()) {
1770 auto aotData = GetGraph()->GetAotData();
1771 intptr_t offset = aotData->GetPltSlotOffset(GetEncoder()->GetCursorOffset(), call->GetCallMethodId());
1772 // PLT CallStatic Resolver transparently uses param_0 (Method) register
1773 GetEncoder()->MakeLoadAotTable(offset, param0);
1774 } else { // usual JIT case
1775 auto method = call->GetCallMethod();
1776 GetEncoder()->EncodeMov(param0, Imm(reinterpret_cast<size_t>(method)));
1777 }
1778 GetEncoder()->MakeCall(MemRef(param0, GetRuntime()->GetCompiledEntryPointOffset(GetArch())));
1779 }
1780 FinalizeCall(call);
1781 }
1782
EmitCallDynamic(CallInst * call)1783 void Codegen::EmitCallDynamic(CallInst *call)
1784 {
1785 SCOPED_DISASM_STR(this, "Create a dynamic call");
1786 if (call->GetSaveState() != nullptr && call->IsInlined()) {
1787 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
1788 if (!GetGraph()->IsAotMode()) {
1789 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
1790 Imm(reinterpret_cast<size_t>(call->GetCallMethod())),
1791 Imm(static_cast<size_t>(events::MethodEnterKind::INLINED)));
1792 }
1793 #endif
1794 return;
1795 }
1796 RuntimeInterface *runtime = GetRuntime();
1797 Encoder *encoder = GetEncoder();
1798
1799 auto dstReg = ConvertRegister(call->GetDstReg(), call->GetType());
1800 Reg methodParamReg = GetTarget().GetParamReg(CallConvDynInfo::REG_METHOD).As(GetPtrRegType());
1801 Reg numArgsParamReg = GetTarget().GetParamReg(CallConvDynInfo::REG_NUM_ARGS);
1802 auto paramFuncObjLoc = Location::MakeStackArgument(CallConvDynInfo::SLOT_CALLEE);
1803
1804 ASSERT(!HasLiveCallerSavedRegs(call));
1805
1806 // Load method from callee object
1807 static_assert(coretypes::TaggedValue::TAG_OBJECT == 0);
1808 encoder->EncodeLdr(methodParamReg, false, MemRef(SpReg(), GetStackOffset(paramFuncObjLoc)));
1809 encoder->EncodeLdr(methodParamReg, false, MemRef(methodParamReg, runtime->GetFunctionTargetOffset(GetArch())));
1810
1811 ASSERT(call->GetInputsCount() > 1);
1812 auto numArgs = static_cast<uint32_t>(call->GetInputsCount() - 1); // '-1' means 1 for spill fill input
1813 encoder->EncodeMov(numArgsParamReg, Imm(numArgs));
1814
1815 size_t entryPointOffset = runtime->GetCompiledEntryPointOffset(GetArch());
1816 encoder->MakeCall(MemRef(methodParamReg, entryPointOffset));
1817
1818 CreateStackMap(call);
1819 // Dynamic callee may have moved sp if there was insufficient num_actual_args
1820 encoder->EncodeSub(
1821 SpReg(), FpReg(),
1822 Imm(GetFrameLayout().GetOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::BYTES>(0)));
1823
1824 if (dstReg.IsValid()) {
1825 Reg retReg = GetTarget().GetReturnReg(dstReg.GetType());
1826 encoder->EncodeMov(dstReg, retReg);
1827 }
1828 }
1829
FinalizeCall(CallInst * call)1830 void Codegen::FinalizeCall(CallInst *call)
1831 {
1832 ASSERT(!call->IsDynamicCall());
1833 CreateStackMap(call);
1834 auto returnType = call->GetType();
1835 auto dstReg = ConvertRegister(call->GetDstReg(), returnType);
1836 // Restore frame pointer in the TLS
1837 GetEncoder()->EncodeStr(FpReg(), MemRef(ThreadReg(), GetRuntime()->GetTlsFrameOffset(GetArch())));
1838 // Sign/Zero extend return_reg if necessary
1839 if (dstReg.IsValid()) {
1840 auto arch = GetArch();
1841 auto returnReg = GetTarget().GetReturnReg(dstReg.GetType());
1842 // INT8 and INT16 must be sign extended to INT32
1843 // UINT8 and UINT16 must be zero extended to UINT32
1844 if (DataType::ShiftByType(returnType, arch) < DataType::ShiftByType(DataType::INT32, arch)) {
1845 bool isSigned = DataType::IsTypeSigned(returnType);
1846 GetEncoder()->EncodeCast(dstReg, isSigned, Reg(returnReg.GetId(), INT32_TYPE), isSigned);
1847 } else {
1848 GetEncoder()->EncodeMov(dstReg, returnReg);
1849 }
1850 }
1851 }
1852
1853 template <bool IS_CLASS>
CreatePreWRB(Inst * inst,MemRef mem,RegMask preserved,bool storePair)1854 void Codegen::CreatePreWRB(Inst *inst, MemRef mem, RegMask preserved, bool storePair)
1855 {
1856 auto runtime = GetRuntime();
1857 auto *enc = GetEncoder();
1858
1859 SCOPED_DISASM_STR(this, "Pre WRB");
1860 ScopedTmpReg entrypointReg(enc, enc->IsLrAsTempRegEnabledAndReleased());
1861 GetEncoder()->EncodeLdr(entrypointReg, false,
1862 MemRef(ThreadReg(), runtime->GetTlsPreWrbEntrypointOffset(GetArch())));
1863
1864 // Check entrypoint address
1865 auto label = GetEncoder()->CreateLabel();
1866 enc->EncodeJump(label, entrypointReg, Condition::EQ);
1867 auto refType =
1868 inst->GetType() == DataType::REFERENCE ? DataType::GetIntTypeForReference(enc->GetArch()) : DataType::INT64;
1869 ScopedTmpReg tmpRef(enc, ConvertDataType(refType, GetArch()));
1870 auto prevOffset = enc->GetCursorOffset();
1871 // Load old value
1872 if (IsVolatileMemInst(inst)) {
1873 enc->EncodeLdrAcquire(tmpRef, false, mem);
1874 } else {
1875 enc->EncodeLdr(tmpRef, false, mem);
1876 }
1877 TryInsertImplicitNullCheck(inst, prevOffset);
1878 if constexpr (IS_CLASS) {
1879 enc->EncodeLdr(tmpRef, false, MemRef(tmpRef, runtime->GetManagedClassOffset(GetArch())));
1880 } else {
1881 CheckObject(tmpRef, label);
1882 }
1883 auto [live_regs, live_vregs] = GetLiveRegisters<true>(inst);
1884 live_regs |= preserved;
1885 CallBarrier(live_regs, live_vregs, entrypointReg.GetReg(), tmpRef);
1886
1887 if (storePair) {
1888 // store pair doesn't support index and scalar
1889 ASSERT(!mem.HasIndex() && !mem.HasScale());
1890 // calculate offset for second store
1891 // NOLINTNEXTLINE(bugprone-misplaced-widening-cast)
1892 auto secondOffset = static_cast<ssize_t>(1U << DataType::ShiftByType(DataType::REFERENCE, enc->GetArch()));
1893 if (mem.HasDisp()) {
1894 secondOffset += mem.GetDisp();
1895 }
1896 // Load old value
1897 if (IsVolatileMemInst(inst)) {
1898 enc->EncodeLdrAcquire(tmpRef, false, MemRef(mem.GetBase(), secondOffset));
1899 } else {
1900 enc->EncodeLdr(tmpRef, false, MemRef(mem.GetBase(), secondOffset));
1901 }
1902 CheckObject(tmpRef, label);
1903 CallBarrier(live_regs, live_vregs, entrypointReg.GetReg(), tmpRef);
1904 }
1905 enc->BindLabel(label);
1906 }
1907
1908 // Force template instantiation to avoid "undefined reference" error from ld.
1909 template void Codegen::CreatePreWRB<true>(Inst *inst, MemRef mem, RegMask preserved, bool storePair);
1910 template void Codegen::CreatePreWRB<false>(Inst *inst, MemRef mem, RegMask preserved, bool storePair);
1911
1912 /* RegMask 'preserved' is a way to explicitly specify registers that must be
1913 * saved before and restored after the PostWRB call.
1914 *
1915 * If codegen emits some new instructions after the PostWRB call and these
1916 * instructions use 'inst' operand registers, then the registers should live
1917 * through the PostWRB call. The problem is that the liveness analysis can't
1918 * detect if those registers are live because there are no instructions using
1919 * them at the moment.
1920 *
1921 * Note, only CALLER_REG_MASK registers are taken into account.
1922 */
CreatePostWRB(Inst * inst,MemRef mem,Reg reg1,Reg reg2,RegMask preserved)1923 void Codegen::CreatePostWRB(Inst *inst, MemRef mem, Reg reg1, Reg reg2, RegMask preserved)
1924 {
1925 ASSERT(reg1 != INVALID_REGISTER);
1926
1927 if (!GetGraph()->SupportsIrtocBarriers() || !GetGraph()->IsOfflineCompilationMode()) {
1928 auto barrierType = GetRuntime()->GetPostType();
1929 if (barrierType == ark::mem::BarrierType::POST_WRB_NONE) {
1930 return;
1931 }
1932 ASSERT(barrierType == ark::mem::BarrierType::POST_INTERGENERATIONAL_BARRIER ||
1933 barrierType == ark::mem::BarrierType::POST_INTERREGION_BARRIER);
1934 }
1935
1936 // For dynamic methods, another check
1937 if (GetGraph()->IsDynamicMethod()) {
1938 CreatePostWRBForDynamic(inst, mem, reg1, reg2, preserved);
1939 return;
1940 }
1941 PostWriteBarrier pwb(this, inst);
1942 Inst *secondValue;
1943 Inst *val = InstStoredValue(inst, &secondValue);
1944 ASSERT(secondValue == nullptr || reg2 != INVALID_REGISTER);
1945 if (val->GetOpcode() == Opcode::NullPtr) {
1946 // We can don't encode Post barrier for nullptr
1947 if (secondValue == nullptr || secondValue->GetOpcode() == Opcode::NullPtr) {
1948 return;
1949 }
1950 // CallPostWRB only for second reg
1951 auto secondMemOffset = reg1.GetSize() / BITS_PER_BYTE;
1952 if (!mem.HasIndex()) {
1953 MemRef secondMem(mem.GetBase(), mem.GetDisp() + secondMemOffset);
1954 pwb.Encode(secondMem, reg2, INVALID_REGISTER, !IsInstNotNull(secondValue), preserved);
1955 return;
1956 }
1957 ASSERT(mem.GetScale() == 0 && !mem.HasDisp());
1958 ASSERT(GetTarget().GetTempRegsMask().Test(mem.GetIndex().GetId()));
1959 GetEncoder()->EncodeAdd(mem.GetIndex(), mem.GetIndex(), Imm(secondMemOffset));
1960 pwb.Encode(mem, reg2, INVALID_REGISTER, !IsInstNotNull(secondValue), preserved);
1961 return;
1962 }
1963 // Create PostWRB only for first value
1964 if (secondValue != nullptr && secondValue->GetOpcode() == Opcode::NullPtr) {
1965 reg2 = INVALID_REGISTER;
1966 }
1967 bool checkObject = true;
1968 if (reg2 == INVALID_REGISTER) {
1969 if (IsInstNotNull(val)) {
1970 checkObject = false;
1971 }
1972 } else {
1973 if (IsInstNotNull(val) && IsInstNotNull(secondValue)) {
1974 checkObject = false;
1975 }
1976 }
1977 pwb.Encode(mem, reg1, reg2, checkObject, preserved);
1978 }
1979
CreatePostWRBForDynamic(Inst * inst,MemRef mem,Reg reg1,Reg reg2,RegMask preserved)1980 void Codegen::CreatePostWRBForDynamic(Inst *inst, MemRef mem, Reg reg1, Reg reg2, RegMask preserved)
1981 {
1982 PostWriteBarrier pwb(this, inst);
1983 if (reg2 == INVALID_REGISTER) {
1984 int storeIndex;
1985 if (inst->GetOpcode() == Opcode::StoreObject || inst->GetOpcode() == Opcode::StoreI ||
1986 inst->GetOpcode() == Opcode::StoreArrayI) {
1987 storeIndex = 1_I;
1988 } else {
1989 ASSERT(inst->GetOpcode() == Opcode::StoreArray || inst->GetOpcode() == Opcode::Store);
1990 storeIndex = 2_I;
1991 }
1992 if (StoreValueCanBeObject(inst->GetDataFlowInput(storeIndex))) {
1993 pwb.Encode(mem, reg1, reg2, true, preserved);
1994 }
1995 } else {
1996 int storeIndex;
1997 if (inst->GetOpcode() == Opcode::StoreArrayPairI) {
1998 storeIndex = 1_I;
1999 } else {
2000 ASSERT(inst->GetOpcode() == Opcode::StoreArrayPair);
2001 storeIndex = 2_I;
2002 }
2003 bool firstIsObject = StoreValueCanBeObject(inst->GetDataFlowInput(storeIndex));
2004 bool secondIsObject = StoreValueCanBeObject(inst->GetDataFlowInput(storeIndex + 1));
2005 if (firstIsObject || secondIsObject) {
2006 if (firstIsObject && !secondIsObject) {
2007 reg2 = INVALID_REGISTER;
2008 } else if (!firstIsObject && secondIsObject) {
2009 reg1 = reg2;
2010 reg2 = INVALID_REGISTER;
2011 }
2012 pwb.Encode(mem, reg1, reg2, true, preserved);
2013 }
2014 }
2015 }
2016
CheckObject(Reg reg,LabelHolder::LabelId label)2017 void Codegen::CheckObject(Reg reg, LabelHolder::LabelId label)
2018 {
2019 auto graph = GetGraph();
2020 auto *enc = GetEncoder();
2021
2022 // interpreter use x20 in IrToc and we don't have enough temporary registers
2023 // remove after PR 98 or PR 47
2024 if (graph->IsDynamicMethod()) {
2025 ASSERT(reg.IsScalar());
2026 reg = reg.As(INT64_TYPE);
2027 auto tagMask = graph->GetRuntime()->GetTaggedTagMask();
2028 // Check that the value is object(not int and not double)
2029 enc->EncodeJumpTest(label, reg, Imm(tagMask), Condition::TST_NE);
2030 auto specialMask = graph->GetRuntime()->GetTaggedSpecialMask();
2031 // Check that the value is not special value
2032 enc->EncodeJumpTest(label, reg, Imm(~specialMask), Condition::TST_EQ);
2033 } else {
2034 enc->EncodeJump(label, reg, Condition::EQ);
2035 }
2036 }
2037
HasLiveCallerSavedRegs(Inst * inst)2038 bool Codegen::HasLiveCallerSavedRegs(Inst *inst)
2039 {
2040 auto [live_regs, live_fp_regs] = GetLiveRegisters<false>(inst);
2041 live_regs &= GetCallerRegsMask(GetArch(), false);
2042 live_fp_regs &= GetCallerRegsMask(GetArch(), true);
2043 return live_regs.Any() || live_fp_regs.Any();
2044 }
2045
SaveCallerRegisters(RegMask liveRegs,VRegMask liveVregs,bool adjustRegs)2046 void Codegen::SaveCallerRegisters(RegMask liveRegs, VRegMask liveVregs, bool adjustRegs)
2047 {
2048 SCOPED_DISASM_STR(this, "Save caller saved regs");
2049 auto base = GetFrameInfo()->GetCallersRelativeFp() ? GetTarget().GetFrameReg() : GetTarget().GetStackReg();
2050 liveRegs &= ~GetEncoder()->GetAvailableScratchRegisters();
2051 liveVregs &= ~GetEncoder()->GetAvailableScratchFpRegisters();
2052 if (adjustRegs) {
2053 liveRegs &= GetRegfile()->GetCallerSavedRegMask();
2054 liveVregs &= GetRegfile()->GetCallerSavedVRegMask();
2055 } else {
2056 liveRegs &= GetCallerRegsMask(GetArch(), false);
2057 liveVregs &= GetCallerRegsMask(GetArch(), true);
2058 }
2059 if (GetFrameInfo()->GetPushCallers()) {
2060 GetEncoder()->PushRegisters(liveRegs, liveVregs);
2061 } else {
2062 GetEncoder()->SaveRegisters(liveRegs, false, GetFrameInfo()->GetCallersOffset(), base,
2063 GetCallerRegsMask(GetArch(), false));
2064 GetEncoder()->SaveRegisters(liveVregs, true, GetFrameInfo()->GetFpCallersOffset(), base,
2065 GetFrameInfo()->GetPositionedCallers() ? GetCallerRegsMask(GetArch(), true)
2066 : RegMask());
2067 }
2068 }
2069
LoadCallerRegisters(RegMask liveRegs,VRegMask liveVregs,bool adjustRegs)2070 void Codegen::LoadCallerRegisters(RegMask liveRegs, VRegMask liveVregs, bool adjustRegs)
2071 {
2072 SCOPED_DISASM_STR(this, "Restore caller saved regs");
2073 auto base = GetFrameInfo()->GetCallersRelativeFp() ? GetTarget().GetFrameReg() : GetTarget().GetStackReg();
2074 liveRegs &= ~GetEncoder()->GetAvailableScratchRegisters();
2075 liveVregs &= ~GetEncoder()->GetAvailableScratchFpRegisters();
2076 if (adjustRegs) {
2077 liveRegs &= GetRegfile()->GetCallerSavedRegMask();
2078 liveVregs &= GetRegfile()->GetCallerSavedVRegMask();
2079 } else {
2080 liveRegs &= GetCallerRegsMask(GetArch(), false);
2081 liveVregs &= GetCallerRegsMask(GetArch(), true);
2082 }
2083 if (GetFrameInfo()->GetPushCallers()) {
2084 GetEncoder()->PopRegisters(liveRegs, liveVregs);
2085 } else {
2086 GetEncoder()->LoadRegisters(liveRegs, false, GetFrameInfo()->GetCallersOffset(), base,
2087 GetCallerRegsMask(GetArch(), false));
2088 GetEncoder()->LoadRegisters(liveVregs, true, GetFrameInfo()->GetFpCallersOffset(), base,
2089 GetCallerRegsMask(GetArch(), true));
2090 }
2091 }
2092
RegisterKeepCallArgument(CallInst * callInst,Reg reg)2093 bool Codegen::RegisterKeepCallArgument(CallInst *callInst, Reg reg)
2094 {
2095 for (auto i = 0U; i < callInst->GetInputsCount(); i++) {
2096 auto location = callInst->GetLocation(i);
2097 if (location.IsRegister() && location.GetValue() == reg.GetId()) {
2098 return true;
2099 }
2100 }
2101 return false;
2102 }
2103
LoadMethod(Reg dst)2104 void Codegen::LoadMethod(Reg dst)
2105 {
2106 ASSERT((CFrameMethod::GetOffsetFromSpInBytes(GetFrameLayout()) -
2107 (GetFrameLayout().GetMethodOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::BYTES>())) ==
2108 0);
2109 auto spOffset = CFrameMethod::GetOffsetFromSpInBytes(GetFrameLayout());
2110 auto mem = MemRef(SpReg(), spOffset);
2111 GetEncoder()->EncodeLdr(dst, false, mem);
2112 }
2113
StoreFreeSlot(Reg src)2114 void Codegen::StoreFreeSlot(Reg src)
2115 {
2116 ASSERT(src.GetSize() <= (GetFrameLayout().GetSlotSize() << 3U));
2117 auto spOffset =
2118 GetFrameLayout().GetFreeSlotOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::BYTES>();
2119 auto mem = MemRef(SpReg(), spOffset);
2120 GetEncoder()->EncodeStr(src, mem);
2121 }
2122
LoadFreeSlot(Reg dst)2123 void Codegen::LoadFreeSlot(Reg dst)
2124 {
2125 ASSERT(dst.GetSize() <= (GetFrameLayout().GetSlotSize() << 3U));
2126 auto spOffset =
2127 GetFrameLayout().GetFreeSlotOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::BYTES>();
2128 auto mem = MemRef(SpReg(), spOffset);
2129 GetEncoder()->EncodeLdr(dst, false, mem);
2130 }
2131
CreateReturn(const Inst * inst)2132 void Codegen::CreateReturn(const Inst *inst)
2133 {
2134 if (GetGraph()->GetMethodProperties().GetLastReturn() == inst) {
2135 GetEncoder()->BindLabel(GetLabelExit());
2136 GenerateEpilogue();
2137 } else {
2138 GetEncoder()->EncodeJump(GetLabelExit());
2139 }
2140 }
2141
EncodeDynamicCast(Inst * inst,Reg dst,bool dstSigned,Reg src)2142 void Codegen::EncodeDynamicCast(Inst *inst, Reg dst, bool dstSigned, Reg src)
2143 {
2144 CHECK_EQ(src.GetSize(), BITS_PER_UINT64);
2145 CHECK_GE(dst.GetSize(), BITS_PER_UINT32);
2146
2147 bool isDst64 {dst.GetSize() == BITS_PER_UINT64};
2148 dst = dst.As(INT32_TYPE);
2149
2150 auto enc {GetEncoder()};
2151 if (g_options.IsCpuFeatureEnabled(CpuFeature::JSCVT)) {
2152 // no slow path intended
2153 enc->EncodeFastPathDynamicCast(dst, src, LabelHolder::INVALID_LABEL);
2154 } else {
2155 auto slowPath {CreateSlowPath<SlowPathJsCastDoubleToInt32>(inst)};
2156 slowPath->SetDstReg(dst);
2157 slowPath->SetSrcReg(src);
2158
2159 enc->EncodeFastPathDynamicCast(dst, src, slowPath->GetLabel());
2160 slowPath->BindBackLabel(enc);
2161 }
2162
2163 if (isDst64) {
2164 enc->EncodeCast(dst.As(INT64_TYPE), dstSigned, dst, dstSigned);
2165 }
2166 }
2167
2168 // Next visitors use calling convention
VisitCallIndirect(CallIndirectInst * inst)2169 void Codegen::VisitCallIndirect(CallIndirectInst *inst)
2170 {
2171 auto location = inst->GetLocation(0);
2172 ASSERT(location.IsFixedRegister() && location.IsRegisterValid());
2173 auto src = Reg(location.GetValue(), GetTarget().GetPtrRegType());
2174 auto dst = ConvertRegister(inst->GetDstReg(), inst->GetType());
2175
2176 GetEncoder()->MakeCall(src);
2177 if (inst->HasUsers()) {
2178 GetEncoder()->EncodeMov(dst, GetTarget().GetReturnReg(dst.GetType()));
2179 }
2180 }
2181
VisitCall(CallInst * inst)2182 void Codegen::VisitCall(CallInst *inst)
2183 {
2184 ASSERT(GetGraph()->GetRelocationHandler() != nullptr);
2185 ASSERT(!HasLiveCallerSavedRegs(inst));
2186
2187 RelocationInfo relocation;
2188 relocation.data = inst->GetCallMethodId();
2189 GetEncoder()->MakeCall(&relocation);
2190 GetGraph()->GetRelocationHandler()->AddRelocation(relocation);
2191
2192 if (inst->HasUsers()) {
2193 auto dstReg = ConvertRegister(inst->GetDstReg(), inst->GetType());
2194 ASSERT(dstReg.IsValid());
2195 GetEncoder()->EncodeMov(dstReg, GetTarget().GetReturnReg(dstReg.GetType()));
2196 }
2197 }
2198
GetEntryPointId(uint64_t elementSize,RuntimeInterface::EntrypointId & eid)2199 static void GetEntryPointId(uint64_t elementSize, RuntimeInterface::EntrypointId &eid)
2200 {
2201 switch (elementSize) {
2202 case sizeof(uint8_t):
2203 eid = RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB8;
2204 break;
2205 case sizeof(uint16_t):
2206 eid = RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB16;
2207 break;
2208 case sizeof(uint32_t):
2209 eid = RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB32;
2210 break;
2211 case sizeof(uint64_t):
2212 eid = RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB64;
2213 break;
2214 default:
2215 UNREACHABLE();
2216 }
2217 }
2218
VisitNewArray(Inst * inst)2219 void Codegen::VisitNewArray(Inst *inst)
2220 {
2221 auto method = inst->CastToNewArray()->GetMethod();
2222 auto dst = ConvertRegister(inst->GetDstReg(), inst->GetType());
2223 auto srcClass = ConvertRegister(inst->GetSrcReg(NewArrayInst::INDEX_CLASS), DataType::POINTER);
2224 auto srcSize = ConvertRegister(inst->GetSrcReg(NewArrayInst::INDEX_SIZE), DataType::Type::INT32);
2225 auto arrayType = inst->CastToNewArray()->GetTypeId();
2226 auto runtime = GetGraph()->GetRuntime();
2227 auto encoder = GetEncoder();
2228
2229 auto maxTlabSize = runtime->GetTLABMaxSize();
2230 // NOTE(msherstennikov): support NewArray fast path for arm32
2231 if (maxTlabSize == 0 || GetArch() == Arch::AARCH32) {
2232 CallRuntime(inst, EntrypointId::CREATE_ARRAY, dst, RegMask::GetZeroMask(), srcClass, srcSize);
2233 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
2234 encoder->EncodeMemoryBarrier(memory_order::RELEASE);
2235 }
2236 return;
2237 }
2238
2239 auto lenInst = inst->GetDataFlowInput(0);
2240 auto classArraySize = runtime->GetClassArraySize(GetArch());
2241 uint64_t arraySize = 0;
2242 uint64_t elementSize = runtime->GetArrayElementSize(method, arrayType);
2243 uint64_t alignment = runtime->GetTLABAlignment();
2244 if (lenInst->GetOpcode() == Opcode::Constant) {
2245 ASSERT(lenInst->GetType() == DataType::INT64);
2246 arraySize = lenInst->CastToConstant()->GetIntValue() * elementSize + classArraySize;
2247 arraySize = (arraySize & ~(alignment - 1U)) + ((arraySize % alignment) != 0U ? alignment : 0U);
2248 if (arraySize > maxTlabSize) {
2249 CallRuntime(inst, EntrypointId::CREATE_ARRAY, dst, RegMask::GetZeroMask(), srcClass, srcSize);
2250 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
2251 encoder->EncodeMemoryBarrier(memory_order::RELEASE);
2252 }
2253 return;
2254 }
2255 }
2256
2257 EntrypointId eid;
2258 GetEntryPointId(elementSize, eid);
2259 CallFastPath(inst, eid, dst, RegMask::GetZeroMask(), srcClass, srcSize);
2260 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
2261 encoder->EncodeMemoryBarrier(memory_order::RELEASE);
2262 }
2263 }
2264
CreateMonitorCall(MonitorInst * inst)2265 void Codegen::CreateMonitorCall(MonitorInst *inst)
2266 {
2267 auto src = ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // obj
2268 auto id = inst->IsExit() ? EntrypointId::MONITOR_EXIT_FAST_PATH : EntrypointId::MONITOR_ENTER_FAST_PATH;
2269 CallFastPath(inst, id, INVALID_REGISTER, RegMask::GetZeroMask(), src);
2270 }
2271
CreateMonitorCallOld(MonitorInst * inst)2272 void Codegen::CreateMonitorCallOld(MonitorInst *inst)
2273 {
2274 auto src = ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // obj
2275 auto dst = ConvertRegister(inst->GetDstReg(), inst->GetType());
2276 auto id = inst->IsExit() ? EntrypointId::UNLOCK_OBJECT : EntrypointId::LOCK_OBJECT;
2277 CallRuntime(inst, id, dst, RegMask::GetZeroMask(), src);
2278 }
2279
CreateCheckCastInterfaceCall(Inst * inst)2280 void Codegen::CreateCheckCastInterfaceCall(Inst *inst)
2281 {
2282 auto obj = ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE);
2283 auto interface = ConvertRegister(inst->GetSrcReg(SECOND_OPERAND), DataType::REFERENCE);
2284 CallFastPath(inst, EntrypointId::CHECK_CAST_INTERFACE, INVALID_REGISTER, RegMask::GetZeroMask(), obj, interface);
2285 }
2286
TryInsertImplicitNullCheck(Inst * inst,size_t prevOffset)2287 void Codegen::TryInsertImplicitNullCheck(Inst *inst, size_t prevOffset)
2288 {
2289 if (!IsSuitableForImplicitNullCheck(inst)) {
2290 return;
2291 }
2292 if (!inst->CanThrow()) {
2293 return;
2294 }
2295
2296 auto nullcheck = inst->GetInput(0).GetInst();
2297 ASSERT(nullcheck->GetOpcode() == Opcode::NullCheck && nullcheck->CastToNullCheck()->IsImplicit());
2298 auto currOffset = GetEncoder()->GetCursorOffset();
2299 ASSERT(currOffset > prevOffset);
2300 GetCodeBuilder()->AddImplicitNullCheck(currOffset, currOffset - prevOffset);
2301 CreateStackMap(nullcheck, inst);
2302 }
2303
CreateFloatIsInf(IntrinsicInst * inst,Reg dst,SRCREGS src)2304 void Codegen::CreateFloatIsInf([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2305 {
2306 GetEncoder()->EncodeIsInf(dst, src[0]);
2307 }
2308
CreateFloatIsInteger(IntrinsicInst * inst,Reg dst,SRCREGS src)2309 void Codegen::CreateFloatIsInteger([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2310 {
2311 GetEncoder()->EncodeIsInteger(dst, src[0]);
2312 }
2313
CreateFloatIsSafeInteger(IntrinsicInst * inst,Reg dst,SRCREGS src)2314 void Codegen::CreateFloatIsSafeInteger([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2315 {
2316 GetEncoder()->EncodeIsSafeInteger(dst, src[0]);
2317 }
2318
CreateStringEquals(IntrinsicInst * inst,Reg dst,SRCREGS src)2319 void Codegen::CreateStringEquals([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2320 {
2321 auto entrypointId = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::STRING_EQUALS_COMPRESSED
2322 : EntrypointId::STRING_EQUALS;
2323 CallFastPath(inst, entrypointId, dst, {}, src[0], src[1U]);
2324 }
2325
CreateMathCeil(IntrinsicInst * inst,Reg dst,SRCREGS src)2326 void Codegen::CreateMathCeil([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2327 {
2328 GetEncoder()->EncodeCeil(dst, src[0]);
2329 }
2330
CreateMathFloor(IntrinsicInst * inst,Reg dst,SRCREGS src)2331 void Codegen::CreateMathFloor([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2332 {
2333 GetEncoder()->EncodeFloor(dst, src[0]);
2334 }
2335
CreateCountLeadingZeroBits(IntrinsicInst * inst,Reg dst,SRCREGS src)2336 void Codegen::CreateCountLeadingZeroBits([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2337 {
2338 GetEncoder()->EncodeCountLeadingZeroBits(Reg(dst.GetId(), src[0].GetType()), src[0]);
2339 }
2340
CreateStringBuilderChar(IntrinsicInst * inst,Reg dst,SRCREGS src)2341 void Codegen::CreateStringBuilderChar(IntrinsicInst *inst, [[maybe_unused]] Reg dst, SRCREGS src)
2342 {
2343 CallFastPath(inst, EntrypointId::STRING_BUILDER_CHAR, dst, RegMask::GetZeroMask(), src[0], src[1U]);
2344 }
2345
CreateStringBuilderBool(IntrinsicInst * inst,Reg dst,SRCREGS src)2346 void Codegen::CreateStringBuilderBool(IntrinsicInst *inst, [[maybe_unused]] Reg dst, SRCREGS src)
2347 {
2348 CallFastPath(inst, EntrypointId::STRING_BUILDER_BOOL, dst, RegMask::GetZeroMask(), src[0], src[1U]);
2349 }
2350
CreateStringBuilderString(IntrinsicInst * inst,Reg dst,SRCREGS src)2351 void Codegen::CreateStringBuilderString(IntrinsicInst *inst, [[maybe_unused]] Reg dst, SRCREGS src)
2352 {
2353 auto entrypointId = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::STRING_BUILDER_STRING_COMPRESSED
2354 : EntrypointId::STRING_BUILDER_STRING;
2355
2356 CallFastPath(inst, entrypointId, dst, RegMask::GetZeroMask(), src[0], src[1U]);
2357 }
2358
CreateStringFromCharArrayTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)2359 void Codegen::CreateStringFromCharArrayTlab(IntrinsicInst *inst, Reg dst, SRCREGS src)
2360 {
2361 CreateStringFromCharArrayTlab(static_cast<Inst *>(inst), dst, src);
2362 }
2363
CreateStringFromCharArrayTlab(Inst * inst,Reg dst,SRCREGS src)2364 void Codegen::CreateStringFromCharArrayTlab(Inst *inst, Reg dst, SRCREGS src)
2365 {
2366 auto runtime = GetGraph()->GetRuntime();
2367 auto offset = src[FIRST_OPERAND];
2368 auto count = src[SECOND_OPERAND];
2369 auto array = src[THIRD_OPERAND];
2370
2371 auto entryId = GetRuntime()->IsCompressedStringsEnabled()
2372 ? EntrypointId::CREATE_STRING_FROM_CHAR_ARRAY_TLAB_COMPRESSED
2373 : EntrypointId::CREATE_STRING_FROM_CHAR_ARRAY_TLAB;
2374
2375 if (GetRegfile()->GetZeroReg().GetId() == offset.GetId()) {
2376 entryId = GetRuntime()->IsCompressedStringsEnabled()
2377 ? EntrypointId::CREATE_STRING_FROM_ZERO_BASED_CHAR_ARRAY_TLAB_COMPRESSED
2378 : EntrypointId::CREATE_STRING_FROM_ZERO_BASED_CHAR_ARRAY_TLAB;
2379 }
2380
2381 if (GetGraph()->IsAotMode()) {
2382 ScopedTmpReg klassReg(GetEncoder());
2383 GetEncoder()->EncodeLdr(klassReg, false,
2384 MemRef(ThreadReg(), runtime->GetStringClassPointerTlsOffset(GetArch())));
2385 if (GetRegfile()->GetZeroReg().GetId() == offset.GetId()) {
2386 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), count, array, klassReg);
2387 } else {
2388 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), offset, count, array, klassReg);
2389 }
2390 } else {
2391 auto klassImm = TypedImm(reinterpret_cast<uintptr_t>(runtime->GetStringClass(GetGraph()->GetMethod())));
2392 if (GetRegfile()->GetZeroReg().GetId() == offset.GetId()) {
2393 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), count, array, klassImm);
2394 } else {
2395 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), offset, count, array, klassImm);
2396 }
2397 }
2398 }
2399
CreateStringFromStringTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)2400 void Codegen::CreateStringFromStringTlab(IntrinsicInst *inst, Reg dst, SRCREGS src)
2401 {
2402 auto entryId = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::CREATE_STRING_FROM_STRING_TLAB_COMPRESSED
2403 : EntrypointId::CREATE_STRING_FROM_STRING_TLAB;
2404 auto srcStr = src[FIRST_OPERAND];
2405 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), srcStr);
2406 }
2407
CreateStringSubstringTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)2408 void Codegen::CreateStringSubstringTlab([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2409 {
2410 auto entrypointId = GetRuntime()->IsCompressedStringsEnabled()
2411 ? EntrypointId::SUB_STRING_FROM_STRING_TLAB_COMPRESSED
2412 : EntrypointId::SUB_STRING_FROM_STRING_TLAB;
2413 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND]);
2414 }
2415
CreateStringGetCharsTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)2416 void Codegen::CreateStringGetCharsTlab([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2417 {
2418 auto entrypointId = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::STRING_GET_CHARS_TLAB_COMPRESSED
2419 : EntrypointId::STRING_GET_CHARS_TLAB;
2420 auto runtime = GetGraph()->GetRuntime();
2421 if (GetGraph()->IsAotMode()) {
2422 ScopedTmpReg klassReg(GetEncoder());
2423 GetEncoder()->EncodeLdr(klassReg, false,
2424 MemRef(ThreadReg(), runtime->GetArrayU16ClassPointerTlsOffset(GetArch())));
2425 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
2426 klassReg);
2427 } else {
2428 auto klassImm = TypedImm(reinterpret_cast<uintptr_t>(runtime->GetArrayU16Class(GetGraph()->GetMethod())));
2429 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
2430 klassImm);
2431 }
2432 }
2433
CreateStringHashCode(IntrinsicInst * inst,Reg dst,SRCREGS src)2434 void Codegen::CreateStringHashCode([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2435 {
2436 auto entrypoint = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::STRING_HASH_CODE_COMPRESSED
2437 : EntrypointId::STRING_HASH_CODE;
2438 auto strReg = src[FIRST_OPERAND];
2439 auto mref = MemRef(strReg, ark::coretypes::String::GetHashcodeOffset());
2440 auto slowPath = CreateSlowPath<SlowPathStringHashCode>(inst, entrypoint);
2441 slowPath->SetDstReg(dst);
2442 slowPath->SetSrcReg(strReg);
2443 if (dst.GetId() != strReg.GetId()) {
2444 GetEncoder()->EncodeLdr(ConvertRegister(dst.GetId(), DataType::INT32), false, mref);
2445 GetEncoder()->EncodeJump(slowPath->GetLabel(), dst, Condition::EQ);
2446 } else {
2447 ScopedTmpReg hashReg(GetEncoder(), INT32_TYPE);
2448 GetEncoder()->EncodeLdr(hashReg, false, mref);
2449 GetEncoder()->EncodeJump(slowPath->GetLabel(), hashReg, Condition::EQ);
2450 GetEncoder()->EncodeMov(dst, hashReg);
2451 }
2452 slowPath->BindBackLabel(GetEncoder());
2453 }
CreateStringCompareTo(IntrinsicInst * inst,Reg dst,SRCREGS src)2454 void Codegen::CreateStringCompareTo([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2455 {
2456 auto str1 = src[FIRST_OPERAND];
2457 auto str2 = src[SECOND_OPERAND];
2458 CallFastPath(inst, EntrypointId::STRING_COMPARE_TO, dst, {}, str1, str2);
2459 }
2460 #include "intrinsics_codegen.inl"
2461
CreateBuiltinIntrinsic(IntrinsicInst * inst)2462 void Codegen::CreateBuiltinIntrinsic(IntrinsicInst *inst)
2463 {
2464 Reg dst = INVALID_REGISTER;
2465 SRCREGS src;
2466
2467 if (!inst->NoDest()) {
2468 dst = ConvertRegister(inst->GetDstReg(), inst->GetType());
2469 }
2470
2471 for (size_t i = 0; i < inst->GetInputsCount(); ++i) {
2472 if (inst->GetInput(i).GetInst()->IsSaveState()) {
2473 continue;
2474 }
2475 auto location = inst->GetLocation(i);
2476 auto type = inst->GetInputType(i);
2477 src[i] = ConvertRegister(location.GetValue(), type);
2478 }
2479 FillBuiltin(inst, src, dst);
2480 }
2481
GetNeedBarrierProperty(const Inst * inst)2482 static bool GetNeedBarrierProperty(const Inst *inst)
2483 {
2484 Opcode op = inst->GetOpcode();
2485 if (op == Opcode::LoadObject) {
2486 return inst->CastToLoadObject()->GetNeedBarrier();
2487 }
2488 if (op == Opcode::StoreObject) {
2489 return inst->CastToStoreObject()->GetNeedBarrier();
2490 }
2491 if (op == Opcode::LoadObjectPair) {
2492 return inst->CastToLoadObjectPair()->GetNeedBarrier();
2493 }
2494 if (op == Opcode::StoreObjectPair) {
2495 return inst->CastToStoreObjectPair()->GetNeedBarrier();
2496 }
2497 if (op == Opcode::LoadArray) {
2498 return inst->CastToLoadArray()->GetNeedBarrier();
2499 }
2500 if (op == Opcode::StoreArray) {
2501 return inst->CastToStoreArray()->GetNeedBarrier();
2502 }
2503 if (op == Opcode::LoadArrayI) {
2504 return inst->CastToLoadArrayI()->GetNeedBarrier();
2505 }
2506 if (op == Opcode::StoreArrayI) {
2507 return inst->CastToStoreArrayI()->GetNeedBarrier();
2508 }
2509 if (op == Opcode::LoadArrayPair) {
2510 return inst->CastToLoadArrayPair()->GetNeedBarrier();
2511 }
2512 if (op == Opcode::StoreArrayPair) {
2513 return inst->CastToStoreArrayPair()->GetNeedBarrier();
2514 }
2515 if (op == Opcode::LoadArrayPairI) {
2516 return inst->CastToLoadArrayPairI()->GetNeedBarrier();
2517 }
2518 if (op == Opcode::StoreArrayPairI) {
2519 return inst->CastToStoreArrayPairI()->GetNeedBarrier();
2520 }
2521 if (op == Opcode::LoadStatic) {
2522 return inst->CastToLoadStatic()->GetNeedBarrier();
2523 }
2524 if (op == Opcode::StoreStatic) {
2525 return inst->CastToStoreStatic()->GetNeedBarrier();
2526 }
2527 return false;
2528 }
2529
2530 /**
2531 * Returns true if codegen emits call(s) to some library function(s)
2532 * while processing the instruction.
2533 */
InstEncodedWithLibCall(const Inst * inst,Arch arch)2534 bool Codegen::InstEncodedWithLibCall(const Inst *inst, Arch arch)
2535 {
2536 ASSERT(inst != nullptr);
2537 Opcode op = inst->GetOpcode();
2538 if (op == Opcode::Mod) {
2539 auto dstType = inst->GetType();
2540 if (arch == Arch::AARCH64 || arch == Arch::X86_64) {
2541 return dstType == DataType::FLOAT32 || dstType == DataType::FLOAT64;
2542 }
2543 return arch == Arch::AARCH32;
2544 }
2545 if (op == Opcode::Div && arch == Arch::AARCH32) {
2546 auto dstType = inst->GetType();
2547 return dstType == DataType::INT64 || dstType == DataType::UINT64;
2548 }
2549 if (op == Opcode::Cast && arch == Arch::AARCH32) {
2550 auto dstType = inst->GetType();
2551 auto srcType = inst->GetInputType(0);
2552 if (dstType == DataType::FLOAT32 || dstType == DataType::FLOAT64) {
2553 return srcType == DataType::INT64 || srcType == DataType::UINT64;
2554 }
2555 if (srcType == DataType::FLOAT32 || srcType == DataType::FLOAT64) {
2556 return dstType == DataType::INT64 || dstType == DataType::UINT64;
2557 }
2558 return false;
2559 }
2560
2561 return GetNeedBarrierProperty(inst);
2562 }
2563
ConvertInstTmpReg(const Inst * inst,DataType::Type type) const2564 Reg Codegen::ConvertInstTmpReg(const Inst *inst, DataType::Type type) const
2565 {
2566 ASSERT(inst->GetTmpLocation().IsFixedRegister());
2567 return Reg(inst->GetTmpLocation().GetValue(), ConvertDataType(type, GetArch()));
2568 }
2569
ConvertInstTmpReg(const Inst * inst) const2570 Reg Codegen::ConvertInstTmpReg(const Inst *inst) const
2571 {
2572 return ConvertInstTmpReg(inst, Is64BitsArch(GetArch()) ? DataType::INT64 : DataType::INT32);
2573 }
2574
Encode(MemRef mem,Reg reg1,Reg reg2,bool checkObject,RegMask preserved)2575 void PostWriteBarrier::Encode(MemRef mem, Reg reg1, Reg reg2, bool checkObject, RegMask preserved)
2576 {
2577 ASSERT(reg1.IsValid());
2578 auto reference {TypeInfo::FromDataType(DataType::REFERENCE, cg_->GetArch())};
2579 PostWriteBarrier::Args args;
2580 args.mem = mem;
2581 args.reg1 = reg1.As(reference);
2582 args.reg2 = reg2.IsValid() ? reg2.As(reference) : INVALID_REGISTER;
2583 args.checkObject = checkObject;
2584 args.preserved = preserved;
2585 if (cg_->GetGraph()->SupportsIrtocBarriers()) {
2586 if (cg_->GetGraph()->IsOfflineCompilationMode()) {
2587 EncodeOfflineIrtocBarrier(args);
2588 } else {
2589 EncodeOnlineIrtocBarrier(args);
2590 }
2591 } else if (type_ == ark::mem::BarrierType::POST_INTERREGION_BARRIER) {
2592 EncodeInterRegionBarrier(args);
2593 } else {
2594 ASSERT(type_ == ark::mem::BarrierType::POST_INTERGENERATIONAL_BARRIER);
2595 EncodeInterGenerationalBarrier(mem.GetBase().As(reference));
2596 }
2597 }
2598
EncodeOfflineIrtocBarrier(Args args)2599 void PostWriteBarrier::EncodeOfflineIrtocBarrier(Args args)
2600 {
2601 auto hasObj2 = HasObject2(args);
2602 auto paramRegs = GetParamRegs(hasObj2 ? 4U : 3U, args);
2603 cg_->SaveCallerRegisters(paramRegs, VRegMask(), false);
2604 if (hasObj2) {
2605 FillCallParams(args.mem, args.reg1, args.reg2);
2606 } else {
2607 FillCallParams(args.mem, args.reg1);
2608 }
2609 // load function pointer from tls field
2610 auto offset {hasObj2 ? cross_values::GetManagedThreadPostWrbTwoObjectsOffset(cg_->GetArch())
2611 : cross_values::GetManagedThreadPostWrbOneObjectOffset(cg_->GetArch())};
2612 MemRef entry(cg_->ThreadReg(), offset);
2613 cg_->GetEncoder()->MakeCall(entry);
2614 cg_->LoadCallerRegisters(paramRegs, VRegMask(), false);
2615 }
2616
EncodeOnlineIrtocBarrier(Args args)2617 void PostWriteBarrier::EncodeOnlineIrtocBarrier(Args args)
2618 {
2619 SCOPED_DISASM_STR(cg_, "Post Online Irtoc-WRB");
2620 auto *enc {cg_->GetEncoder()};
2621 auto hasObj2 = HasObject2(args);
2622 if (type_ == ark::mem::BarrierType::POST_INTERREGION_BARRIER) {
2623 if (hasObj2) {
2624 EncodeOnlineIrtocRegionTwoRegsBarrier(args);
2625 } else {
2626 EncodeOnlineIrtocRegionOneRegBarrier(args);
2627 }
2628 } else {
2629 static constexpr auto ENTRYPOINT_ID {RuntimeInterface::EntrypointId::POST_INTER_GENERATIONAL_BARRIER0};
2630 auto base = GetBase(args);
2631 auto paramRegs = GetParamRegs(1U, args);
2632 cg_->SaveCallerRegisters(paramRegs, VRegMask(), false);
2633 auto paramReg0 = enc->GetTarget().GetParamReg(0);
2634 enc->EncodeMov(paramReg0, base);
2635 MemRef entry(cg_->ThreadReg(), cg_->GetRuntime()->GetEntrypointTlsOffset(cg_->GetArch(), ENTRYPOINT_ID));
2636 enc->MakeCall(entry);
2637 cg_->LoadCallerRegisters(paramRegs, VRegMask(), false);
2638 }
2639 }
2640
2641 /**
2642 * Post-write barrier for StorePair case.
2643 * The code below may mark 0, 1 or 2 cards. The idea is following:
2644 * if (the second object isn't null and is allocated in other from base object's region)
2645 * MarkOneCard(CardOfSecondField(mem))
2646 * if (address of the second field isn't aligned at the size of a card)
2647 * # i.e. each of store address (fields of the base objects) are related to the same card
2648 * goto Done
2649 *
2650 * if (the first object isn't null and is allocated in other from base object's region)
2651 * MarkOneCard(CardOfFirstField(mem))
2652 *
2653 * label: Done
2654 */
EncodeOnlineIrtocRegionTwoRegsBarrier(Args args)2655 void PostWriteBarrier::EncodeOnlineIrtocRegionTwoRegsBarrier(Args args)
2656 {
2657 static constexpr auto ENTRYPOINT_ID {RuntimeInterface::EntrypointId::POST_INTER_REGION_BARRIER_SLOW};
2658 auto *enc {cg_->GetEncoder()};
2659 auto base = GetBase(args);
2660 auto paramReg0 = enc->GetTarget().GetParamReg(0);
2661 MemRef entry(cg_->ThreadReg(), cg_->GetRuntime()->GetEntrypointTlsOffset(cg_->GetArch(), ENTRYPOINT_ID));
2662 auto paramRegs = GetParamRegs(1U, args);
2663 auto lblMarkCardAndExit = enc->CreateLabel();
2664 auto lblCheck1Obj = enc->CreateLabel();
2665 auto lblDone = enc->CreateLabel();
2666 EncodeCheckObject(base, args.reg2, lblCheck1Obj, args.checkObject);
2667 cg_->SaveCallerRegisters(paramRegs, VRegMask(), false);
2668 EncodeWrapOneArg(paramReg0, base, args.mem, args.reg1.GetSize() / BITS_PER_BYTE);
2669 {
2670 ScopedTmpReg tmp(enc, cg_->ConvertDataType(DataType::REFERENCE, cg_->GetArch()));
2671 enc->EncodeAnd(tmp, paramReg0, Imm(cross_values::GetCardAlignmentMask(cg_->GetArch())));
2672 enc->EncodeJump(lblMarkCardAndExit, tmp, Condition::NE);
2673 }
2674 enc->MakeCall(entry);
2675 cg_->LoadCallerRegisters(paramRegs, VRegMask(), false);
2676 enc->BindLabel(lblCheck1Obj);
2677 EncodeCheckObject(base, args.reg1, lblDone, args.checkObject);
2678 cg_->SaveCallerRegisters(paramRegs, VRegMask(), false);
2679 EncodeWrapOneArg(paramReg0, base, args.mem);
2680 enc->BindLabel(lblMarkCardAndExit);
2681 enc->MakeCall(entry);
2682 cg_->LoadCallerRegisters(paramRegs, VRegMask(), false);
2683 enc->BindLabel(lblDone);
2684 }
2685
EncodeOnlineIrtocRegionOneRegBarrier(Args args)2686 void PostWriteBarrier::EncodeOnlineIrtocRegionOneRegBarrier(Args args)
2687 {
2688 static constexpr auto ENTRYPOINT_ID {RuntimeInterface::EntrypointId::POST_INTER_REGION_BARRIER_SLOW};
2689 auto *enc {cg_->GetEncoder()};
2690 auto base = GetBase(args);
2691 auto paramReg0 = enc->GetTarget().GetParamReg(0);
2692 MemRef entry(cg_->ThreadReg(), cg_->GetRuntime()->GetEntrypointTlsOffset(cg_->GetArch(), ENTRYPOINT_ID));
2693 auto paramRegs = GetParamRegs(1U, args);
2694 auto skip = enc->CreateLabel();
2695 EncodeCheckObject(base, args.reg1, skip, args.checkObject);
2696 cg_->SaveCallerRegisters(paramRegs, VRegMask(), false);
2697 EncodeWrapOneArg(paramReg0, base, args.mem);
2698 enc->MakeCall(entry);
2699 cg_->LoadCallerRegisters(paramRegs, VRegMask(), false);
2700 enc->BindLabel(skip);
2701 }
2702
EncodeInterRegionBarrier(Args args)2703 void PostWriteBarrier::EncodeInterRegionBarrier(Args args)
2704 {
2705 SCOPED_DISASM_STR(cg_, "Post IR-WRB");
2706 ASSERT(type_ == ark::mem::BarrierType::POST_INTERREGION_BARRIER);
2707
2708 static constexpr auto ENTRYPOINT_ID {RuntimeInterface::EntrypointId::POST_WRB_UPDATE_CARD_FUNC_NO_BRIDGE};
2709 auto *enc {cg_->GetEncoder()};
2710 auto label = enc->CreateLabel();
2711 if (args.checkObject) {
2712 cg_->CheckObject(args.reg1, label);
2713 }
2714 auto regionSizeBit = GetBarrierOperandValue<uint8_t>(BARRIER_POSITION, "REGION_SIZE_BITS");
2715 auto base = GetBase(args);
2716 ScopedTmpReg tmp(enc, cg_->ConvertDataType(DataType::REFERENCE, cg_->GetArch()));
2717 // Compare first store value with mem
2718 enc->EncodeXor(tmp, base, args.reg1);
2719 enc->EncodeShr(tmp, tmp, Imm(regionSizeBit));
2720 enc->EncodeJump(label, tmp, Condition::EQ);
2721 auto [live_regs, live_vregs] = cg_->GetLiveRegisters<true>(inst_);
2722 live_regs |= args.preserved;
2723
2724 if (args.mem.HasIndex()) {
2725 ASSERT(args.mem.GetScale() == 0 && !args.mem.HasDisp());
2726 enc->EncodeAdd(tmp, base, args.mem.GetIndex());
2727 cg_->CallBarrier(live_regs, live_vregs, ENTRYPOINT_ID, tmp, args.reg1);
2728 } else if (args.mem.HasDisp()) {
2729 ASSERT(!args.mem.HasIndex());
2730 enc->EncodeAdd(tmp, base, Imm(args.mem.GetDisp()));
2731 cg_->CallBarrier(live_regs, live_vregs, ENTRYPOINT_ID, tmp, args.reg1);
2732 } else {
2733 cg_->CallBarrier(live_regs, live_vregs, ENTRYPOINT_ID, base, args.reg1);
2734 }
2735 enc->BindLabel(label);
2736
2737 if (HasObject2(args)) {
2738 auto label1 = enc->CreateLabel();
2739 if (args.checkObject) {
2740 cg_->CheckObject(args.reg2, label1);
2741 }
2742 // Compare second store value with mem
2743 enc->EncodeXor(tmp, base, args.reg2);
2744 enc->EncodeShr(tmp, tmp, Imm(regionSizeBit));
2745 enc->EncodeJump(label1, tmp, Condition::EQ);
2746 enc->EncodeAdd(tmp, base, Imm(args.reg1.GetSize() / BITS_PER_BYTE));
2747 if (args.mem.HasIndex()) {
2748 ASSERT(args.mem.GetScale() == 0 && !args.mem.HasDisp());
2749 enc->EncodeAdd(tmp, tmp, args.mem.GetIndex());
2750 } else if (args.mem.HasDisp()) {
2751 ASSERT(!args.mem.HasIndex());
2752 enc->EncodeAdd(tmp, tmp, Imm(args.mem.GetDisp()));
2753 }
2754 cg_->CallBarrier(live_regs, live_vregs, ENTRYPOINT_ID, tmp, args.reg2);
2755 enc->BindLabel(label1);
2756 }
2757 }
2758
EncodeInterGenerationalBarrier(Reg base)2759 void PostWriteBarrier::EncodeInterGenerationalBarrier(Reg base)
2760 {
2761 ASSERT(type_ == ark::mem::BarrierType::POST_INTERGENERATIONAL_BARRIER);
2762 SCOPED_DISASM_STR(cg_, "Post IG-WRB");
2763 auto *runtime {cg_->GetRuntime()};
2764 auto *enc {cg_->GetEncoder()};
2765 ScopedTmpReg tmp(enc);
2766 ScopedTmpReg tmp1(enc);
2767 // load AddressOf(MIN_ADDR) -> min_addr
2768 if (cg_->GetGraph()->IsOfflineCompilationMode()) {
2769 enc->EncodeLdr(tmp, false, MemRef(cg_->ThreadReg(), runtime->GetTlsCardTableMinAddrOffset(cg_->GetArch())));
2770 } else {
2771 auto minAddress = reinterpret_cast<uintptr_t>(GetBarrierOperandValue<void *>(BARRIER_POSITION, "MIN_ADDR"));
2772 enc->EncodeMov(tmp, Imm(minAddress));
2773 }
2774 // card_index = (AddressOf(obj.field) - min_addr) >> CARD_BITS
2775 EncodeCalculateCardIndex(base, &tmp, &tmp1);
2776 // load AddressOf(CARD_TABLE_ADDR) -> card_table_addr
2777 if (cg_->GetGraph()->IsOfflineCompilationMode()) {
2778 enc->EncodeLdr(tmp1.GetReg().As(INT64_TYPE), false,
2779 MemRef(cg_->ThreadReg(), runtime->GetTlsCardTableAddrOffset(cg_->GetArch())));
2780 } else {
2781 auto cardTableAddr =
2782 reinterpret_cast<uintptr_t>(GetBarrierOperandValue<uint8_t *>(BARRIER_POSITION, "CARD_TABLE_ADDR"));
2783 enc->EncodeMov(tmp1, Imm(cardTableAddr));
2784 }
2785 // card_addr = card_table_addr + card_index
2786 enc->EncodeAdd(tmp, tmp1, tmp);
2787 // store card_addr <- DIRTY_VAL
2788 auto dirtyVal = GetBarrierOperandValue<uint8_t>(BARRIER_POSITION, "DIRTY_VAL");
2789 auto tmp1B = cg_->ConvertRegister(tmp1.GetReg().GetId(), DataType::INT8);
2790 enc->EncodeMov(tmp1B, Imm(dirtyVal));
2791 enc->EncodeStr(tmp1B, MemRef(tmp));
2792 }
2793
EncodeCalculateCardIndex(Reg baseReg,ScopedTmpReg * tmp,ScopedTmpReg * tmp1)2794 void PostWriteBarrier::EncodeCalculateCardIndex(Reg baseReg, ScopedTmpReg *tmp, ScopedTmpReg *tmp1)
2795 {
2796 ASSERT(baseReg != INVALID_REGISTER);
2797 auto tmpType = tmp->GetReg().GetType();
2798 auto tmp1Type = tmp1->GetReg().GetType();
2799 if (baseReg.GetSize() < Reg(*tmp).GetSize()) {
2800 tmp->ChangeType(baseReg.GetType());
2801 tmp1->ChangeType(baseReg.GetType());
2802 }
2803 auto enc {cg_->GetEncoder()};
2804 enc->EncodeSub(*tmp, baseReg, *tmp);
2805 enc->EncodeShr(*tmp, *tmp, Imm(GetBarrierOperandValue<uint8_t>(BARRIER_POSITION, "CARD_BITS")));
2806 tmp->ChangeType(tmpType);
2807 tmp1->ChangeType(tmp1Type);
2808 }
2809
EncodeCheckObject(Reg base,Reg reg1,LabelHolder::LabelId skipLabel,bool checkNull)2810 void PostWriteBarrier::EncodeCheckObject(Reg base, Reg reg1, LabelHolder::LabelId skipLabel, bool checkNull)
2811 {
2812 auto enc {cg_->GetEncoder()};
2813 if (checkNull) {
2814 // Fast null check in-place for one register
2815 enc->EncodeJump(skipLabel, reg1, Condition::EQ);
2816 }
2817 ScopedTmpReg tmp(enc, cg_->ConvertDataType(DataType::REFERENCE, cg_->GetArch()));
2818 auto regionSizeBit = GetBarrierOperandValue<uint8_t>(BARRIER_POSITION, "REGION_SIZE_BITS");
2819 enc->EncodeXor(tmp, base, reg1);
2820 enc->EncodeShr(tmp, tmp, Imm(regionSizeBit));
2821 enc->EncodeJump(skipLabel, tmp, Condition::EQ);
2822 }
2823
EncodeWrapOneArg(Reg param,Reg base,MemRef mem,size_t additionalOffset)2824 void PostWriteBarrier::EncodeWrapOneArg(Reg param, Reg base, MemRef mem, size_t additionalOffset)
2825 {
2826 auto enc {cg_->GetEncoder()};
2827 if (mem.HasIndex()) {
2828 ASSERT(mem.GetScale() == 0 && !mem.HasDisp());
2829 enc->EncodeAdd(param, base, mem.GetIndex());
2830 if (additionalOffset != 0) {
2831 enc->EncodeAdd(param, param, Imm(additionalOffset));
2832 }
2833 } else if (mem.HasDisp()) {
2834 ASSERT(!mem.HasIndex());
2835 enc->EncodeAdd(param, base, Imm(mem.GetDisp() + additionalOffset));
2836 } else {
2837 enc->EncodeAdd(param, base, Imm(additionalOffset));
2838 }
2839 }
2840
2841 } // namespace ark::compiler
2842