1 /*
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 /*
17 Codegen Hi-Level implementation
18 */
19 #include "operands.h"
20 #include "codegen.h"
21 #include "encode_visitor.h"
22 #include "compiler_options.h"
23 #include "relocations.h"
24 #include "include/compiler_interface.h"
25 #include "ir-dyn-base-types.h"
26 #include "runtime/include/coretypes/string.h"
27 #include "compiler/optimizer/ir/analysis.h"
28 #include "compiler/optimizer/ir/locations.h"
29 #include "compiler/optimizer/analysis/liveness_analyzer.h"
30 #include "optimizer/code_generator/method_properties.h"
31 #include "events/events.h"
32 #include "libpandabase/utils/tsan_interface.h"
33 #include "libpandabase/utils/utils.h"
34 #include <algorithm>
35 #include <iomanip>
36
37 namespace ark::compiler {
38
39 class OsrEntryStub {
FixIntervals(Codegen * codegen,Encoder * encoder)40 void FixIntervals(Codegen *codegen, Encoder *encoder)
41 {
42 auto &la = codegen->GetGraph()->GetAnalysis<LivenessAnalyzer>();
43 la.EnumerateLiveIntervalsForInst(saveState_, [this, codegen, encoder](const auto &li) {
44 auto inst = li->GetInst();
45 auto location = li->GetLocation();
46 // Skip live registers that are already in the input list of the OsrSaveState
47 const auto &ssInputs = saveState_->GetInputs();
48 if (std::find_if(ssInputs.begin(), ssInputs.end(),
49 [inst](auto &input) { return input.GetInst() == inst; }) != ssInputs.end()) {
50 return;
51 }
52 // Only constants allowed
53 switch (inst->GetOpcode()) {
54 case Opcode::LoadAndInitClass: {
55 auto klass = reinterpret_cast<uintptr_t>(inst->CastToLoadAndInitClass()->GetClass());
56 encoder->EncodeMov(codegen->ConvertRegister(inst->GetDstReg(), inst->GetType()),
57 Imm(reinterpret_cast<uintptr_t>(klass)));
58 break;
59 }
60 case Opcode::Constant: {
61 if (location.IsFixedRegister()) {
62 EncodeConstantMove(li, encoder);
63 } else if (location.IsStack()) {
64 auto slot = location.GetValue();
65 encoder->EncodeSti(
66 bit_cast<int64_t>(inst->CastToConstant()->GetRawValue()), DOUBLE_WORD_SIZE_BYTES,
67 MemRef(codegen->SpReg(), codegen->GetFrameLayout().GetSpillOffsetFromSpInBytes(slot)));
68 } else {
69 ASSERT(location.IsConstant());
70 }
71 break;
72 }
73 // NOTE (ekudriashov): UnresolvedLoadAndInitClass
74 default:
75 break;
76 }
77 });
78 }
79
EncodeConstantMove(const LifeIntervals * li,Encoder * encoder)80 static void EncodeConstantMove(const LifeIntervals *li, Encoder *encoder)
81 {
82 auto inst = li->GetInst();
83 switch (li->GetType()) {
84 case DataType::FLOAT64:
85 encoder->EncodeMov(Reg(li->GetReg(), FLOAT64_TYPE),
86 Imm(bit_cast<double>(inst->CastToConstant()->GetDoubleValue())));
87 break;
88 case DataType::FLOAT32:
89 encoder->EncodeMov(Reg(li->GetReg(), FLOAT32_TYPE),
90 Imm(bit_cast<float>(inst->CastToConstant()->GetFloatValue())));
91 break;
92 case DataType::UINT32:
93 encoder->EncodeMov(Reg(li->GetReg(), INT32_TYPE), Imm(inst->CastToConstant()->GetRawValue()));
94 break;
95 case DataType::UINT64:
96 encoder->EncodeMov(Reg(li->GetReg(), INT64_TYPE), Imm(inst->CastToConstant()->GetRawValue()));
97 break;
98 default:
99 UNREACHABLE();
100 }
101 }
102
103 public:
OsrEntryStub(Codegen * codegen,SaveStateInst * inst)104 OsrEntryStub(Codegen *codegen, SaveStateInst *inst) : label_(codegen->GetEncoder()->CreateLabel()), saveState_(inst)
105 {
106 }
107
108 DEFAULT_MOVE_SEMANTIC(OsrEntryStub);
109 DEFAULT_COPY_SEMANTIC(OsrEntryStub);
110 ~OsrEntryStub() = default;
111
Generate(Codegen * codegen)112 void Generate(Codegen *codegen)
113 {
114 auto encoder = codegen->GetEncoder();
115 auto lr = codegen->GetTarget().GetLinkReg();
116 auto fl = codegen->GetFrameLayout();
117 codegen->CreateStackMap(saveState_->CastToSaveStateOsr());
118 ssize_t slot = CFrameLayout::LOCALS_START_SLOT + CFrameLayout::GetLocalsCount() - 1U;
119 encoder->EncodeStp(
120 codegen->FpReg(), lr,
121 MemRef(codegen->FpReg(),
122 -fl.GetOffset<CFrameLayout::OffsetOrigin::FP, CFrameLayout::OffsetUnit::BYTES>(slot)));
123
124 FixIntervals(codegen, encoder);
125 encoder->EncodeJump(label_);
126 }
127
GetInst()128 SaveStateInst *GetInst()
129 {
130 return saveState_;
131 }
132
GetLabel()133 auto &GetLabel()
134 {
135 return label_;
136 }
137
138 private:
139 LabelHolder::LabelId label_;
140 SaveStateInst *saveState_ {nullptr};
141 };
142
Codegen(Graph * graph)143 Codegen::Codegen(Graph *graph)
144 : Optimization(graph),
145 allocator_(graph->GetAllocator()),
146 localAllocator_(graph->GetLocalAllocator()),
147 codeBuilder_(allocator_->New<CodeInfoBuilder>(graph->GetArch(), allocator_)),
148 slowPaths_(graph->GetLocalAllocator()->Adapter()),
149 slowPathsMap_(graph->GetLocalAllocator()->Adapter()),
150 frameLayout_(CFrameLayout(graph->GetArch(), graph->GetStackSlotsCount(), !graph->GetMode().IsFastPath())),
151 osrEntries_(graph->GetLocalAllocator()->Adapter()),
152 vregIndices_(GetAllocator()->Adapter()),
153 runtime_(graph->GetRuntime()),
154 target_(graph->GetArch()),
155 liveOuts_(graph->GetLocalAllocator()->Adapter()),
156 disasm_(this),
157 spillFillsResolver_(graph)
158 {
159 graph->SetCodeBuilder(codeBuilder_);
160 regfile_ = graph->GetRegisters();
161 if (regfile_ != nullptr) {
162 ASSERT(regfile_->IsValid());
163 ArenaVector<Reg> regsUsage(allocator_->Adapter());
164 Convert(®sUsage, graph->GetUsedRegs<DataType::INT64>(), INT64_TYPE);
165 Convert(®sUsage, graph->GetUsedRegs<DataType::FLOAT64>(), FLOAT64_TYPE);
166 regfile_->SetUsedRegs(regsUsage);
167 #ifndef NDEBUG
168 COMPILER_LOG(DEBUG, CODEGEN) << "Regalloc used registers scalar " << graph->GetUsedRegs<DataType::INT64>();
169 COMPILER_LOG(DEBUG, CODEGEN) << "Regalloc used registers vector " << graph->GetUsedRegs<DataType::FLOAT64>();
170 #endif
171 }
172
173 enc_ = graph->GetEncoder();
174 ASSERT(enc_ != nullptr && enc_->IsValid());
175 enc_->SetRegfile(regfile_);
176 if (enc_->InitMasm()) {
177 enc_->SetFrameLayout(GetFrameLayout());
178 }
179
180 callconv_ = graph->GetCallingConvention();
181 if (callconv_ != nullptr) {
182 ASSERT(callconv_->IsValid());
183 if (callconv_->GetEncoder() == nullptr) {
184 callconv_->SetEncoder(enc_);
185 }
186 }
187
188 auto method = graph->GetMethod();
189 // workaround for test
190 if (method != nullptr) {
191 methodId_ = graph->GetRuntime()->GetMethodId(method);
192 }
193 GetDisasm()->Init();
194 GetDisasm()->SetEncoder(GetEncoder());
195 }
196
GetPassName() const197 const char *Codegen::GetPassName() const
198 {
199 return "Codegen";
200 }
201
AbortIfFailed() const202 bool Codegen::AbortIfFailed() const
203 {
204 return true;
205 }
206
CreateFrameInfo()207 void Codegen::CreateFrameInfo()
208 {
209 // Create FrameInfo for CFrame
210 auto &fl = GetFrameLayout();
211 auto frame = GetGraph()->GetLocalAllocator()->New<FrameInfo>(
212 FrameInfo::PositionedCallers::Encode(true) | FrameInfo::PositionedCallees::Encode(true) |
213 FrameInfo::CallersRelativeFp::Encode(false) | FrameInfo::CalleesRelativeFp::Encode(true));
214 frame->SetFrameSize(fl.GetFrameSize<CFrameLayout::OffsetUnit::BYTES>());
215 frame->SetSpillsCount(fl.GetSpillsCount());
216
217 frame->SetCallersOffset(fl.GetOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::SLOTS>(
218 fl.GetStackStartSlot() + fl.GetCallerLastSlot(false)));
219 frame->SetFpCallersOffset(fl.GetOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::SLOTS>(
220 fl.GetStackStartSlot() + fl.GetCallerLastSlot(true)));
221 frame->SetCalleesOffset(-fl.GetOffset<CFrameLayout::OffsetOrigin::FP, CFrameLayout::OffsetUnit::SLOTS>(
222 fl.GetStackStartSlot() + fl.GetCalleeLastSlot(false)));
223 frame->SetFpCalleesOffset(-fl.GetOffset<CFrameLayout::OffsetOrigin::FP, CFrameLayout::OffsetUnit::SLOTS>(
224 fl.GetStackStartSlot() + fl.GetCalleeLastSlot(true)));
225
226 frame->SetSetupFrame(true);
227 frame->SetSaveFrameAndLinkRegs(true);
228 frame->SetSaveUnusedCalleeRegs(!GetGraph()->GetMethodProperties().GetCompactPrologueAllowed());
229 frame->SetAdjustSpReg(true);
230 frame->SetHasFloatRegs(GetGraph()->HasFloatRegs());
231
232 GetCodeBuilder()->SetHasFloatRegs(GetGraph()->HasFloatRegs());
233
234 SetFrameInfo(frame);
235 }
236
FillOnlyParameters(RegMask * liveRegs,uint32_t numParams,bool isFastpath) const237 void Codegen::FillOnlyParameters(RegMask *liveRegs, uint32_t numParams, bool isFastpath) const
238 {
239 ASSERT(numParams <= 6U);
240 if (GetArch() == Arch::AARCH64 && !isFastpath) {
241 numParams = AlignUp(numParams, 2U);
242 }
243 *liveRegs &= GetTarget().GetParamRegsMask(numParams);
244 }
245
Convert(ArenaVector<Reg> * regsUsage,const ArenaVector<bool> * mask,TypeInfo typeInfo)246 void Codegen::Convert(ArenaVector<Reg> *regsUsage, const ArenaVector<bool> *mask, TypeInfo typeInfo)
247 {
248 ASSERT(regsUsage != nullptr);
249 // There are no used registers
250 if (mask == nullptr) {
251 return;
252 }
253 ASSERT(mask->size() == MAX_NUM_REGS);
254 for (uint32_t i = 0; i < MAX_NUM_REGS; ++i) {
255 if ((*mask)[i]) {
256 regsUsage->emplace_back(i, typeInfo);
257 }
258 }
259 }
260
261 #ifdef IRTOC_INTRINSICS_ENABLED
EmitSimdIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)262 void Codegen::EmitSimdIntrinsic([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst,
263 [[maybe_unused]] SRCREGS src)
264 {
265 ASSERT(0);
266 GetEncoder()->SetFalseResult();
267 }
268
EmitReverseIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)269 void Codegen::EmitReverseIntrinsic([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst,
270 [[maybe_unused]] SRCREGS src)
271 {
272 ASSERT(0);
273 GetEncoder()->SetFalseResult();
274 }
275
EmitMarkWordIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)276 void Codegen::EmitMarkWordIntrinsic([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst,
277 [[maybe_unused]] SRCREGS src)
278 {
279 ASSERT(0);
280 GetEncoder()->SetFalseResult();
281 }
282
EmitDataMemoryBarrierFullIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)283 void Codegen::EmitDataMemoryBarrierFullIntrinsic([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst,
284 [[maybe_unused]] SRCREGS src)
285 {
286 ASSERT(0);
287 GetEncoder()->SetFalseResult();
288 }
289
EmitWriteTlabStatsSafeIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)290 void Codegen::EmitWriteTlabStatsSafeIntrinsic([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst,
291 [[maybe_unused]] SRCREGS src)
292 {
293 ASSERT(0);
294 GetEncoder()->SetFalseResult();
295 }
296
EmitExpandU8ToU16Intrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)297 void Codegen::EmitExpandU8ToU16Intrinsic([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst,
298 [[maybe_unused]] SRCREGS src)
299 {
300 ASSERT(0);
301 GetEncoder()->SetFalseResult();
302 }
303
EmitAtomicByteOrIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)304 void Codegen::EmitAtomicByteOrIntrinsic([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst,
305 [[maybe_unused]] SRCREGS src)
306 {
307 ASSERT(0);
308 GetEncoder()->SetFalseResult();
309 }
310
EmitSaveOrRestoreRegsEpIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)311 void Codegen::EmitSaveOrRestoreRegsEpIntrinsic([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst,
312 [[maybe_unused]] SRCREGS src)
313 {
314 ASSERT(0);
315 GetEncoder()->SetFalseResult();
316 }
317
EmitTailCallIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)318 void Codegen::EmitTailCallIntrinsic([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst,
319 [[maybe_unused]] SRCREGS src)
320 {
321 ASSERT(0);
322 GetEncoder()->SetFalseResult();
323 }
324
EmitSlowPathEntryIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)325 void Codegen::EmitSlowPathEntryIntrinsic([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst,
326 [[maybe_unused]] SRCREGS src)
327 {
328 ASSERT(0);
329 GetEncoder()->SetFalseResult();
330 }
331
332 /* It is the same for CodegenFastPath and CodegenInterpreter */
EmitUnreachableIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)333 void Codegen::EmitUnreachableIntrinsic([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst,
334 [[maybe_unused]] SRCREGS src)
335 {
336 ASSERT(inst->GetIntrinsicId() == RuntimeInterface::IntrinsicId::INTRINSIC_UNREACHABLE);
337 GetEncoder()->EncodeAbort();
338 }
339
340 /* It is the same for CodegenFastPath and CodegenInterpreter */
EmitInterpreterReturnIntrinsic(IntrinsicInst * inst,Reg dst,SRCREGS src)341 void Codegen::EmitInterpreterReturnIntrinsic([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst,
342 [[maybe_unused]] SRCREGS src)
343 {
344 ASSERT(inst->GetIntrinsicId() == RuntimeInterface::IntrinsicId::INTRINSIC_INTERPRETER_RETURN);
345 GetCallingConvention()->GenerateNativeEpilogue(*GetFrameInfo(), []() {});
346 }
347 #endif // IRTOC_INTRINSICS_ENABLED
348
BeginMethod()349 bool Codegen::BeginMethod()
350 {
351 // Do not try to encode too large graph
352 auto estimatedCodeSize = GetGraph()->EstimateCodeSize();
353 if (estimatedCodeSize > g_options.GetCompilerMaxGenCodeSize()) {
354 COMPILER_LOG(WARNING, CODEGEN) << "Code is too large - estimated code size: " << estimatedCodeSize;
355 return false;
356 }
357 // Limit encoder to match estimated to actual code size.
358 // After this - encoder aborted, if allocated too much size.
359 GetEncoder()->SetMaxAllocatedBytes(g_options.GetCompilerMaxGenCodeSize());
360
361 if (!IsCompressedStringsEnabled()) {
362 #ifndef NDEBUG
363 LOG(FATAL, COMPILER) << "String compression must be enabled";
364 #else
365 LOG(ERROR, COMPILER) << "String compression must be enabled";
366 #endif
367 return false;
368 }
369
370 if (GetGraph()->IsAotMode()) {
371 GetEncoder()->SetCodeOffset(GetGraph()->GetAotData()->GetCodeOffset() + CodeInfo::GetCodeOffset(GetArch()));
372 } else {
373 GetEncoder()->SetCodeOffset(0);
374 }
375
376 codeBuilder_->BeginMethod(GetFrameLayout().GetFrameSize<CFrameLayout::OffsetUnit::BYTES>(),
377 GetGraph()->GetVRegsCount() + GetGraph()->GetEnvCount());
378
379 GetEncoder()->BindLabel(GetLabelEntry());
380 SetStartCodeOffset(GetEncoder()->GetCursorOffset());
381
382 GeneratePrologue();
383
384 return GetEncoder()->GetResult();
385 }
386
GeneratePrologue()387 void Codegen::GeneratePrologue()
388 {
389 SCOPED_DISASM_STR(this, "Method Prologue");
390
391 GetCallingConvention()->GeneratePrologue(*frameInfo_);
392
393 if (!GetGraph()->GetMode().IsNative()) {
394 GetEncoder()->EncodeSti(1, 1, MemRef(ThreadReg(), GetRuntime()->GetTlsFrameKindOffset(GetArch())));
395 }
396 if (!GetGraph()->GetMode().IsNative()) {
397 // Create stack overflow check
398 GetEncoder()->EncodeStackOverflowCheck(-GetRuntime()->GetStackOverflowCheckOffset());
399 // Create empty stackmap for the stack overflow check
400 SaveStateInst ss(Opcode::SaveState, 0, nullptr, nullptr, 0);
401 GetCodeBuilder()->BeginStackMap(0, 0, &ss, false);
402 GetCodeBuilder()->EndStackMap();
403 }
404
405 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
406 MakeTrace();
407 #endif
408 }
409
410 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
MakeTrace()411 void Codegen::MakeTrace()
412 {
413 if (GetGraph()->IsAotMode()) {
414 SCOPED_DISASM_STR(this, "LoadMethod for trace");
415 ScopedTmpReg method_reg(GetEncoder());
416 LoadMethod(method_reg);
417 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_ENTER)), method_reg,
418 Imm(static_cast<size_t>(events::MethodEnterKind::COMPILED)));
419 } else {
420 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
421 Imm(reinterpret_cast<size_t>(GetGraph()->GetMethod())),
422 Imm(static_cast<size_t>(events::MethodEnterKind::COMPILED)));
423 }
424 }
425 #endif
426
GenerateEpilogue()427 void Codegen::GenerateEpilogue()
428 {
429 SCOPED_DISASM_STR(this, "Method Epilogue");
430
431 #if defined(EVENT_METHOD_EXIT_ENABLED) && EVENT_METHOD_EXIT_ENABLED != 0
432 GetCallingConvention()->GenerateEpilogue(*frame_info_, MakeTrace);
433 #else
434 GetCallingConvention()->GenerateEpilogue(*frameInfo_, []() {});
435 #endif
436 }
437
VisitGraph()438 bool Codegen::VisitGraph()
439 {
440 EncodeVisitor visitor(this);
441 visitor_ = &visitor;
442
443 const auto &blocks = GetGraph()->GetBlocksLinearOrder();
444
445 for (auto bb : blocks) {
446 GetEncoder()->BindLabel(bb->GetId());
447 for (auto inst : bb->AllInsts()) {
448 SCOPED_DISASM_INST(this, inst);
449
450 #ifdef PANDA_COMPILER_DEBUG_INFO
451 auto debugInfo = inst->GetDebugInfo();
452 if (GetGraph()->IsLineDebugInfoEnabled() && debugInfo != nullptr) {
453 debugInfo->SetOffset(GetEncoder()->GetCursorOffset());
454 }
455 #endif
456 visitor.VisitInstruction(inst);
457 if (!visitor.GetResult()) {
458 COMPILER_LOG(DEBUG, CODEGEN)
459 << "Can't encode instruction: " << GetOpcodeString(inst->GetOpcode()) << *inst;
460 break;
461 }
462 }
463
464 if (bb->NeedsJump()) {
465 EmitJump(bb);
466 }
467
468 if (!visitor.GetResult()) {
469 return false;
470 }
471 }
472
473 auto estimatedSlowPathsSize = slowPaths_.size() * SLOW_PATH_SIZE;
474 auto estimatedCodeSize = GetGraph()->EstimateCodeSize<true>() + estimatedSlowPathsSize;
475 if (estimatedCodeSize > g_options.GetCompilerMaxGenCodeSize()) {
476 COMPILER_LOG(WARNING, CODEGEN) << "Code is too large - code size: " << GetGraph()->EstimateCodeSize<true>()
477 << ", slow paths' code estimation: " << estimatedSlowPathsSize;
478 return false;
479 }
480
481 EmitSlowPaths();
482 visitor_ = nullptr;
483
484 return true;
485 }
486
EmitJump(const BasicBlock * bb)487 void Codegen::EmitJump(const BasicBlock *bb)
488 {
489 BasicBlock *sucBb = nullptr;
490
491 if (bb->GetLastInst() == nullptr) {
492 ASSERT(bb->IsEmpty());
493 sucBb = bb->GetSuccsBlocks()[0];
494 } else {
495 switch (bb->GetLastInst()->GetOpcode()) {
496 case Opcode::If:
497 case Opcode::IfImm:
498 ASSERT(bb->GetSuccsBlocks().size() == MAX_SUCCS_NUM);
499 sucBb = bb->GetFalseSuccessor();
500 break;
501 default:
502 sucBb = bb->GetSuccsBlocks()[0];
503 break;
504 }
505 }
506 SCOPED_DISASM_STR(this, std::string("Jump from BB_") + std::to_string(bb->GetId()) + " to BB_" +
507 std::to_string(sucBb->GetId()));
508
509 auto label = sucBb->GetId();
510 GetEncoder()->EncodeJump(label);
511 }
512
EndMethod()513 void Codegen::EndMethod()
514 {
515 for (auto &osrStub : osrEntries_) {
516 SCOPED_DISASM_STR(this,
517 std::string("Osr stub for OsrStateStump ") + std::to_string(osrStub->GetInst()->GetId()));
518 osrStub->Generate(this);
519 }
520
521 GetEncoder()->Finalize();
522 }
523
524 // Allocates memory, copies generated code to it, sets the code to the graph's codegen data. Also this function
525 // encodes the code info and sets it to the graph.
CopyToCodeCache()526 bool Codegen::CopyToCodeCache()
527 {
528 auto codeEntry = reinterpret_cast<char *>(GetEncoder()->GetLabelAddress(GetLabelEntry()));
529 auto codeSize = GetEncoder()->GetCursorOffset();
530 bool saveAllCalleeRegisters = !GetGraph()->GetMethodProperties().GetCompactPrologueAllowed();
531
532 auto code = reinterpret_cast<uint8_t *>(GetAllocator()->Alloc(codeSize));
533 if (code == nullptr) {
534 return false;
535 }
536 std::copy_n(codeEntry, codeSize, code);
537 GetGraph()->SetCode(EncodeDataType(code, codeSize));
538
539 RegMask calleeRegs;
540 VRegMask calleeVregs;
541 GetRegfile()->FillUsedCalleeSavedRegisters(&calleeRegs, &calleeVregs, saveAllCalleeRegisters);
542 constexpr size_t MAX_NUM_REGISTERS = 32;
543 static_assert(MAX_NUM_REGS <= MAX_NUM_REGISTERS && MAX_NUM_VREGS <= MAX_NUM_REGISTERS);
544 codeBuilder_->SetSavedCalleeRegsMask(static_cast<uint32_t>(calleeRegs.to_ulong()),
545 static_cast<uint32_t>(calleeVregs.to_ulong()));
546
547 ArenaVector<uint8_t> codeInfoData(GetGraph()->GetAllocator()->Adapter());
548 codeBuilder_->Encode(&codeInfoData);
549 GetGraph()->SetCodeInfo(Span(codeInfoData));
550
551 return true;
552 }
553
IssueDisasm()554 void Codegen::IssueDisasm()
555 {
556 if (GetGraph()->GetMode().SupportManagedCode() && g_options.IsCompilerDisasmDumpCodeInfo()) {
557 GetDisasm()->PrintCodeInfo(this);
558 }
559 GetDisasm()->PrintCodeStatistics(this);
560 auto &stream = GetDisasm()->GetStream();
561 for (auto &item : GetDisasm()->GetItems()) {
562 if (std::holds_alternative<CodeItem>(item)) {
563 auto &code = std::get<CodeItem>(item);
564 for (size_t pc = code.GetPosition(); pc < code.GetCursorOffset();) {
565 stream << GetDisasm()->GetIndent(code.GetDepth());
566 auto newPc = GetEncoder()->DisasmInstr(stream, pc, 0);
567 stream << "\n";
568 pc = newPc;
569 }
570 } else if (std::holds_alternative<std::string>(item)) {
571 stream << (std::get<std::string>(item));
572 }
573 }
574 }
575
RunImpl()576 bool Codegen::RunImpl()
577 {
578 Initialize();
579
580 auto encoder = GetEncoder();
581 encoder->GetLabels()->CreateLabels(GetGraph()->GetVectorBlocks().size());
582 labelEntry_ = encoder->CreateLabel();
583 labelExit_ = encoder->CreateLabel();
584
585 #ifndef NDEBUG
586 if (g_options.IsCompilerNonOptimizing()) {
587 // In case of non-optimizing compiler lowering pass is not run but low-level instructions may
588 // also appear on codegen so, to satisfy GraphChecker, the flag should be raised.
589 GetGraph()->SetLowLevelInstructionsEnabled();
590 }
591 #endif // NDEBUG
592
593 if ((GetCallingConvention() == nullptr) || (GetEncoder() == nullptr)) {
594 return false;
595 }
596
597 if (GetCallingConvention()->IsDynCallMode()) {
598 auto numExpectedArgs = GetRuntime()->GetMethodTotalArgumentsCount(GetGraph()->GetMethod());
599 if (numExpectedArgs > GetRuntime()->GetDynamicNumFixedArgs()) {
600 CallConvDynInfo dynInfo(numExpectedArgs,
601 GetRuntime()->GetEntrypointTlsOffset(
602 GetArch(), RuntimeInterface::EntrypointId::EXPAND_COMPILED_CODE_ARGS_DYN));
603 GetCallingConvention()->SetDynInfo(dynInfo);
604 frameInfo_->SetSaveFrameAndLinkRegs(true);
605 } else {
606 GetCallingConvention()->SetDynInfo(CallConvDynInfo());
607 }
608 }
609
610 if (!GetEncoder()->GetResult()) {
611 return false;
612 }
613
614 // Remove registers from the temp registers, if they are in the regalloc mask, i.e. available for regalloc.
615 auto usedRegs = ~GetGraph()->GetArchUsedRegs();
616 auto forbiddenTemps = usedRegs & GetTarget().GetTempRegsMask();
617 if (forbiddenTemps.Any()) {
618 for (size_t i = forbiddenTemps.GetMinRegister(); i <= forbiddenTemps.GetMaxRegister(); i++) {
619 if (forbiddenTemps[i] && encoder->IsScratchRegisterReleased(Reg(i, INT64_TYPE))) {
620 encoder->AcquireScratchRegister(Reg(i, INT64_TYPE));
621 }
622 }
623 }
624
625 return Finalize();
626 }
627
Initialize()628 void Codegen::Initialize()
629 {
630 CreateFrameInfo();
631
632 GetRegfile()->SetCalleeSaved(GetRegfile()->GetCalleeSaved());
633
634 if (!GetGraph()->SupportManagedCode()) {
635 for (auto inst : GetGraph()->GetStartBlock()->AllInsts()) {
636 if (inst->GetOpcode() == Opcode::LiveIn && GetTarget().GetTempRegsMask().Test(inst->GetDstReg())) {
637 GetEncoder()->AcquireScratchRegister(Reg(inst->GetDstReg(), INT64_TYPE));
638 }
639 }
640 }
641
642 bool hasCalls = false;
643
644 for (auto bb : GetGraph()->GetBlocksLinearOrder()) {
645 // Calls may be in the middle of method
646 for (auto inst : bb->Insts()) {
647 // For throw instruction need jump2runtime same way
648 if (inst->IsCall() || inst->GetOpcode() == Opcode::Throw) {
649 hasCalls = true;
650 break;
651 }
652 }
653 if (hasCalls) {
654 break;
655 }
656 }
657
658 /* Convert Graph::GetUsedRegs(), which is std::vector<bool>, to simple
659 * RegMask and save it in the Codegen. These masks are used to determine
660 * which registers we need to save in prologue.
661 *
662 * NB! It's related to IRTOC specific prologue generation (see CodegenFastPath etc.).
663 * Arch specific CallingConvention::GenerateProlog() relies on reg usage information
664 * prepared in the Codegen constructor (before Initialize() is called).
665 */
666 auto fillMask = [](RegMask *mask, auto *vector) {
667 if (vector == nullptr) {
668 return;
669 }
670 ASSERT(mask->size() >= vector->size());
671 mask->reset();
672 for (size_t i = 0; i < mask->size(); i++) {
673 if ((*vector)[i]) {
674 mask->set(i);
675 }
676 }
677 };
678 fillMask(&usedRegs_, GetGraph()->GetUsedRegs<DataType::INT64>());
679 fillMask(&usedVregs_, GetGraph()->GetUsedRegs<DataType::FLOAT64>());
680 usedVregs_ &= GetTarget().GetAvailableVRegsMask();
681 usedRegs_ &= GetTarget().GetAvailableRegsMask();
682 usedRegs_ &= ~GetGraph()->GetArchUsedRegs();
683 usedVregs_ &= ~GetGraph()->GetArchUsedVRegs();
684
685 /* Remove used registers from Encoder's scratch registers */
686 RegMask usedTemps = usedRegs_ & GetTarget().GetTempRegsMask();
687 if (usedTemps.any()) {
688 for (size_t i = 0; i < usedTemps.size(); i++) {
689 if (usedTemps[i]) {
690 GetEncoder()->AcquireScratchRegister(Reg(i, INT64_TYPE));
691 }
692 }
693 }
694 }
695
Finalize()696 bool Codegen::Finalize()
697 {
698 if (GetDisasm()->IsEnabled()) {
699 GetDisasm()->PrintMethodEntry(this);
700 }
701
702 if (!BeginMethod()) {
703 return false;
704 }
705
706 if (!VisitGraph()) {
707 return false;
708 }
709 EndMethod();
710
711 if (!CopyToCodeCache()) {
712 return false;
713 }
714
715 if (GetDisasm()->IsEnabled()) {
716 IssueDisasm();
717 }
718
719 return true;
720 }
721
ConvertRegister(Register r,DataType::Type type)722 Reg Codegen::ConvertRegister(Register r, DataType::Type type)
723 {
724 switch (type) {
725 case DataType::BOOL:
726 case DataType::UINT8:
727 case DataType::INT8: {
728 return Reg(r, INT8_TYPE);
729 }
730 case DataType::UINT16:
731 case DataType::INT16: {
732 return Reg(r, INT16_TYPE);
733 }
734 case DataType::UINT32:
735 case DataType::INT32: {
736 return Reg(r, INT32_TYPE);
737 }
738 case DataType::UINT64:
739 case DataType::INT64:
740 case DataType::ANY: {
741 return Reg(r, INT64_TYPE);
742 }
743 case DataType::FLOAT32: {
744 return Reg(r, FLOAT32_TYPE);
745 }
746 case DataType::FLOAT64: {
747 return Reg(r, FLOAT64_TYPE);
748 }
749 case DataType::REFERENCE: {
750 return ConvertRegister(r, DataType::GetIntTypeForReference(GetArch()));
751 }
752 case DataType::POINTER: {
753 return Reg(r, ConvertDataType(DataType::POINTER, GetArch()));
754 }
755 default:
756 // Invalid converted register
757 return INVALID_REGISTER;
758 }
759 }
760
761 // Panda don't support types less then 32, so we need sign or zero extended to 32
ConvertImmWithExtend(uint64_t imm,DataType::Type type)762 Imm Codegen::ConvertImmWithExtend(uint64_t imm, DataType::Type type)
763 {
764 switch (type) {
765 case DataType::BOOL:
766 case DataType::UINT8:
767 return Imm(static_cast<uint32_t>(static_cast<uint8_t>(imm)));
768 case DataType::INT8:
769 return Imm(static_cast<int32_t>(bit_cast<int8_t, uint8_t>(imm)));
770 case DataType::UINT16:
771 return Imm(static_cast<uint32_t>(static_cast<uint16_t>(imm)));
772 case DataType::INT16:
773 return Imm(static_cast<int32_t>(bit_cast<int16_t, uint16_t>(imm)));
774 // NOLINTNEXTLINE(bugprone-branch-clone)
775 case DataType::UINT32:
776 return Imm(bit_cast<int32_t, uint32_t>(imm));
777 case DataType::INT32:
778 return Imm(bit_cast<int32_t, uint32_t>(imm));
779 // NOLINTNEXTLINE(bugprone-branch-clone)
780 case DataType::UINT64:
781 return Imm(bit_cast<int64_t, uint64_t>(imm));
782 case DataType::INT64:
783 return Imm(bit_cast<int64_t, uint64_t>(imm));
784 case DataType::FLOAT32:
785 return Imm(bit_cast<float, uint32_t>(static_cast<uint32_t>(imm)));
786 case DataType::FLOAT64:
787 return Imm(bit_cast<double, uint64_t>(imm));
788 case DataType::ANY:
789 return Imm(bit_cast<uint64_t, uint64_t>(imm));
790 case DataType::REFERENCE:
791 if (imm == 0) {
792 return Imm(0);
793 }
794 [[fallthrough]]; /* fall-through */
795 default:
796 // Invalid converted immediate
797 UNREACHABLE();
798 }
799 return Imm(0);
800 }
801
ConvertCc(ConditionCode cc)802 Condition Codegen::ConvertCc(ConditionCode cc)
803 {
804 switch (cc) {
805 case CC_EQ:
806 return Condition::EQ;
807 case CC_NE:
808 return Condition::NE;
809 case CC_LT:
810 return Condition::LT;
811 case CC_LE:
812 return Condition::LE;
813 case CC_GT:
814 return Condition::GT;
815 case CC_GE:
816 return Condition::GE;
817 case CC_B:
818 return Condition::LO;
819 case CC_BE:
820 return Condition::LS;
821 case CC_A:
822 return Condition::HI;
823 case CC_AE:
824 return Condition::HS;
825 case CC_TST_EQ:
826 return Condition::TST_EQ;
827 case CC_TST_NE:
828 return Condition::TST_NE;
829 default:
830 UNREACHABLE();
831 return Condition::EQ;
832 }
833 return Condition::EQ;
834 }
835
ConvertCcOverflow(ConditionCode cc)836 Condition Codegen::ConvertCcOverflow(ConditionCode cc)
837 {
838 switch (cc) {
839 case CC_EQ:
840 return Condition::VS;
841 case CC_NE:
842 return Condition::VC;
843 default:
844 UNREACHABLE();
845 return Condition::VS;
846 }
847 return Condition::VS;
848 }
849
EmitSlowPaths()850 void Codegen::EmitSlowPaths()
851 {
852 for (auto slowPath : slowPaths_) {
853 slowPath->Generate(this);
854 }
855 }
856
CreateStackMap(Inst * inst,Inst * user)857 void Codegen::CreateStackMap(Inst *inst, Inst *user)
858 {
859 SaveStateInst *saveState = nullptr;
860 if (inst->IsSaveState()) {
861 saveState = static_cast<SaveStateInst *>(inst);
862 } else {
863 saveState = inst->GetSaveState();
864 }
865 ASSERT(saveState != nullptr);
866
867 bool requireVregMap = inst->RequireRegMap();
868 uint32_t outerBpc = inst->GetPc();
869 for (auto callInst = saveState->GetCallerInst(); callInst != nullptr;
870 callInst = callInst->GetSaveState()->GetCallerInst()) {
871 outerBpc = callInst->GetPc();
872 }
873 codeBuilder_->BeginStackMap(outerBpc, GetEncoder()->GetCursorOffset(), saveState, requireVregMap);
874 if (user == nullptr) {
875 user = inst;
876 if (inst == saveState && inst->HasUsers()) {
877 auto users = inst->GetUsers();
878 auto it = std::find_if(users.begin(), users.end(),
879 [](auto &u) { return u.GetInst()->GetOpcode() != Opcode::ReturnInlined; });
880 user = it->GetInst();
881 }
882 }
883 CreateStackMapRec(saveState, requireVregMap, user);
884
885 codeBuilder_->EndStackMap();
886 if (GetDisasm()->IsEnabled()) {
887 GetDisasm()->PrintStackMap(this);
888 }
889 }
890
CreateStackMapRec(SaveStateInst * saveState,bool requireVregMap,Inst * targetSite)891 void Codegen::CreateStackMapRec(SaveStateInst *saveState, bool requireVregMap, Inst *targetSite)
892 {
893 bool hasInlineInfo = saveState->GetCallerInst() != nullptr;
894 size_t vregsCount = 0;
895 if (requireVregMap) {
896 auto runtime = GetRuntime();
897 if (auto caller = saveState->GetCallerInst()) {
898 vregsCount = runtime->GetMethodRegistersCount(caller->GetCallMethod()) +
899 runtime->GetMethodArgumentsCount(caller->GetCallMethod());
900 } else {
901 vregsCount = runtime->GetMethodRegistersCount(saveState->GetMethod()) +
902 runtime->GetMethodArgumentsCount(saveState->GetMethod());
903 }
904 ASSERT(!GetGraph()->IsBytecodeOptimizer());
905 // 1 for acc, number of env regs for dynamic method
906 vregsCount += 1U + GetGraph()->GetEnvCount();
907 #ifndef NDEBUG
908 ASSERT_PRINT(!saveState->GetInputsWereDeleted(), "Some vregs were deleted from the save state");
909 #endif
910 }
911
912 if (auto callInst = saveState->GetCallerInst()) {
913 CreateStackMapRec(callInst->GetSaveState(), requireVregMap, targetSite);
914 auto method = GetGraph()->IsAotMode() ? nullptr : callInst->GetCallMethod();
915 codeBuilder_->BeginInlineInfo(method, GetRuntime()->GetMethodId(callInst->GetCallMethod()), saveState->GetPc(),
916 vregsCount);
917 }
918
919 if (requireVregMap) {
920 CreateVRegMap(saveState, vregsCount, targetSite);
921 }
922
923 if (hasInlineInfo) {
924 codeBuilder_->EndInlineInfo();
925 }
926 }
927
CreateVRegMap(SaveStateInst * saveState,size_t vregsCount,Inst * targetSite)928 void Codegen::CreateVRegMap(SaveStateInst *saveState, size_t vregsCount, Inst *targetSite)
929 {
930 vregIndices_.clear();
931 vregIndices_.resize(vregsCount, {-1, -1});
932 FillVregIndices(saveState);
933
934 ASSERT(GetGraph()->IsAnalysisValid<LivenessAnalyzer>());
935 auto &la = GetGraph()->GetAnalysis<LivenessAnalyzer>();
936 auto targetLifeNumber = la.GetInstLifeIntervals(targetSite)->GetBegin();
937
938 for (auto &inputIndex : vregIndices_) {
939 if (inputIndex.first == -1 && inputIndex.second == -1) {
940 codeBuilder_->AddVReg(VRegInfo());
941 continue;
942 }
943 if (inputIndex.second != -1) {
944 auto imm = saveState->GetImmediate(inputIndex.second);
945 codeBuilder_->AddConstant(imm.value, IrTypeToMetainfoType(imm.type), imm.vregType);
946 continue;
947 }
948 ASSERT(inputIndex.first != -1);
949 auto vreg = saveState->GetVirtualRegister(inputIndex.first);
950 auto inputInst = saveState->GetDataFlowInput(inputIndex.first);
951 auto interval = la.GetInstLifeIntervals(inputInst)->FindSiblingAt(targetLifeNumber);
952 ASSERT(interval != nullptr);
953 CreateVreg(interval->GetLocation(), inputInst, vreg);
954 }
955 }
956
CreateVreg(const Location & location,Inst * inst,const VirtualRegister & vreg)957 void Codegen::CreateVreg(const Location &location, Inst *inst, const VirtualRegister &vreg)
958 {
959 switch (location.GetKind()) {
960 case LocationType::FP_REGISTER:
961 case LocationType::REGISTER: {
962 CreateVRegForRegister(location, inst, vreg);
963 break;
964 }
965 case LocationType::STACK_PARAMETER: {
966 auto slot = location.GetValue();
967 codeBuilder_->AddVReg(VRegInfo(GetFrameLayout().GetStackArgsStartSlot() - slot - CFrameSlots::Start(),
968 VRegInfo::Location::SLOT, IrTypeToMetainfoType(inst->GetType()),
969 vreg.GetVRegType()));
970 break;
971 }
972 case LocationType::STACK: {
973 auto slot = location.GetValue();
974 if (!Is64BitsArch(GetArch())) {
975 slot = ((location.GetValue() << 1U) + 1);
976 }
977 codeBuilder_->AddVReg(VRegInfo(GetFrameLayout().GetFirstSpillSlot() + slot, VRegInfo::Location::SLOT,
978 IrTypeToMetainfoType(inst->GetType()), vreg.GetVRegType()));
979 break;
980 }
981 case LocationType::IMMEDIATE: {
982 ASSERT(inst->IsConst());
983 auto type = inst->GetType();
984 // There are no int64 type for dynamic methods.
985 if (GetGraph()->IsDynamicMethod() && type == DataType::INT64) {
986 type = DataType::INT32;
987 }
988 codeBuilder_->AddConstant(inst->CastToConstant()->GetRawValue(), IrTypeToMetainfoType(type),
989 vreg.GetVRegType());
990 break;
991 }
992 default:
993 // Reg-to-reg spill fill must not occurs within SaveState
994 UNREACHABLE();
995 }
996 }
997
FillVregIndices(SaveStateInst * saveState)998 void Codegen::FillVregIndices(SaveStateInst *saveState)
999 {
1000 for (size_t i = 0; i < saveState->GetInputsCount(); ++i) {
1001 size_t vregIndex = saveState->GetVirtualRegister(i).Value();
1002 if (vregIndex == VirtualRegister::BRIDGE) {
1003 continue;
1004 }
1005 ASSERT(vregIndex < vregIndices_.size());
1006 vregIndices_[vregIndex].first = i;
1007 }
1008 for (size_t i = 0; i < saveState->GetImmediatesCount(); i++) {
1009 auto vregImm = saveState->GetImmediate(i);
1010 if (vregImm.vreg == VirtualRegister::BRIDGE) {
1011 continue;
1012 }
1013 ASSERT(vregImm.vreg < vregIndices_.size());
1014 ASSERT(vregIndices_[vregImm.vreg].first == -1);
1015 vregIndices_[vregImm.vreg].second = i;
1016 }
1017 }
1018
CreateVRegForRegister(const Location & location,Inst * inst,const VirtualRegister & vreg)1019 void Codegen::CreateVRegForRegister(const Location &location, Inst *inst, const VirtualRegister &vreg)
1020 {
1021 bool isOsr = GetGraph()->IsOsrMode();
1022 bool isFp = (location.GetKind() == LocationType::FP_REGISTER);
1023 auto regNum = location.GetValue();
1024 auto reg = Reg(regNum, isFp ? FLOAT64_TYPE : INT64_TYPE);
1025 if (!isOsr && GetRegfile()->GetZeroReg() == reg) {
1026 codeBuilder_->AddConstant(0, IrTypeToMetainfoType(inst->GetType()), vreg.GetVRegType());
1027 } else if (isOsr || GetRegfile()->IsCalleeRegister(reg)) {
1028 if (isFp) {
1029 ASSERT(inst->GetType() != DataType::REFERENCE);
1030 ASSERT(isOsr || regNum >= GetFirstCalleeReg(GetArch(), true));
1031 codeBuilder_->AddVReg(VRegInfo(regNum, VRegInfo::Location::FP_REGISTER,
1032 IrTypeToMetainfoType(inst->GetType()), vreg.GetVRegType()));
1033 } else {
1034 ASSERT(isOsr || regNum >= GetFirstCalleeReg(GetArch(), false));
1035 codeBuilder_->AddVReg(VRegInfo(regNum, VRegInfo::Location::REGISTER, IrTypeToMetainfoType(inst->GetType()),
1036 vreg.GetVRegType()));
1037 }
1038 } else {
1039 ASSERT(regNum >= GetFirstCallerReg(GetArch(), isFp));
1040 auto lastSlot = GetFrameLayout().GetCallerLastSlot(isFp);
1041 RegMask mask(GetCallerRegsMask(GetArch(), isFp));
1042 regNum = mask.GetDistanceFromTail(regNum);
1043 codeBuilder_->AddVReg(VRegInfo(lastSlot - regNum, VRegInfo::Location::SLOT,
1044 IrTypeToMetainfoType(inst->GetType()), vreg.GetVRegType()));
1045 }
1046 }
1047
CreateOsrEntry(SaveStateInst * saveState)1048 void Codegen::CreateOsrEntry(SaveStateInst *saveState)
1049 {
1050 auto &stub = osrEntries_.emplace_back(GetAllocator()->New<OsrEntryStub>(this, saveState));
1051 GetEncoder()->BindLabel(stub->GetLabel());
1052 }
1053
CallIntrinsic(Inst * inst,RuntimeInterface::IntrinsicId id)1054 void Codegen::CallIntrinsic(Inst *inst, RuntimeInterface::IntrinsicId id)
1055 {
1056 SCOPED_DISASM_STR(this, "CallIntrinsic");
1057 // Call intrinsics isn't supported for IrToc, because we don't know address of intrinsics during IRToc compilation.
1058 // We store adresses of intrinsics in aot file in AOT mode.
1059 ASSERT(GetGraph()->SupportManagedCode());
1060 if (GetGraph()->IsAotMode()) {
1061 auto aotData = GetGraph()->GetAotData();
1062 intptr_t offset = aotData->GetEntrypointOffset(GetEncoder()->GetCursorOffset(), static_cast<int32_t>(id));
1063 GetEncoder()->MakeCallAot(offset);
1064 } else {
1065 GetEncoder()->MakeCall(reinterpret_cast<const void *>(GetRuntime()->GetIntrinsicAddress(
1066 inst->IsRuntimeCall(), GetRuntime()->GetMethodSourceLanguage(GetGraph()->GetMethod()), id)));
1067 }
1068 }
1069
EmitCallRuntimeCode(Inst * inst,std::variant<EntrypointId,Reg> entrypoint)1070 bool Codegen::EmitCallRuntimeCode(Inst *inst, std::variant<EntrypointId, Reg> entrypoint)
1071 {
1072 auto encoder = GetEncoder();
1073 if (std::holds_alternative<Reg>(entrypoint)) {
1074 auto reg = std::get<Reg>(entrypoint);
1075 ASSERT(reg.IsValid());
1076 encoder->MakeCall(reg);
1077 } else {
1078 auto id = std::get<EntrypointId>(entrypoint);
1079 MemRef entry(ThreadReg(), GetRuntime()->GetEntrypointTlsOffset(GetArch(), id));
1080 encoder->MakeCall(entry);
1081 }
1082
1083 SaveStateInst *saveState =
1084 (inst == nullptr || inst->IsSaveState()) ? static_cast<SaveStateInst *>(inst) : inst->GetSaveState();
1085 // StackMap should follow the call as the bridge function expects retaddr points to the stackmap
1086 if (saveState != nullptr) {
1087 CreateStackMap(inst);
1088 }
1089
1090 if (std::holds_alternative<EntrypointId>(entrypoint) &&
1091 GetRuntime()->IsEntrypointNoreturn(std::get<EntrypointId>(entrypoint))) {
1092 if constexpr (DEBUG_BUILD) { // NOLINT
1093 encoder->EncodeAbort();
1094 }
1095 return false;
1096 }
1097 ASSERT(saveState == nullptr || inst->IsRuntimeCall());
1098
1099 return true;
1100 }
1101
SaveRegistersForImplicitRuntime(Inst * inst,RegMask * paramsMask,RegMask * mask)1102 void Codegen::SaveRegistersForImplicitRuntime(Inst *inst, RegMask *paramsMask, RegMask *mask)
1103 {
1104 auto &rootsMask = inst->GetSaveState()->GetRootsRegsMask();
1105 for (auto i = 0U; i < inst->GetInputsCount(); i++) {
1106 auto input = inst->GetDataFlowInput(i);
1107 if (!input->IsMovableObject()) {
1108 continue;
1109 }
1110 auto location = inst->GetLocation(i);
1111 if (location.IsRegister() && location.IsRegisterValid()) {
1112 auto val = location.GetValue();
1113 auto reg = Reg(val, INT64_TYPE);
1114 GetEncoder()->SetRegister(mask, nullptr, reg, true);
1115 if (DataType::IsReference(inst->GetInputType(i)) && !rootsMask.test(val)) {
1116 paramsMask->set(val);
1117 rootsMask.set(val);
1118 }
1119 }
1120 }
1121 }
1122
CreateCheckForTLABWithConstSize(Inst * inst,Reg regTlabStart,Reg regTlabSize,size_t size,LabelHolder::LabelId label)1123 void Codegen::CreateCheckForTLABWithConstSize([[maybe_unused]] Inst *inst, Reg regTlabStart, Reg regTlabSize,
1124 size_t size, LabelHolder::LabelId label)
1125 {
1126 SCOPED_DISASM_STR(this, "CreateCheckForTLABWithConstSize");
1127 auto encoder = GetEncoder();
1128 if (encoder->CanEncodeImmAddSubCmp(size, WORD_SIZE, false)) {
1129 encoder->EncodeJump(label, regTlabSize, Imm(size), Condition::LO);
1130 // Change pointer to start free memory
1131 encoder->EncodeAdd(regTlabSize, regTlabStart, Imm(size));
1132 } else {
1133 ScopedTmpReg sizeReg(encoder);
1134 encoder->EncodeMov(sizeReg, Imm(size));
1135 encoder->EncodeJump(label, regTlabSize, sizeReg, Condition::LO);
1136 encoder->EncodeAdd(regTlabSize, regTlabStart, sizeReg);
1137 }
1138 }
1139
CreateDebugRuntimeCallsForNewObject(Inst * inst,Reg regTlabStart,size_t allocSize,RegMask preserved)1140 void Codegen::CreateDebugRuntimeCallsForNewObject(Inst *inst, [[maybe_unused]] Reg regTlabStart, size_t allocSize,
1141 RegMask preserved)
1142 {
1143 #if defined(PANDA_ASAN_ON) || defined(PANDA_TSAN_ON)
1144 CallRuntime(inst, EntrypointId::ANNOTATE_SANITIZERS, INVALID_REGISTER, preserved, regTlabStart,
1145 TypedImm(allocSize));
1146 #endif
1147 if (GetRuntime()->IsTrackTlabAlloc()) {
1148 CallRuntime(inst, EntrypointId::WRITE_TLAB_STATS, INVALID_REGISTER, preserved, regTlabStart,
1149 TypedImm(allocSize));
1150 }
1151 }
1152
CreateNewObjCall(NewObjectInst * newObj)1153 void Codegen::CreateNewObjCall(NewObjectInst *newObj)
1154 {
1155 auto dst = ConvertRegister(newObj->GetDstReg(), newObj->GetType());
1156 auto src = ConvertRegister(newObj->GetSrcReg(0), newObj->GetInputType(0));
1157 auto initClass = newObj->GetInput(0).GetInst();
1158 auto srcClass = ConvertRegister(newObj->GetSrcReg(0), DataType::POINTER);
1159 auto runtime = GetRuntime();
1160
1161 auto maxTlabSize = runtime->GetTLABMaxSize();
1162 if (maxTlabSize == 0 ||
1163 (initClass->GetOpcode() != Opcode::LoadAndInitClass && initClass->GetOpcode() != Opcode::LoadImmediate)) {
1164 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1165 return;
1166 }
1167 RuntimeInterface::ClassPtr klass;
1168 if (initClass->GetOpcode() == Opcode::LoadAndInitClass) {
1169 klass = initClass->CastToLoadAndInitClass()->GetClass();
1170 } else {
1171 ASSERT(initClass->GetOpcode() == Opcode::LoadImmediate);
1172 klass = initClass->CastToLoadImmediate()->GetClass();
1173 }
1174 if (klass == nullptr || !runtime->CanUseTlabForClass(klass)) {
1175 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1176 return;
1177 }
1178 auto classSize = runtime->GetClassSize(klass);
1179 auto alignment = runtime->GetTLABAlignment();
1180
1181 classSize = (classSize & ~(alignment - 1U)) + ((classSize % alignment) != 0U ? alignment : 0U);
1182 if (classSize > maxTlabSize) {
1183 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1184 return;
1185 }
1186 CallFastPath(newObj, EntrypointId::ALLOCATE_OBJECT_TLAB, dst, RegMask::GetZeroMask(), srcClass,
1187 TypedImm(classSize));
1188 }
1189
CreateNewObjCallOld(NewObjectInst * newObj)1190 void Codegen::CreateNewObjCallOld(NewObjectInst *newObj)
1191 {
1192 auto dst = ConvertRegister(newObj->GetDstReg(), newObj->GetType());
1193 auto src = ConvertRegister(newObj->GetSrcReg(0), newObj->GetInputType(0));
1194 auto initClass = newObj->GetInput(0).GetInst();
1195 auto runtime = GetRuntime();
1196 auto maxTlabSize = runtime->GetTLABMaxSize();
1197 auto encoder = GetEncoder();
1198
1199 if (maxTlabSize == 0 ||
1200 (initClass->GetOpcode() != Opcode::LoadAndInitClass && initClass->GetOpcode() != Opcode::LoadImmediate)) {
1201 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1202 return;
1203 }
1204 RuntimeInterface::ClassPtr klass;
1205 if (initClass->GetOpcode() == Opcode::LoadAndInitClass) {
1206 klass = initClass->CastToLoadAndInitClass()->GetClass();
1207 } else {
1208 ASSERT(initClass->GetOpcode() == Opcode::LoadImmediate);
1209 klass = initClass->CastToLoadImmediate()->GetClass();
1210 }
1211 if (klass == nullptr || !runtime->CanUseTlabForClass(klass)) {
1212 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1213 return;
1214 }
1215 auto classSize = runtime->GetClassSize(klass);
1216 auto alignment = runtime->GetTLABAlignment();
1217
1218 classSize = (classSize & ~(alignment - 1U)) + ((classSize % alignment) != 0U ? alignment : 0U);
1219 if (classSize > maxTlabSize) {
1220 CallRuntime(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
1221 return;
1222 }
1223 ScopedLiveTmpReg regTlabStart(encoder);
1224 ScopedLiveTmpReg regTlabSize(encoder);
1225
1226 auto slowPath = CreateSlowPath<SlowPathEntrypoint>(newObj, EntrypointId::CREATE_OBJECT_BY_CLASS);
1227 CreateLoadTLABInformation(regTlabStart, regTlabSize);
1228 CreateCheckForTLABWithConstSize(newObj, regTlabStart, regTlabSize, classSize, slowPath->GetLabel());
1229
1230 RegMask preservedRegs;
1231 encoder->SetRegister(&preservedRegs, nullptr, src);
1232 CreateDebugRuntimeCallsForNewObject(newObj, regTlabStart, reinterpret_cast<size_t>(classSize), preservedRegs);
1233
1234 ScopedTmpReg tlabReg(encoder);
1235 // Load pointer to tlab
1236 encoder->EncodeLdr(tlabReg, false, MemRef(ThreadReg(), runtime->GetCurrentTLABOffset(GetArch())));
1237
1238 // Store pointer to the class
1239 encoder->EncodeStr(src, MemRef(regTlabStart, runtime->GetObjClassOffset(GetArch())));
1240 encoder->EncodeMov(dst, regTlabStart);
1241 regTlabStart.Release();
1242 // Store new pointer to start free memory in TLAB
1243 encoder->EncodeStrRelease(regTlabSize, MemRef(tlabReg, runtime->GetTLABFreePointerOffset(GetArch())));
1244 slowPath->BindBackLabel(encoder);
1245 }
1246
LoadClassFromObject(Reg classReg,Reg objReg)1247 void Codegen::LoadClassFromObject(Reg classReg, Reg objReg)
1248 {
1249 Reg reg = ConvertRegister(classReg.GetId(), DataType::REFERENCE);
1250 GetEncoder()->EncodeLdr(reg, false, MemRef(objReg, GetRuntime()->GetObjClassOffset(GetArch())));
1251 }
1252
CreateMultiArrayCall(CallInst * callInst)1253 void Codegen::CreateMultiArrayCall(CallInst *callInst)
1254 {
1255 SCOPED_DISASM_STR(this, "Create Call for MultiArray");
1256
1257 auto dstReg = ConvertRegister(callInst->GetDstReg(), callInst->GetType());
1258 auto numArgs = callInst->GetInputsCount() - 2U; // first is class, last is save_state
1259
1260 ScopedTmpReg classReg(GetEncoder());
1261 auto classType = ConvertDataType(DataType::REFERENCE, GetArch());
1262 Reg classOrig = classReg.GetReg().As(classType);
1263 auto location = callInst->GetLocation(0);
1264 ASSERT(location.IsFixedRegister() && location.IsRegisterValid());
1265
1266 GetEncoder()->EncodeMov(classOrig, ConvertRegister(location.GetValue(), DataType::INT32));
1267 CallRuntime(callInst, EntrypointId::CREATE_MULTI_ARRAY, dstReg, RegMask::GetZeroMask(), classReg, TypedImm(numArgs),
1268 SpReg());
1269 if (callInst->GetFlag(inst_flags::MEM_BARRIER)) {
1270 GetEncoder()->EncodeMemoryBarrier(memory_order::RELEASE);
1271 }
1272 }
1273
CreateJumpToClassResolverPltShared(Inst * inst,Reg tmpReg,RuntimeInterface::EntrypointId id)1274 void Codegen::CreateJumpToClassResolverPltShared(Inst *inst, Reg tmpReg, RuntimeInterface::EntrypointId id)
1275 {
1276 auto encoder = GetEncoder();
1277 auto graph = GetGraph();
1278 auto aotData = graph->GetAotData();
1279 auto offset = aotData->GetSharedSlowPathOffset(id, encoder->GetCursorOffset());
1280 if (offset == 0 || !encoder->CanMakeCallByOffset(offset)) {
1281 SlowPathShared *slowPath;
1282 auto search = slowPathsMap_.find(id);
1283 if (search != slowPathsMap_.end()) {
1284 slowPath = search->second;
1285 ASSERT(slowPath->GetTmpReg().GetId() == tmpReg.GetId());
1286 } else {
1287 slowPath = CreateSlowPath<SlowPathShared>(inst, id);
1288 slowPath->SetTmpReg(tmpReg);
1289 slowPathsMap_[id] = slowPath;
1290 }
1291 encoder->MakeCall(slowPath->GetLabel());
1292 } else {
1293 encoder->MakeCallByOffset(offset);
1294 }
1295 CreateStackMap(inst);
1296 }
1297
CreateLoadClassFromPLT(Inst * inst,Reg tmpReg,Reg dst,size_t classId)1298 void Codegen::CreateLoadClassFromPLT(Inst *inst, Reg tmpReg, Reg dst, size_t classId)
1299 {
1300 auto encoder = GetEncoder();
1301 auto graph = GetGraph();
1302 auto aotData = graph->GetAotData();
1303 intptr_t offset = aotData->GetClassSlotOffset(encoder->GetCursorOffset(), classId, false);
1304 auto label = encoder->CreateLabel();
1305 ASSERT(tmpReg.GetId() != dst.GetId());
1306 ASSERT(inst->IsRuntimeCall());
1307 encoder->MakeLoadAotTableAddr(offset, tmpReg, dst);
1308 encoder->EncodeJump(label, dst, Condition::NE);
1309
1310 // PLT Class Resolver has special calling convention:
1311 // First encoder temporary (tmp_reg) works as parameter and return value
1312 CHECK_EQ(tmpReg.GetId(), encoder->GetTarget().GetTempRegsMask().GetMinRegister());
1313
1314 CreateJumpToClassResolverPltShared(inst, tmpReg, EntrypointId::CLASS_RESOLVER);
1315
1316 encoder->EncodeMov(dst, tmpReg);
1317 encoder->BindLabel(label);
1318 }
1319
CreateLoadTLABInformation(Reg regTlabStart,Reg regTlabSize)1320 void Codegen::CreateLoadTLABInformation(Reg regTlabStart, Reg regTlabSize)
1321 {
1322 SCOPED_DISASM_STR(this, "LoadTLABInformation");
1323 auto runtime = GetRuntime();
1324 // Load pointer to tlab
1325 GetEncoder()->EncodeLdr(regTlabSize, false, MemRef(ThreadReg(), runtime->GetCurrentTLABOffset(GetArch())));
1326 // Load pointer to start free memory in TLAB
1327 GetEncoder()->EncodeLdr(regTlabStart, false, MemRef(regTlabSize, runtime->GetTLABFreePointerOffset(GetArch())));
1328 // Load pointer to end free memory in TLAB
1329 GetEncoder()->EncodeLdr(regTlabSize, false, MemRef(regTlabSize, runtime->GetTLABEndPointerOffset(GetArch())));
1330 // Calculate size of the free memory
1331 GetEncoder()->EncodeSub(regTlabSize, regTlabSize, regTlabStart);
1332 }
1333
1334 // The function alignment up the value from alignment_reg using tmp_reg.
CreateAlignmentValue(Reg alignmentReg,Reg tmpReg,size_t alignment)1335 void Codegen::CreateAlignmentValue(Reg alignmentReg, Reg tmpReg, size_t alignment)
1336 {
1337 auto andVal = ~(alignment - 1U);
1338 // zeroed lower bits
1339 GetEncoder()->EncodeAnd(tmpReg, alignmentReg, Imm(andVal));
1340 GetEncoder()->EncodeSub(alignmentReg, alignmentReg, tmpReg);
1341
1342 auto endLabel = GetEncoder()->CreateLabel();
1343
1344 // if zeroed value is different, add alignment
1345 GetEncoder()->EncodeJump(endLabel, alignmentReg, Condition::EQ);
1346 GetEncoder()->EncodeAdd(tmpReg, tmpReg, Imm(alignment));
1347
1348 GetEncoder()->BindLabel(endLabel);
1349 GetEncoder()->EncodeMov(alignmentReg, tmpReg);
1350 }
1351
GetObjectReg(Codegen * codegen,Inst * inst)1352 Reg GetObjectReg(Codegen *codegen, Inst *inst)
1353 {
1354 ASSERT(inst->IsCall() || inst->GetOpcode() == Opcode::ResolveVirtual);
1355 auto location = inst->GetLocation(0);
1356 ASSERT(location.IsFixedRegister() && location.IsRegisterValid());
1357 auto objReg = codegen->ConvertRegister(location.GetValue(), inst->GetInputType(0));
1358 ASSERT(objReg != INVALID_REGISTER);
1359 return objReg;
1360 }
1361
CreateCallIntrinsic(IntrinsicInst * inst)1362 void Codegen::CreateCallIntrinsic(IntrinsicInst *inst)
1363 {
1364 if (inst->HasArgumentsOnStack()) {
1365 // Since for some intrinsics(f.e. TimedWaitNanos) we need SaveState instruction and
1366 // more than two arguments, we need to place arguments on the stack, but in same time we need to
1367 // create boundary frame
1368 LOG(WARNING, COMPILER) << "Intrinsics with arguments on stack are not supported";
1369 GetEncoder()->SetFalseResult();
1370 return;
1371 }
1372
1373 ASSERT(!HasLiveCallerSavedRegs(inst));
1374 if (inst->HasImms() && GetGraph()->SupportManagedCode()) {
1375 EncodeImms(inst->GetImms(), (inst->HasIdInput() || inst->IsMethodFirstInput()));
1376 }
1377
1378 size_t explicitArgs;
1379 if (IsStackRangeIntrinsic(inst->GetIntrinsicId(), &explicitArgs)) {
1380 auto paramInfo = GetCallingConvention()->GetParameterInfo(explicitArgs);
1381 auto rangePtrReg =
1382 ConvertRegister(paramInfo->GetNextLocation(DataType::POINTER).GetRegister(), DataType::POINTER);
1383 if (inst->GetInputsCount() > explicitArgs + (inst->RequireState() ? 1U : 0U)) {
1384 auto rangeSpOffs = GetStackOffset(inst->GetLocation(explicitArgs));
1385 GetEncoder()->EncodeAdd(rangePtrReg, GetTarget().GetStackReg(), Imm(rangeSpOffs));
1386 }
1387 }
1388 if (inst->IsMethodFirstInput()) {
1389 Reg param0 = GetTarget().GetParamReg(0);
1390 if (GetGraph()->IsAotMode()) {
1391 LoadMethod(param0);
1392 } else {
1393 GetEncoder()->EncodeMov(param0, Imm(reinterpret_cast<uintptr_t>(inst->GetMethod())));
1394 }
1395 }
1396 CallIntrinsic(inst, inst->GetIntrinsicId());
1397
1398 if (inst->GetSaveState() != nullptr) {
1399 // StackMap should follow the call as the bridge function expects retaddr points to the stackmap
1400 CreateStackMap(inst);
1401 }
1402
1403 if (inst->GetType() != DataType::VOID) {
1404 auto arch = GetArch();
1405 auto returnType = inst->GetType();
1406 auto dstReg = ConvertRegister(inst->GetDstReg(), inst->GetType());
1407 auto returnReg = GetTarget().GetReturnReg(dstReg.GetType());
1408 // We must:
1409 // sign extended INT8 and INT16 to INT32
1410 // zero extended UINT8 and UINT16 to UINT32
1411 if (DataType::ShiftByType(returnType, arch) < DataType::ShiftByType(DataType::INT32, arch)) {
1412 bool isSigned = DataType::IsTypeSigned(returnType);
1413 GetEncoder()->EncodeCast(dstReg, isSigned, Reg(returnReg.GetId(), INT32_TYPE), isSigned);
1414 } else {
1415 GetEncoder()->EncodeMov(dstReg, returnReg);
1416 }
1417 }
1418 }
1419
IntfInlineCachePass(ResolveVirtualInst * resolver,Reg methodReg,Reg tmpReg,Reg objReg)1420 void Codegen::IntfInlineCachePass(ResolveVirtualInst *resolver, Reg methodReg, Reg tmpReg, Reg objReg)
1421 {
1422 // Cache structure:（offset addr)/(class addr) 32bit/32bit
1423 // -----------------------------------------------
1424 // (.aot_got)
1425 // ...
1426 // cache:offset/class <----------|
1427 // ... |
1428 // (.text) |
1429 // interface call start |
1430 // call runtime irtoc function
1431 // if call class == cache.class <------|
1432 // use cache.offset(method) <------|
1433 // else |
1434 // call RESOLVE_VIRTUAL_CALL_AOT |
1435 // save method‘s offset to cache >--|
1436 // return to (.text)
1437 // call method
1438 // -----------------------------------------------
1439 auto aotData = GetGraph()->GetAotData();
1440 uint64_t intfInlineCacheIndex = aotData->GetIntfInlineCacheIndex();
1441 // NOTE(liyiming): do LoadMethod in irtoc to reduce use tmp reg
1442 if (objReg.GetId() != tmpReg.GetId()) {
1443 auto regTmp64 = tmpReg.As(INT64_TYPE);
1444 uint64_t offset = aotData->GetInfInlineCacheSlotOffset(GetEncoder()->GetCursorOffset(), intfInlineCacheIndex);
1445 GetEncoder()->MakeLoadAotTableAddr(offset, regTmp64, INVALID_REGISTER);
1446 LoadMethod(methodReg);
1447 CallFastPath(resolver, EntrypointId::INTF_INLINE_CACHE, methodReg, {}, methodReg, objReg,
1448 TypedImm(resolver->GetCallMethodId()), regTmp64);
1449 } else {
1450 // we don't have tmp reg here, so use x3 directly
1451 Reg reg3 = Reg(3U, INT64_TYPE);
1452 ScopedTmpRegF64 vtmp(GetEncoder());
1453 GetEncoder()->EncodeMov(vtmp, reg3);
1454 uint64_t offset = aotData->GetInfInlineCacheSlotOffset(GetEncoder()->GetCursorOffset(), intfInlineCacheIndex);
1455 GetEncoder()->MakeLoadAotTableAddr(offset, reg3, INVALID_REGISTER);
1456 LoadMethod(methodReg);
1457 CallFastPath(resolver, EntrypointId::INTF_INLINE_CACHE, methodReg, {}, methodReg, objReg,
1458 TypedImm(resolver->GetCallMethodId()), reg3);
1459 GetEncoder()->EncodeMov(reg3, vtmp);
1460 }
1461
1462 intfInlineCacheIndex++;
1463 aotData->SetIntfInlineCacheIndex(intfInlineCacheIndex);
1464 }
1465
1466 /**
1467 * Returns a pointer to a caller of a virtual/static resolver.
1468 *
1469 * InstBuilder uses UnresolvedTypesInterface::AddTableSlot(method_ptr, ...) to allocate cache slots
1470 * for unresolved methods. The codegen uses UnresolvedTypesInterface::GetTableSlot(method_ptr, ...)
1471 * to get corresponding slots for unresolved methods.
1472 *
1473 * Since there is no valid method_ptr for an unresolved method at the point of IR construction
1474 * InstBuilder::BuildCallInst uses the pointer to the caller (GetGraph()->GetMethod()) as an argument
1475 * of UnresolvedTypesInterface::AddTableSlot. So the codegen has to deduce the pointer to the caller
1476 * and pass it to UnresolvedTypesInterface::GetTableSlot. If the caller was not inlined then
1477 * GetGraph()->GetMethod() works just fine, otherwise it is done by means of SaveState,
1478 * which is capable of remebering the corresponding caller instruction.
1479 *
1480 * To clarify the details let's consider the following sequence of calls:
1481 *
1482 * main() -> foo() -> bar()
1483 *
1484 * Suppose that foo() is a static method and bar() is an unresolved virtual method.
1485 *
1486 * 1) foo() is going to be inlined
1487 *
1488 * To inline foo() the inlining pass creates its IR by means of IRBuilder(..., caller_inst),
1489 * where caller_inst (the third argument) represents "CallStatic foo" instruction.
1490 * The caller_inst gets remembered in the corresponding SaveState created for bar's resolver
1491 * (see InstBuilder::CreateSaveState for the details).
1492 * At the same time "CallStatic foo" instruction contains a valid method pointer to foo()
1493 * (it can not be nullptr because we do not inline Unresolved).
1494 *
1495 * So we go through the following chain: bar's resolver->SaveState->caller_inst->pointer to the method.
1496 *
1497 * 2) foo() is not inlined
1498 *
1499 * In this case SaveState.caller_inst_ == nullptr (because foo() is not inlined).
1500 * Thus we just use plain GetGraph()->GetMethod().
1501 */
1502 template <typename T>
GetCallerOfUnresolvedMethod(T * resolver)1503 RuntimeInterface::MethodPtr Codegen::GetCallerOfUnresolvedMethod(T *resolver)
1504 {
1505 ASSERT(resolver->GetCallMethod() == nullptr);
1506 auto saveState = resolver->GetSaveState();
1507 ASSERT(saveState != nullptr);
1508 auto caller = saveState->GetCallerInst();
1509 auto method = (caller == nullptr ? GetGraph()->GetMethod() : caller->GetCallMethod());
1510 ASSERT(method != nullptr);
1511 return method;
1512 }
1513
EmitResolveUnknownVirtual(ResolveVirtualInst * resolver,Reg methodReg)1514 void Codegen::EmitResolveUnknownVirtual(ResolveVirtualInst *resolver, Reg methodReg)
1515 {
1516 SCOPED_DISASM_STR(this, "Create runtime call to resolve an unknown virtual method");
1517 ASSERT(resolver->GetOpcode() == Opcode::ResolveVirtual);
1518 auto method = GetCallerOfUnresolvedMethod(resolver);
1519 ScopedTmpReg tmpReg(GetEncoder(), ConvertDataType(DataType::REFERENCE, GetArch()));
1520 Reg objReg = GetObjectReg(this, resolver);
1521 if (GetGraph()->IsAotMode()) {
1522 LoadMethod(methodReg);
1523 CallRuntime(resolver, EntrypointId::RESOLVE_VIRTUAL_CALL_AOT, methodReg, {}, methodReg, objReg,
1524 TypedImm(resolver->GetCallMethodId()), TypedImm(0));
1525 } else {
1526 auto runtime = GetRuntime();
1527 auto utypes = runtime->GetUnresolvedTypes();
1528 auto skind = UnresolvedTypesInterface::SlotKind::VIRTUAL_METHOD;
1529 // Try to load vtable index
1530 auto methodSlotAddr = utypes->GetTableSlot(method, resolver->GetCallMethodId(), skind);
1531 GetEncoder()->EncodeMov(methodReg, Imm(methodSlotAddr));
1532 GetEncoder()->EncodeLdr(methodReg, false, MemRef(methodReg));
1533 // 0 means the virtual call is uninitialized or it is an interface call
1534 auto entrypoint = EntrypointId::RESOLVE_UNKNOWN_VIRTUAL_CALL;
1535 auto slowPath = CreateSlowPath<SlowPathUnresolved>(resolver, entrypoint);
1536 slowPath->SetUnresolvedType(method, resolver->GetCallMethodId());
1537 slowPath->SetDstReg(methodReg);
1538 slowPath->SetArgReg(objReg);
1539 slowPath->SetSlotAddr(methodSlotAddr);
1540 GetEncoder()->EncodeJump(slowPath->GetLabel(), methodReg, Condition::EQ);
1541 // Load klass into tmp_reg
1542 LoadClassFromObject(tmpReg, objReg);
1543 // Load from VTable, address = (klass + (index << shift)) + vtable_offset
1544 auto tmpReg64 = Reg(tmpReg.GetReg().GetId(), INT64_TYPE);
1545 GetEncoder()->EncodeAdd(tmpReg64, tmpReg, Shift(methodReg, GetVtableShift()));
1546 GetEncoder()->EncodeLdr(methodReg, false,
1547 MemRef(tmpReg64, runtime->GetVTableOffset(GetArch()) - (1U << GetVtableShift())));
1548 slowPath->BindBackLabel(GetEncoder());
1549 }
1550 }
1551
EmitResolveVirtualAot(ResolveVirtualInst * resolver,Reg methodReg)1552 void Codegen::EmitResolveVirtualAot(ResolveVirtualInst *resolver, Reg methodReg)
1553 {
1554 SCOPED_DISASM_STR(this, "AOT ResolveVirtual using PLT-GOT");
1555 ASSERT(resolver->IsRuntimeCall());
1556 ScopedTmpReg tmpReg(GetEncoder(), ConvertDataType(DataType::REFERENCE, GetArch()));
1557 auto methodReg64 = Reg(methodReg.GetId(), INT64_TYPE);
1558 auto tmpReg64 = Reg(tmpReg.GetReg().GetId(), INT64_TYPE);
1559 auto aotData = GetGraph()->GetAotData();
1560 intptr_t offset = aotData->GetVirtIndexSlotOffset(GetEncoder()->GetCursorOffset(), resolver->GetCallMethodId());
1561 GetEncoder()->MakeLoadAotTableAddr(offset, tmpReg64, methodReg64);
1562 auto label = GetEncoder()->CreateLabel();
1563 GetEncoder()->EncodeJump(label, methodReg, Condition::NE);
1564 GetEncoder()->EncodeMov(methodReg64, tmpReg64);
1565 // PLT CallVirtual Resolver has a very special calling convention:
1566 // First encoder temporary (method_reg) works as a parameter and return value
1567 CHECK_EQ(methodReg64.GetId(), GetTarget().GetTempRegsMask().GetMinRegister());
1568 MemRef entry(ThreadReg(), GetRuntime()->GetEntrypointTlsOffset(GetArch(), EntrypointId::CALL_VIRTUAL_RESOLVER));
1569 GetEncoder()->MakeCall(entry);
1570 // Need a stackmap to build correct boundary frame
1571 CreateStackMap(resolver);
1572 GetEncoder()->BindLabel(label);
1573 // Load class into method_reg
1574 Reg objReg = GetObjectReg(this, resolver);
1575 LoadClassFromObject(tmpReg, objReg);
1576 // Load from VTable, address = (klass + (index << shift)) + vtable_offset
1577 GetEncoder()->EncodeAdd(methodReg, tmpReg64, Shift(methodReg, GetVtableShift()));
1578 GetEncoder()->EncodeLdr(methodReg, false,
1579 MemRef(methodReg, GetRuntime()->GetVTableOffset(GetArch()) - (1U << GetVtableShift())));
1580 }
1581
EmitResolveVirtual(ResolveVirtualInst * resolver)1582 void Codegen::EmitResolveVirtual(ResolveVirtualInst *resolver)
1583 {
1584 auto methodReg = ConvertRegister(resolver->GetDstReg(), resolver->GetType());
1585 auto objectReg = ConvertRegister(resolver->GetSrcReg(0), DataType::REFERENCE);
1586 ScopedTmpReg tmpMethodReg(GetEncoder());
1587 if (resolver->GetCallMethod() == nullptr) {
1588 EmitResolveUnknownVirtual(resolver, tmpMethodReg);
1589 GetEncoder()->EncodeMov(methodReg, tmpMethodReg);
1590 } else if (GetRuntime()->IsInterfaceMethod(resolver->GetCallMethod())) {
1591 SCOPED_DISASM_STR(this, "Create runtime call to resolve a known virtual call");
1592 if (GetGraph()->IsAotMode()) {
1593 if (GetArch() == Arch::AARCH64) {
1594 ScopedTmpReg tmpReg(GetEncoder(), ConvertDataType(DataType::REFERENCE, GetArch()));
1595 IntfInlineCachePass(resolver, tmpMethodReg, tmpReg, objectReg);
1596 } else {
1597 LoadMethod(tmpMethodReg);
1598 CallRuntime(resolver, EntrypointId::RESOLVE_VIRTUAL_CALL_AOT, tmpMethodReg, {}, tmpMethodReg, objectReg,
1599 TypedImm(resolver->GetCallMethodId()), TypedImm(0));
1600 }
1601 } else {
1602 CallRuntime(resolver, EntrypointId::RESOLVE_VIRTUAL_CALL, tmpMethodReg, {},
1603 TypedImm(reinterpret_cast<size_t>(resolver->GetCallMethod())), objectReg);
1604 }
1605 GetEncoder()->EncodeMov(methodReg, tmpMethodReg);
1606 } else if (GetGraph()->IsAotNoChaMode()) {
1607 // ResolveVirtualAot requires method_reg to be the first tmp register.
1608 EmitResolveVirtualAot(resolver, tmpMethodReg);
1609 GetEncoder()->EncodeMov(methodReg, tmpMethodReg);
1610 } else {
1611 UNREACHABLE();
1612 }
1613 }
1614
EmitCallResolvedVirtual(CallInst * call)1615 void Codegen::EmitCallResolvedVirtual(CallInst *call)
1616 {
1617 SCOPED_DISASM_STR(this, "Create a call to resolved virtual method");
1618 ASSERT(call->GetOpcode() == Opcode::CallResolvedVirtual);
1619 if (call->GetSaveState() != nullptr && call->IsInlined()) {
1620 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
1621 if (!GetGraph()->IsAotMode()) {
1622 InsertTrace({Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
1623 Imm(reinterpret_cast<size_t>(call->GetCallMethod())),
1624 Imm(static_cast<size_t>(events::MethodEnterKind::INLINED))});
1625 }
1626 #endif
1627 return;
1628 }
1629 ASSERT(!HasLiveCallerSavedRegs(call));
1630 ASSERT(call->GetCallMethod() == nullptr || GetGraph()->GetRuntime()->IsInterfaceMethod(call->GetCallMethod()) ||
1631 GetGraph()->IsAotNoChaMode());
1632 Reg methodReg = ConvertRegister(call->GetSrcReg(0), DataType::POINTER);
1633 Reg param0 = GetTarget().GetParamReg(0);
1634 // Set method
1635 GetEncoder()->EncodeMov(param0, methodReg);
1636 GetEncoder()->MakeCall(MemRef(param0, GetRuntime()->GetCompiledEntryPointOffset(GetArch())));
1637 FinalizeCall(call);
1638 }
1639
EmitCallVirtual(CallInst * call)1640 void Codegen::EmitCallVirtual(CallInst *call)
1641 {
1642 SCOPED_DISASM_STR(this, "Create a call to virtual method");
1643 ASSERT(call->GetOpcode() == Opcode::CallVirtual);
1644 if (call->GetSaveState() != nullptr && call->IsInlined()) {
1645 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
1646 if (!GetGraph()->IsAotMode()) {
1647 InsertTrace({Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
1648 Imm(reinterpret_cast<size_t>(call->GetCallMethod())),
1649 Imm(static_cast<size_t>(events::MethodEnterKind::INLINED))});
1650 }
1651 #endif
1652 return;
1653 }
1654 auto runtime = GetRuntime();
1655 auto method = call->GetCallMethod();
1656 ASSERT(!HasLiveCallerSavedRegs(call));
1657 ASSERT(!call->IsUnresolved() && !runtime->IsInterfaceMethod(method) && !GetGraph()->IsAotNoChaMode());
1658 Reg methodReg = GetTarget().GetParamReg(0);
1659 ASSERT(!RegisterKeepCallArgument(call, methodReg));
1660 LoadClassFromObject(methodReg, GetObjectReg(this, call));
1661 // Get index
1662 auto vtableIndex = runtime->GetVTableIndex(method);
1663 // Load from VTable, address = klass + ((index << shift) + vtable_offset)
1664 auto totalOffset = runtime->GetVTableOffset(GetArch()) + (vtableIndex << GetVtableShift());
1665 // Class ref was loaded to method_reg
1666 GetEncoder()->EncodeLdr(methodReg, false, MemRef(methodReg, totalOffset));
1667 // Set method
1668 GetEncoder()->MakeCall(MemRef(methodReg, runtime->GetCompiledEntryPointOffset(GetArch())));
1669 FinalizeCall(call);
1670 }
1671
EnsureParamsFitIn32Bit(std::initializer_list<std::variant<Reg,TypedImm>> params)1672 bool Codegen::EnsureParamsFitIn32Bit(std::initializer_list<std::variant<Reg, TypedImm>> params)
1673 {
1674 for (auto ¶m : params) {
1675 if (std::holds_alternative<Reg>(param)) {
1676 auto reg = std::get<Reg>(param);
1677 if (reg.GetSize() > WORD_SIZE) {
1678 return false;
1679 }
1680 } else {
1681 auto immType = std::get<TypedImm>(param).GetType();
1682 if (immType.GetSize() > WORD_SIZE) {
1683 return false;
1684 }
1685 }
1686 }
1687 return true;
1688 }
1689
EmitResolveStatic(ResolveStaticInst * resolver)1690 void Codegen::EmitResolveStatic(ResolveStaticInst *resolver)
1691 {
1692 auto methodReg = ConvertRegister(resolver->GetDstReg(), resolver->GetType());
1693 if (GetGraph()->IsAotMode()) {
1694 LoadMethod(methodReg);
1695 CallRuntime(resolver, EntrypointId::GET_UNKNOWN_CALLEE_METHOD, methodReg, RegMask::GetZeroMask(), methodReg,
1696 TypedImm(resolver->GetCallMethodId()), TypedImm(0));
1697 return;
1698 }
1699 auto method = GetCallerOfUnresolvedMethod(resolver);
1700 ScopedTmpReg tmp(GetEncoder());
1701 auto utypes = GetRuntime()->GetUnresolvedTypes();
1702 auto skind = UnresolvedTypesInterface::SlotKind::METHOD;
1703 auto methodAddr = utypes->GetTableSlot(method, resolver->GetCallMethodId(), skind);
1704 GetEncoder()->EncodeMov(tmp.GetReg(), Imm(methodAddr));
1705 GetEncoder()->EncodeLdr(tmp.GetReg(), false, MemRef(tmp.GetReg()));
1706 auto slowPath = CreateSlowPath<SlowPathUnresolved>(resolver, EntrypointId::GET_UNKNOWN_CALLEE_METHOD);
1707 slowPath->SetUnresolvedType(method, resolver->GetCallMethodId());
1708 slowPath->SetDstReg(tmp.GetReg());
1709 slowPath->SetSlotAddr(methodAddr);
1710 GetEncoder()->EncodeJump(slowPath->GetLabel(), tmp.GetReg(), Condition::EQ);
1711 slowPath->BindBackLabel(GetEncoder());
1712 GetEncoder()->EncodeMov(methodReg, tmp.GetReg());
1713 }
1714
EmitCallResolvedStatic(CallInst * call)1715 void Codegen::EmitCallResolvedStatic(CallInst *call)
1716 {
1717 ASSERT(call->GetOpcode() == Opcode::CallResolvedStatic && call->GetCallMethod() == nullptr);
1718 Reg methodReg = ConvertRegister(call->GetSrcReg(0), DataType::POINTER);
1719 Reg param0 = GetTarget().GetParamReg(0);
1720 GetEncoder()->EncodeMov(param0, methodReg);
1721 GetEncoder()->MakeCall(MemRef(param0, GetRuntime()->GetCompiledEntryPointOffset(GetArch())));
1722 FinalizeCall(call);
1723 }
1724
EmitCallStatic(CallInst * call)1725 void Codegen::EmitCallStatic(CallInst *call)
1726 {
1727 ASSERT(call->IsStaticCall() && !call->IsUnresolved());
1728 if (call->GetSaveState() != nullptr && call->IsInlined()) {
1729 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
1730 if (!GetGraph()->IsAotMode()) {
1731 InsertTrace({Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
1732 Imm(reinterpret_cast<size_t>(call->GetCallMethod())),
1733 Imm(static_cast<size_t>(events::MethodEnterKind::INLINED))});
1734 }
1735 #endif
1736 return;
1737 }
1738 SCOPED_DISASM_STR(this, "Create a call to static");
1739 ASSERT(!HasLiveCallerSavedRegs(call));
1740 // Now MakeCallByOffset is not supported in Arch32Encoder (support ADR instruction)
1741 Reg param0 = GetTarget().GetParamReg(0);
1742 if (call->GetCallMethod() == GetGraph()->GetMethod() && GetArch() != Arch::AARCH32 && !GetGraph()->IsOsrMode() &&
1743 !GetGraph()->GetMethodProperties().GetHasDeopt()) {
1744 if (GetGraph()->IsAotMode()) {
1745 LoadMethod(param0);
1746 } else {
1747 GetEncoder()->EncodeMov(param0, Imm(reinterpret_cast<size_t>(GetGraph()->GetMethod())));
1748 }
1749 GetEncoder()->MakeCallByOffset(GetStartCodeOffset() - GetEncoder()->GetCursorOffset());
1750 } else {
1751 if (GetGraph()->IsAotMode()) {
1752 auto aotData = GetGraph()->GetAotData();
1753 intptr_t offset = aotData->GetPltSlotOffset(GetEncoder()->GetCursorOffset(), call->GetCallMethodId());
1754 // PLT CallStatic Resolver transparently uses param_0 (Method) register
1755 GetEncoder()->MakeLoadAotTable(offset, param0);
1756 } else { // usual JIT case
1757 auto method = call->GetCallMethod();
1758 GetEncoder()->EncodeMov(param0, Imm(reinterpret_cast<size_t>(method)));
1759 }
1760 GetEncoder()->MakeCall(MemRef(param0, GetRuntime()->GetCompiledEntryPointOffset(GetArch())));
1761 }
1762 FinalizeCall(call);
1763 }
1764
EmitCallDynamic(CallInst * call)1765 void Codegen::EmitCallDynamic(CallInst *call)
1766 {
1767 SCOPED_DISASM_STR(this, "Create a dynamic call");
1768 if (call->GetSaveState() != nullptr && call->IsInlined()) {
1769 #if defined(EVENT_METHOD_ENTER_ENABLED) && EVENT_METHOD_ENTER_ENABLED != 0
1770 if (!GetGraph()->IsAotMode()) {
1771 InsertTrace(Imm(static_cast<size_t>(TraceId::METHOD_ENTER)),
1772 Imm(reinterpret_cast<size_t>(call->GetCallMethod())),
1773 Imm(static_cast<size_t>(events::MethodEnterKind::INLINED)));
1774 }
1775 #endif
1776 return;
1777 }
1778 RuntimeInterface *runtime = GetRuntime();
1779 Encoder *encoder = GetEncoder();
1780
1781 auto dstReg = ConvertRegister(call->GetDstReg(), call->GetType());
1782 Reg methodParamReg = GetTarget().GetParamReg(CallConvDynInfo::REG_METHOD).As(GetPtrRegType());
1783 Reg numArgsParamReg = GetTarget().GetParamReg(CallConvDynInfo::REG_NUM_ARGS);
1784 auto paramFuncObjLoc = Location::MakeStackArgument(CallConvDynInfo::SLOT_CALLEE);
1785
1786 ASSERT(!HasLiveCallerSavedRegs(call));
1787
1788 // Load method from callee object
1789 static_assert(coretypes::TaggedValue::TAG_OBJECT == 0);
1790 encoder->EncodeLdr(methodParamReg, false, MemRef(SpReg(), GetStackOffset(paramFuncObjLoc)));
1791 encoder->EncodeLdr(methodParamReg, false, MemRef(methodParamReg, runtime->GetFunctionTargetOffset(GetArch())));
1792
1793 ASSERT(call->GetInputsCount() > 1);
1794 auto numArgs = static_cast<uint32_t>(call->GetInputsCount() - 1); // '-1' means 1 for spill fill input
1795 encoder->EncodeMov(numArgsParamReg, Imm(numArgs));
1796
1797 size_t entryPointOffset = runtime->GetCompiledEntryPointOffset(GetArch());
1798 encoder->MakeCall(MemRef(methodParamReg, entryPointOffset));
1799
1800 CreateStackMap(call);
1801 // Dynamic callee may have moved sp if there was insufficient num_actual_args
1802 encoder->EncodeSub(
1803 SpReg(), FpReg(),
1804 Imm(GetFrameLayout().GetOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::BYTES>(0)));
1805
1806 if (dstReg.IsValid()) {
1807 Reg retReg = GetTarget().GetReturnReg(dstReg.GetType());
1808 encoder->EncodeMov(dstReg, retReg);
1809 }
1810 }
1811
FinalizeCall(CallInst * call)1812 void Codegen::FinalizeCall(CallInst *call)
1813 {
1814 ASSERT(!call->IsDynamicCall());
1815 CreateStackMap(call);
1816 auto returnType = call->GetType();
1817 auto dstReg = ConvertRegister(call->GetDstReg(), returnType);
1818 // Restore frame pointer in the TLS
1819 GetEncoder()->EncodeStr(FpReg(), MemRef(ThreadReg(), GetRuntime()->GetTlsFrameOffset(GetArch())));
1820 // Sign/Zero extend return_reg if necessary
1821 if (dstReg.IsValid()) {
1822 auto arch = GetArch();
1823 auto returnReg = GetTarget().GetReturnReg(dstReg.GetType());
1824 // INT8 and INT16 must be sign extended to INT32
1825 // UINT8 and UINT16 must be zero extended to UINT32
1826 if (DataType::ShiftByType(returnType, arch) < DataType::ShiftByType(DataType::INT32, arch)) {
1827 bool isSigned = DataType::IsTypeSigned(returnType);
1828 GetEncoder()->EncodeCast(dstReg, isSigned, Reg(returnReg.GetId(), INT32_TYPE), isSigned);
1829 } else {
1830 GetEncoder()->EncodeMov(dstReg, returnReg);
1831 }
1832 }
1833 }
1834
1835 template <bool IS_CLASS>
CreatePreWRB(Inst * inst,MemRef mem,RegMask preserved,bool storePair)1836 void Codegen::CreatePreWRB(Inst *inst, MemRef mem, RegMask preserved, bool storePair)
1837 {
1838 auto runtime = GetRuntime();
1839 auto *enc = GetEncoder();
1840
1841 SCOPED_DISASM_STR(this, "Pre WRB");
1842 ScopedTmpReg entrypointReg(enc, enc->IsLrAsTempRegEnabledAndReleased());
1843 GetEncoder()->EncodeLdr(entrypointReg, false,
1844 MemRef(ThreadReg(), runtime->GetTlsPreWrbEntrypointOffset(GetArch())));
1845
1846 // Check entrypoint address
1847 auto label = GetEncoder()->CreateLabel();
1848 enc->EncodeJump(label, entrypointReg, Condition::EQ);
1849 auto refType =
1850 inst->GetType() == DataType::REFERENCE ? DataType::GetIntTypeForReference(enc->GetArch()) : DataType::INT64;
1851 ScopedTmpReg tmpRef(enc, ConvertDataType(refType, GetArch()));
1852 auto prevOffset = enc->GetCursorOffset();
1853 // Load old value
1854 if (IsVolatileMemInst(inst)) {
1855 enc->EncodeLdrAcquire(tmpRef, false, mem);
1856 } else {
1857 enc->EncodeLdr(tmpRef, false, mem);
1858 }
1859 TryInsertImplicitNullCheck(inst, prevOffset);
1860 if constexpr (IS_CLASS) {
1861 enc->EncodeLdr(tmpRef, false, MemRef(tmpRef, runtime->GetManagedClassOffset(GetArch())));
1862 } else {
1863 CheckObject(tmpRef, label);
1864 }
1865 auto [live_regs, live_vregs] = GetLiveRegisters<true>(inst);
1866 live_regs |= preserved;
1867 CallBarrier(live_regs, live_vregs, entrypointReg.GetReg(), tmpRef);
1868
1869 if (storePair) {
1870 // store pair doesn't support index and scalar
1871 ASSERT(!mem.HasIndex() && !mem.HasScale());
1872 // calculate offset for second store
1873 // NOLINTNEXTLINE(bugprone-misplaced-widening-cast)
1874 auto secondOffset = static_cast<ssize_t>(1U << DataType::ShiftByType(DataType::REFERENCE, enc->GetArch()));
1875 if (mem.HasDisp()) {
1876 secondOffset += mem.GetDisp();
1877 }
1878 // Load old value
1879 if (IsVolatileMemInst(inst)) {
1880 enc->EncodeLdrAcquire(tmpRef, false, MemRef(mem.GetBase(), secondOffset));
1881 } else {
1882 enc->EncodeLdr(tmpRef, false, MemRef(mem.GetBase(), secondOffset));
1883 }
1884 CheckObject(tmpRef, label);
1885 CallBarrier(live_regs, live_vregs, entrypointReg.GetReg(), tmpRef);
1886 }
1887 enc->BindLabel(label);
1888 }
1889
1890 // Force template instantiation to avoid "undefined reference" error from ld.
1891 template void Codegen::CreatePreWRB<true>(Inst *inst, MemRef mem, RegMask preserved, bool storePair);
1892 template void Codegen::CreatePreWRB<false>(Inst *inst, MemRef mem, RegMask preserved, bool storePair);
1893
1894 /* RegMask 'preserved' is a way to explicitly specify registers that must be
1895 * saved before and restored after the PostWRB call.
1896 *
1897 * If codegen emits some new instructions after the PostWRB call and these
1898 * instructions use 'inst' operand registers, then the registers should live
1899 * through the PostWRB call. The problem is that the liveness analysis can't
1900 * detect if those registers are live because there are no instructions using
1901 * them at the moment.
1902 *
1903 * Note, only CALLER_REG_MASK registers are taken into account.
1904 */
CreatePostWRB(Inst * inst,MemRef mem,Reg reg1,Reg reg2,RegMask preserved)1905 void Codegen::CreatePostWRB(Inst *inst, MemRef mem, Reg reg1, Reg reg2, RegMask preserved)
1906 {
1907 ASSERT(reg1 != INVALID_REGISTER);
1908
1909 if (!GetGraph()->SupportsIrtocBarriers() || !GetGraph()->IsOfflineCompilationMode()) {
1910 auto barrierType = GetRuntime()->GetPostType();
1911 if (barrierType == ark::mem::BarrierType::POST_WRB_NONE) {
1912 return;
1913 }
1914 ASSERT(barrierType == ark::mem::BarrierType::POST_INTERGENERATIONAL_BARRIER ||
1915 barrierType == ark::mem::BarrierType::POST_INTERREGION_BARRIER);
1916 }
1917
1918 // For dynamic methods, another check
1919 if (GetGraph()->IsDynamicMethod()) {
1920 CreatePostWRBForDynamic(inst, mem, reg1, reg2, preserved);
1921 return;
1922 }
1923 PostWriteBarrier pwb(this, inst);
1924 Inst *secondValue;
1925 Inst *val = InstStoredValue(inst, &secondValue);
1926 ASSERT(secondValue == nullptr || reg2 != INVALID_REGISTER);
1927 if (val->GetOpcode() == Opcode::NullPtr) {
1928 // We can don't encode Post barrier for nullptr
1929 if (secondValue == nullptr || secondValue->GetOpcode() == Opcode::NullPtr) {
1930 return;
1931 }
1932 // CallPostWRB only for second reg
1933 auto secondMemOffset = reg1.GetSize() / BITS_PER_BYTE;
1934 if (!mem.HasIndex()) {
1935 MemRef secondMem(mem.GetBase(), mem.GetDisp() + secondMemOffset);
1936 pwb.Encode(secondMem, reg2, INVALID_REGISTER, !IsInstNotNull(secondValue), preserved);
1937 return;
1938 }
1939 ASSERT(mem.GetScale() == 0 && !mem.HasDisp());
1940 ASSERT(GetTarget().GetTempRegsMask().Test(mem.GetIndex().GetId()));
1941 GetEncoder()->EncodeAdd(mem.GetIndex(), mem.GetIndex(), Imm(secondMemOffset));
1942 pwb.Encode(mem, reg2, INVALID_REGISTER, !IsInstNotNull(secondValue), preserved);
1943 return;
1944 }
1945 // Create PostWRB only for first value
1946 if (secondValue != nullptr && secondValue->GetOpcode() == Opcode::NullPtr) {
1947 reg2 = INVALID_REGISTER;
1948 }
1949 bool checkObject = true;
1950 if (reg2 == INVALID_REGISTER) {
1951 if (IsInstNotNull(val)) {
1952 checkObject = false;
1953 }
1954 } else {
1955 if (IsInstNotNull(val) && IsInstNotNull(secondValue)) {
1956 checkObject = false;
1957 }
1958 }
1959 pwb.Encode(mem, reg1, reg2, checkObject, preserved);
1960 }
1961
CreatePostWRBForDynamic(Inst * inst,MemRef mem,Reg reg1,Reg reg2,RegMask preserved)1962 void Codegen::CreatePostWRBForDynamic(Inst *inst, MemRef mem, Reg reg1, Reg reg2, RegMask preserved)
1963 {
1964 PostWriteBarrier pwb(this, inst);
1965 if (reg2 == INVALID_REGISTER) {
1966 int storeIndex;
1967 if (inst->GetOpcode() == Opcode::StoreObject || inst->GetOpcode() == Opcode::StoreI ||
1968 inst->GetOpcode() == Opcode::StoreArrayI) {
1969 storeIndex = 1_I;
1970 } else {
1971 ASSERT(inst->GetOpcode() == Opcode::StoreArray || inst->GetOpcode() == Opcode::Store);
1972 storeIndex = 2_I;
1973 }
1974 if (StoreValueCanBeObject(inst->GetDataFlowInput(storeIndex))) {
1975 pwb.Encode(mem, reg1, reg2, true, preserved);
1976 }
1977 } else {
1978 int storeIndex;
1979 if (inst->GetOpcode() == Opcode::StoreArrayPairI) {
1980 storeIndex = 1_I;
1981 } else {
1982 ASSERT(inst->GetOpcode() == Opcode::StoreArrayPair);
1983 storeIndex = 2_I;
1984 }
1985 bool firstIsObject = StoreValueCanBeObject(inst->GetDataFlowInput(storeIndex));
1986 bool secondIsObject = StoreValueCanBeObject(inst->GetDataFlowInput(storeIndex + 1));
1987 if (firstIsObject || secondIsObject) {
1988 if (firstIsObject && !secondIsObject) {
1989 reg2 = INVALID_REGISTER;
1990 } else if (!firstIsObject && secondIsObject) {
1991 reg1 = reg2;
1992 reg2 = INVALID_REGISTER;
1993 }
1994 pwb.Encode(mem, reg1, reg2, true, preserved);
1995 }
1996 }
1997 }
1998
CheckObject(Reg reg,LabelHolder::LabelId label)1999 void Codegen::CheckObject(Reg reg, LabelHolder::LabelId label)
2000 {
2001 auto graph = GetGraph();
2002 auto *enc = GetEncoder();
2003
2004 // interpreter use x20 in IrToc and we don't have enough temporary registers
2005 // remove after PR 98 or PR 47
2006 if (graph->IsDynamicMethod()) {
2007 ASSERT(reg.IsScalar());
2008 reg = reg.As(INT64_TYPE);
2009 auto tagMask = graph->GetRuntime()->GetTaggedTagMask();
2010 // Check that the value is object(not int and not double)
2011 enc->EncodeJumpTest(label, reg, Imm(tagMask), Condition::TST_NE);
2012 auto specialMask = graph->GetRuntime()->GetTaggedSpecialMask();
2013 // Check that the value is not special value
2014 enc->EncodeJumpTest(label, reg, Imm(~specialMask), Condition::TST_EQ);
2015 } else {
2016 enc->EncodeJump(label, reg, Condition::EQ);
2017 }
2018 }
2019
HasLiveCallerSavedRegs(Inst * inst)2020 bool Codegen::HasLiveCallerSavedRegs(Inst *inst)
2021 {
2022 auto [live_regs, live_fp_regs] = GetLiveRegisters<false>(inst);
2023 live_regs &= GetCallerRegsMask(GetArch(), false);
2024 live_fp_regs &= GetCallerRegsMask(GetArch(), true);
2025 return live_regs.Any() || live_fp_regs.Any();
2026 }
2027
SaveCallerRegisters(RegMask liveRegs,VRegMask liveVregs,bool adjustRegs)2028 void Codegen::SaveCallerRegisters(RegMask liveRegs, VRegMask liveVregs, bool adjustRegs)
2029 {
2030 SCOPED_DISASM_STR(this, "Save caller saved regs");
2031 auto base = GetFrameInfo()->GetCallersRelativeFp() ? GetTarget().GetFrameReg() : GetTarget().GetStackReg();
2032 liveRegs &= ~GetEncoder()->GetAvailableScratchRegisters();
2033 liveVregs &= ~GetEncoder()->GetAvailableScratchFpRegisters();
2034 if (adjustRegs) {
2035 liveRegs &= GetRegfile()->GetCallerSavedRegMask();
2036 liveVregs &= GetRegfile()->GetCallerSavedVRegMask();
2037 } else {
2038 liveRegs &= GetCallerRegsMask(GetArch(), false);
2039 liveVregs &= GetCallerRegsMask(GetArch(), true);
2040 }
2041 if (GetFrameInfo()->GetPushCallers()) {
2042 GetEncoder()->PushRegisters(liveRegs, liveVregs);
2043 } else {
2044 GetEncoder()->SaveRegisters(liveRegs, false, GetFrameInfo()->GetCallersOffset(), base,
2045 GetCallerRegsMask(GetArch(), false));
2046 GetEncoder()->SaveRegisters(liveVregs, true, GetFrameInfo()->GetFpCallersOffset(), base,
2047 GetFrameInfo()->GetPositionedCallers() ? GetCallerRegsMask(GetArch(), true)
2048 : RegMask());
2049 }
2050 }
2051
LoadCallerRegisters(RegMask liveRegs,VRegMask liveVregs,bool adjustRegs)2052 void Codegen::LoadCallerRegisters(RegMask liveRegs, VRegMask liveVregs, bool adjustRegs)
2053 {
2054 SCOPED_DISASM_STR(this, "Restore caller saved regs");
2055 auto base = GetFrameInfo()->GetCallersRelativeFp() ? GetTarget().GetFrameReg() : GetTarget().GetStackReg();
2056 liveRegs &= ~GetEncoder()->GetAvailableScratchRegisters();
2057 liveVregs &= ~GetEncoder()->GetAvailableScratchFpRegisters();
2058 if (adjustRegs) {
2059 liveRegs &= GetRegfile()->GetCallerSavedRegMask();
2060 liveVregs &= GetRegfile()->GetCallerSavedVRegMask();
2061 } else {
2062 liveRegs &= GetCallerRegsMask(GetArch(), false);
2063 liveVregs &= GetCallerRegsMask(GetArch(), true);
2064 }
2065 if (GetFrameInfo()->GetPushCallers()) {
2066 GetEncoder()->PopRegisters(liveRegs, liveVregs);
2067 } else {
2068 GetEncoder()->LoadRegisters(liveRegs, false, GetFrameInfo()->GetCallersOffset(), base,
2069 GetCallerRegsMask(GetArch(), false));
2070 GetEncoder()->LoadRegisters(liveVregs, true, GetFrameInfo()->GetFpCallersOffset(), base,
2071 GetCallerRegsMask(GetArch(), true));
2072 }
2073 }
2074
RegisterKeepCallArgument(CallInst * callInst,Reg reg)2075 bool Codegen::RegisterKeepCallArgument(CallInst *callInst, Reg reg)
2076 {
2077 for (auto i = 0U; i < callInst->GetInputsCount(); i++) {
2078 auto location = callInst->GetLocation(i);
2079 if (location.IsRegister() && location.GetValue() == reg.GetId()) {
2080 return true;
2081 }
2082 }
2083 return false;
2084 }
2085
LoadMethod(Reg dst)2086 void Codegen::LoadMethod(Reg dst)
2087 {
2088 ASSERT((CFrameMethod::GetOffsetFromSpInBytes(GetFrameLayout()) -
2089 (GetFrameLayout().GetMethodOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::BYTES>())) ==
2090 0);
2091 auto spOffset = CFrameMethod::GetOffsetFromSpInBytes(GetFrameLayout());
2092 auto mem = MemRef(SpReg(), spOffset);
2093 GetEncoder()->EncodeLdr(dst, false, mem);
2094 }
2095
StoreFreeSlot(Reg src)2096 void Codegen::StoreFreeSlot(Reg src)
2097 {
2098 ASSERT(src.GetSize() <= (GetFrameLayout().GetSlotSize() << 3U));
2099 auto spOffset =
2100 GetFrameLayout().GetFreeSlotOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::BYTES>();
2101 auto mem = MemRef(SpReg(), spOffset);
2102 GetEncoder()->EncodeStr(src, mem);
2103 }
2104
LoadFreeSlot(Reg dst)2105 void Codegen::LoadFreeSlot(Reg dst)
2106 {
2107 ASSERT(dst.GetSize() <= (GetFrameLayout().GetSlotSize() << 3U));
2108 auto spOffset =
2109 GetFrameLayout().GetFreeSlotOffset<CFrameLayout::OffsetOrigin::SP, CFrameLayout::OffsetUnit::BYTES>();
2110 auto mem = MemRef(SpReg(), spOffset);
2111 GetEncoder()->EncodeLdr(dst, false, mem);
2112 }
2113
CreateReturn(const Inst * inst)2114 void Codegen::CreateReturn(const Inst *inst)
2115 {
2116 if (GetGraph()->GetMethodProperties().GetLastReturn() == inst) {
2117 GetEncoder()->BindLabel(GetLabelExit());
2118 GenerateEpilogue();
2119 } else {
2120 GetEncoder()->EncodeJump(GetLabelExit());
2121 }
2122 }
2123
EncodeDynamicCast(Inst * inst,Reg dst,bool dstSigned,Reg src)2124 void Codegen::EncodeDynamicCast(Inst *inst, Reg dst, bool dstSigned, Reg src)
2125 {
2126 CHECK_EQ(src.GetSize(), BITS_PER_UINT64);
2127 CHECK_GE(dst.GetSize(), BITS_PER_UINT32);
2128
2129 bool isDst64 {dst.GetSize() == BITS_PER_UINT64};
2130 dst = dst.As(INT32_TYPE);
2131
2132 auto enc {GetEncoder()};
2133 if (g_options.IsCpuFeatureEnabled(CpuFeature::JSCVT)) {
2134 // no slow path intended
2135 enc->EncodeFastPathDynamicCast(dst, src, LabelHolder::INVALID_LABEL);
2136 } else {
2137 auto slowPath {CreateSlowPath<SlowPathJsCastDoubleToInt32>(inst)};
2138 slowPath->SetDstReg(dst);
2139 slowPath->SetSrcReg(src);
2140
2141 enc->EncodeFastPathDynamicCast(dst, src, slowPath->GetLabel());
2142 slowPath->BindBackLabel(enc);
2143 }
2144
2145 if (isDst64) {
2146 enc->EncodeCast(dst.As(INT64_TYPE), dstSigned, dst, dstSigned);
2147 }
2148 }
2149
2150 // Next visitors use calling convention
VisitCallIndirect(CallIndirectInst * inst)2151 void Codegen::VisitCallIndirect(CallIndirectInst *inst)
2152 {
2153 auto location = inst->GetLocation(0);
2154 ASSERT(location.IsFixedRegister() && location.IsRegisterValid());
2155 auto src = Reg(location.GetValue(), GetTarget().GetPtrRegType());
2156 auto dst = ConvertRegister(inst->GetDstReg(), inst->GetType());
2157
2158 GetEncoder()->MakeCall(src);
2159 if (inst->HasUsers()) {
2160 GetEncoder()->EncodeMov(dst, GetTarget().GetReturnReg(dst.GetType()));
2161 }
2162 }
2163
VisitCall(CallInst * inst)2164 void Codegen::VisitCall(CallInst *inst)
2165 {
2166 ASSERT(GetGraph()->GetRelocationHandler() != nullptr);
2167
2168 auto mode = GetGraph()->GetMode();
2169 ASSERT(mode.IsFastPath() || mode.IsInterpreter() || mode.IsNative());
2170 ASSERT(mode.IsFastPath() || !HasLiveCallerSavedRegs(inst));
2171
2172 RegMask callerRegs;
2173 RegMask callerFpRegs;
2174 auto dstReg = ConvertRegister(inst->GetDstReg(), inst->GetType());
2175
2176 if (mode.IsFastPath()) {
2177 // irtoc fastpath needs to save all caller registers in case of call native function
2178 callerRegs = GetCallerRegsMask(GetArch(), false);
2179 callerFpRegs = GetCallerRegsMask(GetArch(), true);
2180 GetEncoder()->SetRegister(&callerRegs, &callerFpRegs, dstReg, false);
2181 SaveCallerRegisters(callerRegs, callerFpRegs, false);
2182 }
2183
2184 RelocationInfo relocation;
2185 relocation.data = inst->GetCallMethodId();
2186 GetEncoder()->MakeCall(&relocation);
2187 GetGraph()->GetRelocationHandler()->AddRelocation(relocation);
2188
2189 if (inst->HasUsers()) {
2190 ASSERT(dstReg.IsValid());
2191 GetEncoder()->EncodeMov(dstReg, GetTarget().GetReturnReg(dstReg.GetType()));
2192 }
2193
2194 if (mode.IsFastPath()) {
2195 LoadCallerRegisters(callerRegs, callerFpRegs, false);
2196 }
2197 }
2198
GetEntryPointId(uint64_t elementSize,RuntimeInterface::EntrypointId & eid)2199 static void GetEntryPointId(uint64_t elementSize, RuntimeInterface::EntrypointId &eid)
2200 {
2201 switch (elementSize) {
2202 case sizeof(uint8_t):
2203 eid = RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB8;
2204 break;
2205 case sizeof(uint16_t):
2206 eid = RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB16;
2207 break;
2208 case sizeof(uint32_t):
2209 eid = RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB32;
2210 break;
2211 case sizeof(uint64_t):
2212 eid = RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB64;
2213 break;
2214 default:
2215 UNREACHABLE();
2216 }
2217 }
2218
VisitNewArray(Inst * inst)2219 void Codegen::VisitNewArray(Inst *inst)
2220 {
2221 auto method = inst->CastToNewArray()->GetMethod();
2222 auto dst = ConvertRegister(inst->GetDstReg(), inst->GetType());
2223 auto srcClass = ConvertRegister(inst->GetSrcReg(NewArrayInst::INDEX_CLASS), DataType::POINTER);
2224 auto srcSize = ConvertRegister(inst->GetSrcReg(NewArrayInst::INDEX_SIZE), DataType::Type::INT32);
2225 auto arrayType = inst->CastToNewArray()->GetTypeId();
2226 auto runtime = GetGraph()->GetRuntime();
2227 auto encoder = GetEncoder();
2228
2229 auto maxTlabSize = runtime->GetTLABMaxSize();
2230 // NOTE(msherstennikov): support NewArray fast path for arm32
2231 if (maxTlabSize == 0 || GetArch() == Arch::AARCH32) {
2232 CallRuntime(inst, EntrypointId::CREATE_ARRAY, dst, RegMask::GetZeroMask(), srcClass, srcSize);
2233 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
2234 encoder->EncodeMemoryBarrier(memory_order::RELEASE);
2235 }
2236 return;
2237 }
2238
2239 auto lenInst = inst->GetDataFlowInput(0);
2240 auto classArraySize = runtime->GetClassArraySize(GetArch());
2241 uint64_t arraySize = 0;
2242 uint64_t elementSize = runtime->GetArrayElementSize(method, arrayType);
2243 uint64_t alignment = runtime->GetTLABAlignment();
2244 if (lenInst->GetOpcode() == Opcode::Constant) {
2245 ASSERT(lenInst->GetType() == DataType::INT64);
2246 arraySize = lenInst->CastToConstant()->GetIntValue() * elementSize + classArraySize;
2247 arraySize = (arraySize & ~(alignment - 1U)) + ((arraySize % alignment) != 0U ? alignment : 0U);
2248 if (arraySize > maxTlabSize) {
2249 CallRuntime(inst, EntrypointId::CREATE_ARRAY, dst, RegMask::GetZeroMask(), srcClass, srcSize);
2250 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
2251 encoder->EncodeMemoryBarrier(memory_order::RELEASE);
2252 }
2253 return;
2254 }
2255 }
2256
2257 EntrypointId eid;
2258 GetEntryPointId(elementSize, eid);
2259 CallFastPath(inst, eid, dst, RegMask::GetZeroMask(), srcClass, srcSize);
2260 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
2261 encoder->EncodeMemoryBarrier(memory_order::RELEASE);
2262 }
2263 }
2264
CreateMonitorCall(MonitorInst * inst)2265 void Codegen::CreateMonitorCall(MonitorInst *inst)
2266 {
2267 auto src = ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // obj
2268 auto id = inst->IsExit() ? EntrypointId::MONITOR_EXIT_FAST_PATH : EntrypointId::MONITOR_ENTER_FAST_PATH;
2269 CallFastPath(inst, id, INVALID_REGISTER, RegMask::GetZeroMask(), src);
2270 }
2271
CreateMonitorCallOld(MonitorInst * inst)2272 void Codegen::CreateMonitorCallOld(MonitorInst *inst)
2273 {
2274 auto src = ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE); // obj
2275 auto dst = ConvertRegister(inst->GetDstReg(), inst->GetType());
2276 auto id = inst->IsExit() ? EntrypointId::UNLOCK_OBJECT : EntrypointId::LOCK_OBJECT;
2277 CallRuntime(inst, id, dst, RegMask::GetZeroMask(), src);
2278 }
2279
CreateCheckCastInterfaceCall(Inst * inst)2280 void Codegen::CreateCheckCastInterfaceCall(Inst *inst)
2281 {
2282 auto obj = ConvertRegister(inst->GetSrcReg(0), DataType::REFERENCE);
2283 auto interface = ConvertRegister(inst->GetSrcReg(SECOND_OPERAND), DataType::REFERENCE);
2284 CallFastPath(inst, EntrypointId::CHECK_CAST_INTERFACE, INVALID_REGISTER, RegMask::GetZeroMask(), obj, interface);
2285 }
2286
TryInsertImplicitNullCheck(Inst * inst,size_t prevOffset)2287 void Codegen::TryInsertImplicitNullCheck(Inst *inst, size_t prevOffset)
2288 {
2289 if (!IsSuitableForImplicitNullCheck(inst)) {
2290 return;
2291 }
2292 if (!inst->CanThrow()) {
2293 return;
2294 }
2295
2296 auto nullcheck = inst->GetInput(0).GetInst();
2297 ASSERT(nullcheck->GetOpcode() == Opcode::NullCheck && nullcheck->CastToNullCheck()->IsImplicit());
2298 auto currOffset = GetEncoder()->GetCursorOffset();
2299 ASSERT(currOffset > prevOffset);
2300 GetCodeBuilder()->AddImplicitNullCheck(currOffset, currOffset - prevOffset);
2301 CreateStackMap(nullcheck, inst);
2302 }
2303
CreateMemmoveUnchecked(IntrinsicInst * inst,Reg dst,SRCREGS src)2304 void Codegen::CreateMemmoveUnchecked([[maybe_unused]] IntrinsicInst *inst, [[maybe_unused]] Reg dst, SRCREGS src)
2305 {
2306 auto entrypointId = EntrypointId::INVALID;
2307 switch (inst->GetIntrinsicId()) {
2308 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPILER_MEMMOVE_UNCHECKED_1_BYTE:
2309 entrypointId = EntrypointId::ARRAY_COPY_TO_UNCHECKED_1_BYTE;
2310 break;
2311
2312 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPILER_MEMMOVE_UNCHECKED_2_BYTE:
2313 entrypointId = EntrypointId::ARRAY_COPY_TO_UNCHECKED_2_BYTE;
2314 break;
2315
2316 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPILER_MEMMOVE_UNCHECKED_4_BYTE:
2317 entrypointId = EntrypointId::ARRAY_COPY_TO_UNCHECKED_4_BYTE;
2318 break;
2319
2320 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPILER_MEMMOVE_UNCHECKED_8_BYTE:
2321 entrypointId = EntrypointId::ARRAY_COPY_TO_UNCHECKED_8_BYTE;
2322 break;
2323
2324 default:
2325 UNREACHABLE();
2326 break;
2327 }
2328 ASSERT(entrypointId != EntrypointId::INVALID);
2329 auto srcObj = src[FIRST_OPERAND];
2330 auto dstObj = src[SECOND_OPERAND];
2331 auto dstStart = src[THIRD_OPERAND];
2332 auto srcStart = src[FOURTH_OPERAND];
2333 auto srcEnd = src[FIFTH_OPERAND];
2334 CallFastPath(inst, entrypointId, INVALID_REGISTER, RegMask::GetZeroMask(), srcObj, dstObj, dstStart, srcStart,
2335 srcEnd);
2336 }
2337
CreateFloatIsInf(IntrinsicInst * inst,Reg dst,SRCREGS src)2338 void Codegen::CreateFloatIsInf([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2339 {
2340 GetEncoder()->EncodeIsInf(dst, src[0]);
2341 }
2342
CreateStringEquals(IntrinsicInst * inst,Reg dst,SRCREGS src)2343 void Codegen::CreateStringEquals([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2344 {
2345 auto entrypointId = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::STRING_EQUALS_COMPRESSED
2346 : EntrypointId::STRING_EQUALS;
2347 CallFastPath(inst, entrypointId, dst, {}, src[0], src[1U]);
2348 }
2349
CreateMathCeil(IntrinsicInst * inst,Reg dst,SRCREGS src)2350 void Codegen::CreateMathCeil([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2351 {
2352 GetEncoder()->EncodeCeil(dst, src[0]);
2353 }
2354
CreateMathFloor(IntrinsicInst * inst,Reg dst,SRCREGS src)2355 void Codegen::CreateMathFloor([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2356 {
2357 GetEncoder()->EncodeFloor(dst, src[0]);
2358 }
2359
CreateCountLeadingZeroBits(IntrinsicInst * inst,Reg dst,SRCREGS src)2360 void Codegen::CreateCountLeadingZeroBits([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2361 {
2362 GetEncoder()->EncodeCountLeadingZeroBits(Reg(dst.GetId(), src[0].GetType()), src[0]);
2363 }
2364
CreateStringBuilderChar(IntrinsicInst * inst,Reg dst,SRCREGS src)2365 void Codegen::CreateStringBuilderChar(IntrinsicInst *inst, [[maybe_unused]] Reg dst, SRCREGS src)
2366 {
2367 CallFastPath(inst, EntrypointId::STRING_BUILDER_CHAR, dst, RegMask::GetZeroMask(), src[0], src[1U]);
2368 }
2369
CreateStringBuilderBool(IntrinsicInst * inst,Reg dst,SRCREGS src)2370 void Codegen::CreateStringBuilderBool(IntrinsicInst *inst, [[maybe_unused]] Reg dst, SRCREGS src)
2371 {
2372 CallFastPath(inst, EntrypointId::STRING_BUILDER_BOOL, dst, RegMask::GetZeroMask(), src[0], src[1U]);
2373 }
2374
CreateStringBuilderString(IntrinsicInst * inst,Reg dst,SRCREGS src)2375 void Codegen::CreateStringBuilderString(IntrinsicInst *inst, [[maybe_unused]] Reg dst, SRCREGS src)
2376 {
2377 auto entrypointId = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::STRING_BUILDER_STRING_COMPRESSED
2378 : EntrypointId::STRING_BUILDER_STRING;
2379
2380 CallFastPath(inst, entrypointId, dst, RegMask::GetZeroMask(), src[0], src[1U]);
2381 }
2382
CreateStringFromCharArrayTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)2383 void Codegen::CreateStringFromCharArrayTlab(IntrinsicInst *inst, Reg dst, SRCREGS src)
2384 {
2385 CreateStringFromCharArrayTlab(static_cast<Inst *>(inst), dst, src);
2386 }
2387
CreateStringFromCharArrayTlab(Inst * inst,Reg dst,SRCREGS src)2388 void Codegen::CreateStringFromCharArrayTlab(Inst *inst, Reg dst, SRCREGS src)
2389 {
2390 auto runtime = GetGraph()->GetRuntime();
2391 auto offset = src[FIRST_OPERAND];
2392 auto count = src[SECOND_OPERAND];
2393 auto array = src[THIRD_OPERAND];
2394
2395 auto entryId = GetRuntime()->IsCompressedStringsEnabled()
2396 ? EntrypointId::CREATE_STRING_FROM_CHAR_ARRAY_TLAB_COMPRESSED
2397 : EntrypointId::CREATE_STRING_FROM_CHAR_ARRAY_TLAB;
2398
2399 if (GetRegfile()->GetZeroReg().GetId() == offset.GetId()) {
2400 entryId = GetRuntime()->IsCompressedStringsEnabled()
2401 ? EntrypointId::CREATE_STRING_FROM_ZERO_BASED_CHAR_ARRAY_TLAB_COMPRESSED
2402 : EntrypointId::CREATE_STRING_FROM_ZERO_BASED_CHAR_ARRAY_TLAB;
2403 }
2404
2405 if (GetGraph()->IsAotMode()) {
2406 ScopedTmpReg klassReg(GetEncoder());
2407 GetEncoder()->EncodeLdr(klassReg, false,
2408 MemRef(ThreadReg(), runtime->GetStringClassPointerTlsOffset(GetArch())));
2409 if (GetRegfile()->GetZeroReg().GetId() == offset.GetId()) {
2410 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), count, array, klassReg);
2411 } else {
2412 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), offset, count, array, klassReg);
2413 }
2414 } else {
2415 auto klassImm =
2416 TypedImm(reinterpret_cast<uintptr_t>(runtime->GetStringClass(GetGraph()->GetMethod(), nullptr)));
2417 if (GetRegfile()->GetZeroReg().GetId() == offset.GetId()) {
2418 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), count, array, klassImm);
2419 } else {
2420 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), offset, count, array, klassImm);
2421 }
2422 }
2423 }
2424
CreateStringFromStringTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)2425 void Codegen::CreateStringFromStringTlab(IntrinsicInst *inst, Reg dst, SRCREGS src)
2426 {
2427 auto entryId = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::CREATE_STRING_FROM_STRING_TLAB_COMPRESSED
2428 : EntrypointId::CREATE_STRING_FROM_STRING_TLAB;
2429 auto srcStr = src[FIRST_OPERAND];
2430 CallFastPath(inst, entryId, dst, RegMask::GetZeroMask(), srcStr);
2431 }
2432
CreateStringSubstringTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)2433 void Codegen::CreateStringSubstringTlab([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2434 {
2435 auto entrypointId = GetRuntime()->IsCompressedStringsEnabled()
2436 ? EntrypointId::SUB_STRING_FROM_STRING_TLAB_COMPRESSED
2437 : EntrypointId::SUB_STRING_FROM_STRING_TLAB;
2438 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND]);
2439 }
2440
CreateStringGetCharsTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)2441 void Codegen::CreateStringGetCharsTlab([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2442 {
2443 auto entrypointId = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::STRING_GET_CHARS_TLAB_COMPRESSED
2444 : EntrypointId::STRING_GET_CHARS_TLAB;
2445 auto runtime = GetGraph()->GetRuntime();
2446 if (GetGraph()->IsAotMode()) {
2447 ScopedTmpReg klassReg(GetEncoder());
2448 GetEncoder()->EncodeLdr(klassReg, false,
2449 MemRef(ThreadReg(), runtime->GetArrayU16ClassPointerTlsOffset(GetArch())));
2450 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
2451 klassReg);
2452 } else {
2453 auto klassImm = TypedImm(reinterpret_cast<uintptr_t>(runtime->GetArrayU16Class(GetGraph()->GetMethod())));
2454 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
2455 klassImm);
2456 }
2457 }
2458
CreateStringHashCode(IntrinsicInst * inst,Reg dst,SRCREGS src)2459 void Codegen::CreateStringHashCode([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2460 {
2461 auto entrypoint = GetRuntime()->IsCompressedStringsEnabled() ? EntrypointId::STRING_HASH_CODE_COMPRESSED
2462 : EntrypointId::STRING_HASH_CODE;
2463 auto strReg = src[FIRST_OPERAND];
2464 auto mref = MemRef(strReg, ark::coretypes::String::GetHashcodeOffset());
2465 auto slowPath = CreateSlowPath<SlowPathStringHashCode>(inst, entrypoint);
2466 slowPath->SetDstReg(dst);
2467 slowPath->SetSrcReg(strReg);
2468 if (dst.GetId() != strReg.GetId()) {
2469 GetEncoder()->EncodeLdr(ConvertRegister(dst.GetId(), DataType::INT32), false, mref);
2470 GetEncoder()->EncodeJump(slowPath->GetLabel(), dst, Condition::EQ);
2471 } else {
2472 ScopedTmpReg hashReg(GetEncoder(), INT32_TYPE);
2473 GetEncoder()->EncodeLdr(hashReg, false, mref);
2474 GetEncoder()->EncodeJump(slowPath->GetLabel(), hashReg, Condition::EQ);
2475 GetEncoder()->EncodeMov(dst, hashReg);
2476 }
2477 slowPath->BindBackLabel(GetEncoder());
2478 }
CreateStringCompareTo(IntrinsicInst * inst,Reg dst,SRCREGS src)2479 void Codegen::CreateStringCompareTo([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
2480 {
2481 auto str1 = src[FIRST_OPERAND];
2482 auto str2 = src[SECOND_OPERAND];
2483 CallFastPath(inst, EntrypointId::STRING_COMPARE_TO, dst, {}, str1, str2);
2484 }
2485 #include "intrinsics_codegen.inl"
2486
CreateBuiltinIntrinsic(IntrinsicInst * inst)2487 void Codegen::CreateBuiltinIntrinsic(IntrinsicInst *inst)
2488 {
2489 Reg dst = INVALID_REGISTER;
2490 SRCREGS src;
2491
2492 if (!inst->NoDest()) {
2493 dst = ConvertRegister(inst->GetDstReg(), inst->GetType());
2494 }
2495
2496 for (size_t i = 0; i < inst->GetInputsCount(); ++i) {
2497 if (inst->GetInput(i).GetInst()->IsSaveState()) {
2498 continue;
2499 }
2500 auto location = inst->GetLocation(i);
2501 auto type = inst->GetInputType(i);
2502 src[i] = ConvertRegister(location.GetValue(), type);
2503 }
2504 FillBuiltin(inst, src, dst);
2505 }
2506
GetNeedBarrierProperty(const Inst * inst)2507 static bool GetNeedBarrierProperty(const Inst *inst)
2508 {
2509 Opcode op = inst->GetOpcode();
2510 if (op == Opcode::LoadObject) {
2511 return inst->CastToLoadObject()->GetNeedBarrier();
2512 }
2513 if (op == Opcode::StoreObject) {
2514 return inst->CastToStoreObject()->GetNeedBarrier();
2515 }
2516 if (op == Opcode::LoadObjectPair) {
2517 return inst->CastToLoadObjectPair()->GetNeedBarrier();
2518 }
2519 if (op == Opcode::StoreObjectPair) {
2520 return inst->CastToStoreObjectPair()->GetNeedBarrier();
2521 }
2522 if (op == Opcode::LoadArray) {
2523 return inst->CastToLoadArray()->GetNeedBarrier();
2524 }
2525 if (op == Opcode::StoreArray) {
2526 return inst->CastToStoreArray()->GetNeedBarrier();
2527 }
2528 if (op == Opcode::LoadArrayI) {
2529 return inst->CastToLoadArrayI()->GetNeedBarrier();
2530 }
2531 if (op == Opcode::StoreArrayI) {
2532 return inst->CastToStoreArrayI()->GetNeedBarrier();
2533 }
2534 if (op == Opcode::LoadArrayPair) {
2535 return inst->CastToLoadArrayPair()->GetNeedBarrier();
2536 }
2537 if (op == Opcode::StoreArrayPair) {
2538 return inst->CastToStoreArrayPair()->GetNeedBarrier();
2539 }
2540 if (op == Opcode::LoadArrayPairI) {
2541 return inst->CastToLoadArrayPairI()->GetNeedBarrier();
2542 }
2543 if (op == Opcode::StoreArrayPairI) {
2544 return inst->CastToStoreArrayPairI()->GetNeedBarrier();
2545 }
2546 if (op == Opcode::LoadStatic) {
2547 return inst->CastToLoadStatic()->GetNeedBarrier();
2548 }
2549 if (op == Opcode::StoreStatic) {
2550 return inst->CastToStoreStatic()->GetNeedBarrier();
2551 }
2552 return false;
2553 }
2554
2555 /**
2556 * Returns true if codegen emits call(s) to some library function(s)
2557 * while processing the instruction.
2558 */
InstEncodedWithLibCall(const Inst * inst,Arch arch)2559 bool Codegen::InstEncodedWithLibCall(const Inst *inst, Arch arch)
2560 {
2561 ASSERT(inst != nullptr);
2562 Opcode op = inst->GetOpcode();
2563 if (op == Opcode::Mod) {
2564 auto dstType = inst->GetType();
2565 if (arch == Arch::AARCH64 || arch == Arch::X86_64) {
2566 return dstType == DataType::FLOAT32 || dstType == DataType::FLOAT64;
2567 }
2568 return arch == Arch::AARCH32;
2569 }
2570 if (op == Opcode::Div && arch == Arch::AARCH32) {
2571 auto dstType = inst->GetType();
2572 return dstType == DataType::INT64 || dstType == DataType::UINT64;
2573 }
2574 if (op == Opcode::Cast && arch == Arch::AARCH32) {
2575 auto dstType = inst->GetType();
2576 auto srcType = inst->GetInputType(0);
2577 if (dstType == DataType::FLOAT32 || dstType == DataType::FLOAT64) {
2578 return srcType == DataType::INT64 || srcType == DataType::UINT64;
2579 }
2580 if (srcType == DataType::FLOAT32 || srcType == DataType::FLOAT64) {
2581 return dstType == DataType::INT64 || dstType == DataType::UINT64;
2582 }
2583 return false;
2584 }
2585
2586 return GetNeedBarrierProperty(inst);
2587 }
2588
ConvertInstTmpReg(const Inst * inst,DataType::Type type) const2589 Reg Codegen::ConvertInstTmpReg(const Inst *inst, DataType::Type type) const
2590 {
2591 ASSERT(inst->GetTmpLocation().IsFixedRegister());
2592 return Reg(inst->GetTmpLocation().GetValue(), ConvertDataType(type, GetArch()));
2593 }
2594
ConvertInstTmpReg(const Inst * inst) const2595 Reg Codegen::ConvertInstTmpReg(const Inst *inst) const
2596 {
2597 return ConvertInstTmpReg(inst, Is64BitsArch(GetArch()) ? DataType::INT64 : DataType::INT32);
2598 }
2599
Encode(MemRef mem,Reg reg1,Reg reg2,bool checkObject,RegMask preserved)2600 void PostWriteBarrier::Encode(MemRef mem, Reg reg1, Reg reg2, bool checkObject, RegMask preserved)
2601 {
2602 ASSERT(reg1.IsValid());
2603 auto reference {TypeInfo::FromDataType(DataType::REFERENCE, cg_->GetArch())};
2604 PostWriteBarrier::Args args;
2605 args.mem = mem;
2606 args.reg1 = reg1.As(reference);
2607 args.reg2 = reg2.IsValid() ? reg2.As(reference) : INVALID_REGISTER;
2608 args.checkObject = checkObject;
2609 args.preserved = preserved;
2610 if (cg_->GetGraph()->SupportsIrtocBarriers()) {
2611 if (cg_->GetGraph()->IsOfflineCompilationMode()) {
2612 EncodeOfflineIrtocBarrier(args);
2613 } else {
2614 EncodeOnlineIrtocBarrier(args);
2615 }
2616 } else if (type_ == ark::mem::BarrierType::POST_INTERREGION_BARRIER) {
2617 EncodeInterRegionBarrier(args);
2618 } else {
2619 ASSERT(type_ == ark::mem::BarrierType::POST_INTERGENERATIONAL_BARRIER);
2620 EncodeInterGenerationalBarrier(mem.GetBase().As(reference));
2621 }
2622 }
2623
EncodeOfflineIrtocBarrier(Args args)2624 void PostWriteBarrier::EncodeOfflineIrtocBarrier(Args args)
2625 {
2626 auto hasObj2 = HasObject2(args);
2627 auto paramRegs = GetParamRegs(hasObj2 ? 4U : 3U, args);
2628 cg_->SaveCallerRegisters(paramRegs, VRegMask(), false);
2629 if (hasObj2) {
2630 FillCallParams(args.mem, args.reg1, args.reg2);
2631 } else {
2632 FillCallParams(args.mem, args.reg1);
2633 }
2634 // load function pointer from tls field
2635 auto offset {hasObj2 ? cross_values::GetManagedThreadPostWrbTwoObjectsOffset(cg_->GetArch())
2636 : cross_values::GetManagedThreadPostWrbOneObjectOffset(cg_->GetArch())};
2637 MemRef entry(cg_->ThreadReg(), offset);
2638 cg_->GetEncoder()->MakeCall(entry);
2639 cg_->LoadCallerRegisters(paramRegs, VRegMask(), false);
2640 }
2641
EncodeOnlineIrtocBarrier(Args args)2642 void PostWriteBarrier::EncodeOnlineIrtocBarrier(Args args)
2643 {
2644 SCOPED_DISASM_STR(cg_, "Post Online Irtoc-WRB");
2645 auto *enc {cg_->GetEncoder()};
2646 auto hasObj2 = HasObject2(args);
2647 if (type_ == ark::mem::BarrierType::POST_INTERREGION_BARRIER) {
2648 if (hasObj2) {
2649 EncodeOnlineIrtocRegionTwoRegsBarrier(args);
2650 } else {
2651 EncodeOnlineIrtocRegionOneRegBarrier(args);
2652 }
2653 } else {
2654 static constexpr auto ENTRYPOINT_ID {RuntimeInterface::EntrypointId::POST_INTER_GENERATIONAL_BARRIER0};
2655 auto base = GetBase(args);
2656 auto paramRegs = GetParamRegs(1U, args);
2657 cg_->SaveCallerRegisters(paramRegs, VRegMask(), false);
2658 auto paramReg0 = enc->GetTarget().GetParamReg(0);
2659 enc->EncodeMov(paramReg0, base);
2660 MemRef entry(cg_->ThreadReg(), cg_->GetRuntime()->GetEntrypointTlsOffset(cg_->GetArch(), ENTRYPOINT_ID));
2661 enc->MakeCall(entry);
2662 cg_->LoadCallerRegisters(paramRegs, VRegMask(), false);
2663 }
2664 }
2665
2666 /**
2667 * Post-write barrier for StorePair case.
2668 * The code below may mark 0, 1 or 2 cards. The idea is following:
2669 * if (the second object isn't null and is allocated in other from base object's region)
2670 * MarkOneCard(CardOfSecondField(mem))
2671 * if (address of the second field isn't aligned at the size of a card)
2672 * # i.e. each of store address (fields of the base objects) are related to the same card
2673 * goto Done
2674 *
2675 * if (the first object isn't null and is allocated in other from base object's region)
2676 * MarkOneCard(CardOfFirstField(mem))
2677 *
2678 * label: Done
2679 */
EncodeOnlineIrtocRegionTwoRegsBarrier(Args args)2680 void PostWriteBarrier::EncodeOnlineIrtocRegionTwoRegsBarrier(Args args)
2681 {
2682 static constexpr auto ENTRYPOINT_ID {RuntimeInterface::EntrypointId::POST_INTER_REGION_BARRIER_SLOW};
2683 auto *enc {cg_->GetEncoder()};
2684 auto base = GetBase(args);
2685 auto paramReg0 = enc->GetTarget().GetParamReg(0);
2686 MemRef entry(cg_->ThreadReg(), cg_->GetRuntime()->GetEntrypointTlsOffset(cg_->GetArch(), ENTRYPOINT_ID));
2687 auto paramRegs = GetParamRegs(1U, args);
2688 auto lblMarkCardAndExit = enc->CreateLabel();
2689 auto lblCheck1Obj = enc->CreateLabel();
2690 auto lblDone = enc->CreateLabel();
2691 EncodeCheckObject(base, args.reg2, lblCheck1Obj, args.checkObject);
2692 cg_->SaveCallerRegisters(paramRegs, VRegMask(), false);
2693 EncodeWrapOneArg(paramReg0, base, args.mem, args.reg1.GetSize() / BITS_PER_BYTE);
2694 {
2695 ScopedTmpReg tmp(enc, cg_->ConvertDataType(DataType::REFERENCE, cg_->GetArch()));
2696 enc->EncodeAnd(tmp, paramReg0, Imm(cross_values::GetCardAlignmentMask(cg_->GetArch())));
2697 enc->EncodeJump(lblMarkCardAndExit, tmp, Condition::NE);
2698 }
2699 enc->MakeCall(entry);
2700 cg_->LoadCallerRegisters(paramRegs, VRegMask(), false);
2701 enc->BindLabel(lblCheck1Obj);
2702 EncodeCheckObject(base, args.reg1, lblDone, args.checkObject);
2703 cg_->SaveCallerRegisters(paramRegs, VRegMask(), false);
2704 EncodeWrapOneArg(paramReg0, base, args.mem);
2705 enc->BindLabel(lblMarkCardAndExit);
2706 enc->MakeCall(entry);
2707 cg_->LoadCallerRegisters(paramRegs, VRegMask(), false);
2708 enc->BindLabel(lblDone);
2709 }
2710
EncodeOnlineIrtocRegionOneRegBarrier(Args args)2711 void PostWriteBarrier::EncodeOnlineIrtocRegionOneRegBarrier(Args args)
2712 {
2713 static constexpr auto ENTRYPOINT_ID {RuntimeInterface::EntrypointId::POST_INTER_REGION_BARRIER_SLOW};
2714 auto *enc {cg_->GetEncoder()};
2715 auto base = GetBase(args);
2716 auto paramReg0 = enc->GetTarget().GetParamReg(0);
2717 MemRef entry(cg_->ThreadReg(), cg_->GetRuntime()->GetEntrypointTlsOffset(cg_->GetArch(), ENTRYPOINT_ID));
2718 auto paramRegs = GetParamRegs(1U, args);
2719 auto skip = enc->CreateLabel();
2720 EncodeCheckObject(base, args.reg1, skip, args.checkObject);
2721 cg_->SaveCallerRegisters(paramRegs, VRegMask(), false);
2722 EncodeWrapOneArg(paramReg0, base, args.mem);
2723 enc->MakeCall(entry);
2724 cg_->LoadCallerRegisters(paramRegs, VRegMask(), false);
2725 enc->BindLabel(skip);
2726 }
2727
EncodeInterRegionBarrier(Args args)2728 void PostWriteBarrier::EncodeInterRegionBarrier(Args args)
2729 {
2730 SCOPED_DISASM_STR(cg_, "Post IR-WRB");
2731 ASSERT(type_ == ark::mem::BarrierType::POST_INTERREGION_BARRIER);
2732
2733 static constexpr auto ENTRYPOINT_ID {RuntimeInterface::EntrypointId::POST_WRB_UPDATE_CARD_FUNC_NO_BRIDGE};
2734 auto *enc {cg_->GetEncoder()};
2735 auto label = enc->CreateLabel();
2736 if (args.checkObject) {
2737 cg_->CheckObject(args.reg1, label);
2738 }
2739 auto regionSizeBit = GetBarrierOperandValue<uint8_t>(BARRIER_POSITION, "REGION_SIZE_BITS");
2740 auto base = GetBase(args);
2741 ScopedTmpReg tmp(enc, cg_->ConvertDataType(DataType::REFERENCE, cg_->GetArch()));
2742 // Compare first store value with mem
2743 enc->EncodeXor(tmp, base, args.reg1);
2744 enc->EncodeShr(tmp, tmp, Imm(regionSizeBit));
2745 enc->EncodeJump(label, tmp, Condition::EQ);
2746 auto [live_regs, live_vregs] = cg_->GetLiveRegisters<true>(inst_);
2747 live_regs |= args.preserved;
2748
2749 if (args.mem.HasIndex()) {
2750 ASSERT(args.mem.GetScale() == 0 && !args.mem.HasDisp());
2751 enc->EncodeAdd(tmp, base, args.mem.GetIndex());
2752 cg_->CallBarrier(live_regs, live_vregs, ENTRYPOINT_ID, tmp, args.reg1);
2753 } else if (args.mem.HasDisp()) {
2754 ASSERT(!args.mem.HasIndex());
2755 enc->EncodeAdd(tmp, base, Imm(args.mem.GetDisp()));
2756 cg_->CallBarrier(live_regs, live_vregs, ENTRYPOINT_ID, tmp, args.reg1);
2757 } else {
2758 cg_->CallBarrier(live_regs, live_vregs, ENTRYPOINT_ID, base, args.reg1);
2759 }
2760 enc->BindLabel(label);
2761
2762 if (HasObject2(args)) {
2763 auto label1 = enc->CreateLabel();
2764 if (args.checkObject) {
2765 cg_->CheckObject(args.reg2, label1);
2766 }
2767 // Compare second store value with mem
2768 enc->EncodeXor(tmp, base, args.reg2);
2769 enc->EncodeShr(tmp, tmp, Imm(regionSizeBit));
2770 enc->EncodeJump(label1, tmp, Condition::EQ);
2771 enc->EncodeAdd(tmp, base, Imm(args.reg1.GetSize() / BITS_PER_BYTE));
2772 if (args.mem.HasIndex()) {
2773 ASSERT(args.mem.GetScale() == 0 && !args.mem.HasDisp());
2774 enc->EncodeAdd(tmp, tmp, args.mem.GetIndex());
2775 } else if (args.mem.HasDisp()) {
2776 ASSERT(!args.mem.HasIndex());
2777 enc->EncodeAdd(tmp, tmp, Imm(args.mem.GetDisp()));
2778 }
2779 cg_->CallBarrier(live_regs, live_vregs, ENTRYPOINT_ID, tmp, args.reg2);
2780 enc->BindLabel(label1);
2781 }
2782 }
2783
EncodeInterGenerationalBarrier(Reg base)2784 void PostWriteBarrier::EncodeInterGenerationalBarrier(Reg base)
2785 {
2786 ASSERT(type_ == ark::mem::BarrierType::POST_INTERGENERATIONAL_BARRIER);
2787 SCOPED_DISASM_STR(cg_, "Post IG-WRB");
2788 auto *runtime {cg_->GetRuntime()};
2789 auto *enc {cg_->GetEncoder()};
2790 ScopedTmpReg tmp(enc);
2791 ScopedTmpReg tmp1(enc);
2792 // load AddressOf(MIN_ADDR) -> min_addr
2793 if (cg_->GetGraph()->IsOfflineCompilationMode()) {
2794 enc->EncodeLdr(tmp, false, MemRef(cg_->ThreadReg(), runtime->GetTlsCardTableMinAddrOffset(cg_->GetArch())));
2795 } else {
2796 auto minAddress = reinterpret_cast<uintptr_t>(GetBarrierOperandValue<void *>(BARRIER_POSITION, "MIN_ADDR"));
2797 enc->EncodeMov(tmp, Imm(minAddress));
2798 }
2799 // card_index = (AddressOf(obj.field) - min_addr) >> CARD_BITS
2800 EncodeCalculateCardIndex(base, &tmp, &tmp1);
2801 // load AddressOf(CARD_TABLE_ADDR) -> card_table_addr
2802 if (cg_->GetGraph()->IsOfflineCompilationMode()) {
2803 enc->EncodeLdr(tmp1.GetReg().As(INT64_TYPE), false,
2804 MemRef(cg_->ThreadReg(), runtime->GetTlsCardTableAddrOffset(cg_->GetArch())));
2805 } else {
2806 auto cardTableAddr =
2807 reinterpret_cast<uintptr_t>(GetBarrierOperandValue<uint8_t *>(BARRIER_POSITION, "CARD_TABLE_ADDR"));
2808 enc->EncodeMov(tmp1, Imm(cardTableAddr));
2809 }
2810 // card_addr = card_table_addr + card_index
2811 enc->EncodeAdd(tmp, tmp1, tmp);
2812 // store card_addr <- DIRTY_VAL
2813 auto dirtyVal = GetBarrierOperandValue<uint8_t>(BARRIER_POSITION, "DIRTY_VAL");
2814 auto tmp1B = cg_->ConvertRegister(tmp1.GetReg().GetId(), DataType::INT8);
2815 enc->EncodeMov(tmp1B, Imm(dirtyVal));
2816 enc->EncodeStr(tmp1B, MemRef(tmp));
2817 }
2818
EncodeCalculateCardIndex(Reg baseReg,ScopedTmpReg * tmp,ScopedTmpReg * tmp1)2819 void PostWriteBarrier::EncodeCalculateCardIndex(Reg baseReg, ScopedTmpReg *tmp, ScopedTmpReg *tmp1)
2820 {
2821 ASSERT(baseReg != INVALID_REGISTER);
2822 auto tmpType = tmp->GetReg().GetType();
2823 auto tmp1Type = tmp1->GetReg().GetType();
2824 if (baseReg.GetSize() < Reg(*tmp).GetSize()) {
2825 tmp->ChangeType(baseReg.GetType());
2826 tmp1->ChangeType(baseReg.GetType());
2827 }
2828 auto enc {cg_->GetEncoder()};
2829 enc->EncodeSub(*tmp, baseReg, *tmp);
2830 enc->EncodeShr(*tmp, *tmp, Imm(GetBarrierOperandValue<uint8_t>(BARRIER_POSITION, "CARD_BITS")));
2831 tmp->ChangeType(tmpType);
2832 tmp1->ChangeType(tmp1Type);
2833 }
2834
EncodeCheckObject(Reg base,Reg reg1,LabelHolder::LabelId skipLabel,bool checkNull)2835 void PostWriteBarrier::EncodeCheckObject(Reg base, Reg reg1, LabelHolder::LabelId skipLabel, bool checkNull)
2836 {
2837 auto enc {cg_->GetEncoder()};
2838 if (checkNull) {
2839 // Fast null check in-place for one register
2840 enc->EncodeJump(skipLabel, reg1, Condition::EQ);
2841 }
2842 ScopedTmpReg tmp(enc, cg_->ConvertDataType(DataType::REFERENCE, cg_->GetArch()));
2843 auto regionSizeBit = GetBarrierOperandValue<uint8_t>(BARRIER_POSITION, "REGION_SIZE_BITS");
2844 enc->EncodeXor(tmp, base, reg1);
2845 enc->EncodeShr(tmp, tmp, Imm(regionSizeBit));
2846 enc->EncodeJump(skipLabel, tmp, Condition::EQ);
2847 }
2848
EncodeWrapOneArg(Reg param,Reg base,MemRef mem,size_t additionalOffset)2849 void PostWriteBarrier::EncodeWrapOneArg(Reg param, Reg base, MemRef mem, size_t additionalOffset)
2850 {
2851 auto enc {cg_->GetEncoder()};
2852 if (mem.HasIndex()) {
2853 ASSERT(mem.GetScale() == 0 && !mem.HasDisp());
2854 enc->EncodeAdd(param, base, mem.GetIndex());
2855 if (additionalOffset != 0) {
2856 enc->EncodeAdd(param, param, Imm(additionalOffset));
2857 }
2858 } else if (mem.HasDisp()) {
2859 ASSERT(!mem.HasIndex());
2860 enc->EncodeAdd(param, base, Imm(mem.GetDisp() + additionalOffset));
2861 } else {
2862 enc->EncodeAdd(param, base, Imm(additionalOffset));
2863 }
2864 }
2865
2866 } // namespace ark::compiler
2867