1 /*
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "slow_path.h"
17 #include "codegen.h"
18
19 namespace ark::compiler {
20
Generate(Codegen * codegen)21 void SlowPathBase::Generate(Codegen *codegen)
22 {
23 ASSERT(!generated_);
24
25 #ifndef NDEBUG
26 std::string opcodeStr(GetInst()->GetOpcodeStr());
27 if (GetInst()->IsIntrinsic()) {
28 opcodeStr += "." + GetIntrinsicName(static_cast<IntrinsicInst *>(GetInst())->GetIntrinsicId());
29 }
30 #endif
31 SCOPED_DISASM_STR(codegen,
32 std::string("SlowPath for inst ") + std::to_string(GetInst()->GetId()) + ". " + opcodeStr);
33 Encoder *encoder = codegen->GetEncoder();
34 ASSERT(encoder->IsValid());
35 encoder->BindLabel(GetLabel());
36
37 GenerateImpl(codegen);
38
39 if (encoder->IsLabelValid(labelBack_)) {
40 codegen->GetEncoder()->EncodeJump(GetBackLabel());
41 }
42 #ifndef NDEBUG
43 generated_ = true;
44 #endif
45 }
46
47 // ARRAY_INDEX_OUT_OF_BOUNDS_EXCEPTION, STRING_INDEX_OUT_OF_BOUNDS_EXCEPTION
GenerateThrowOutOfBoundsException(Codegen * codegen)48 bool SlowPathEntrypoint::GenerateThrowOutOfBoundsException(Codegen *codegen)
49 {
50 auto lenReg = codegen->ConvertRegister(GetInst()->GetSrcReg(0), GetInst()->GetInputType(0));
51 if (GetInst()->GetOpcode() == Opcode::BoundsCheckI) {
52 ScopedTmpReg indexReg(codegen->GetEncoder());
53 codegen->GetEncoder()->EncodeMov(indexReg, Imm(GetInst()->CastToBoundsCheckI()->GetImm()));
54 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, RegMask::GetZeroMask(), indexReg, lenReg);
55 } else {
56 ASSERT(GetInst()->GetOpcode() == Opcode::BoundsCheck);
57 auto indexReg = codegen->ConvertRegister(GetInst()->GetSrcReg(1), GetInst()->GetInputType(1));
58 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, RegMask::GetZeroMask(), indexReg, lenReg);
59 }
60 return true;
61 }
62
63 // INITIALIZE_CLASS
GenerateInitializeClass(Codegen * codegen)64 bool SlowPathEntrypoint::GenerateInitializeClass(Codegen *codegen)
65 {
66 auto inst = GetInst();
67 if (GetInst()->GetDstReg() != INVALID_REG) {
68 ASSERT(inst->GetOpcode() == Opcode::LoadAndInitClass);
69 Reg klassReg {codegen->ConvertRegister(GetInst()->GetDstReg(), DataType::REFERENCE)};
70 RegMask preservedRegs;
71 codegen->GetEncoder()->SetRegister(&preservedRegs, nullptr, klassReg);
72 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, preservedRegs, klassReg);
73 } else {
74 ASSERT(inst->GetOpcode() == Opcode::InitClass);
75 ASSERT(!codegen->GetGraph()->IsAotMode());
76 // check uintptr_t for cross:
77 auto klass = reinterpret_cast<uintptr_t>(inst->CastToInitClass()->GetClass());
78 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, RegMask::GetZeroMask(), TypedImm(klass));
79 }
80 return true;
81 }
82
83 // IS_INSTANCE
GenerateIsInstance(Codegen * codegen)84 bool SlowPathEntrypoint::GenerateIsInstance(Codegen *codegen)
85 {
86 auto src = codegen->ConvertRegister(GetInst()->GetSrcReg(0), DataType::REFERENCE); // obj
87 auto klass = codegen->ConvertRegister(GetInst()->GetSrcReg(1), DataType::REFERENCE);
88 auto dst = codegen->ConvertRegister(GetInst()->GetDstReg(), GetInst()->GetType());
89 codegen->CallRuntime(GetInst(), EntrypointId::IS_INSTANCE, dst, RegMask::GetZeroMask(), src, klass);
90 return true;
91 }
92
93 // CHECK_CAST
GenerateCheckCast(Codegen * codegen)94 bool SlowPathEntrypoint::GenerateCheckCast(Codegen *codegen)
95 {
96 auto src = codegen->ConvertRegister(GetInst()->GetSrcReg(0), DataType::REFERENCE); // obj
97 auto klass = codegen->ConvertRegister(GetInst()->GetSrcReg(1), DataType::REFERENCE);
98 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, RegMask::GetZeroMask(), src, klass);
99 return true;
100 }
101
102 // CREATE_OBJECT
GenerateCreateObject(Codegen * codegen)103 bool SlowPathEntrypoint::GenerateCreateObject(Codegen *codegen)
104 {
105 auto inst = GetInst();
106 auto dst = codegen->ConvertRegister(inst->GetDstReg(), inst->GetType());
107 auto src = codegen->ConvertRegister(inst->GetSrcReg(0), inst->GetInputType(0));
108
109 codegen->CallRuntime(inst, EntrypointId::CREATE_OBJECT_BY_CLASS, dst, RegMask::GetZeroMask(), src);
110
111 return true;
112 }
113
GenerateByEntry(Codegen * codegen)114 bool SlowPathEntrypoint::GenerateByEntry(Codegen *codegen)
115 {
116 switch (GetEntrypoint()) {
117 case EntrypointId::THROW_EXCEPTION: {
118 auto src = codegen->ConvertRegister(GetInst()->GetSrcReg(0), DataType::Type::REFERENCE);
119 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, RegMask::GetZeroMask(), src);
120 return true;
121 }
122 case EntrypointId::NULL_POINTER_EXCEPTION:
123 case EntrypointId::ARITHMETIC_EXCEPTION:
124 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, {});
125 return true;
126 case EntrypointId::ARRAY_INDEX_OUT_OF_BOUNDS_EXCEPTION:
127 case EntrypointId::STRING_INDEX_OUT_OF_BOUNDS_EXCEPTION:
128 return GenerateThrowOutOfBoundsException(codegen);
129 case EntrypointId::NEGATIVE_ARRAY_SIZE_EXCEPTION: {
130 auto size = codegen->ConvertRegister(GetInst()->GetSrcReg(0), GetInst()->GetInputType(0));
131 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, RegMask::GetZeroMask(), size);
132 return true;
133 }
134 case EntrypointId::INITIALIZE_CLASS:
135 return GenerateInitializeClass(codegen);
136 case EntrypointId::IS_INSTANCE:
137 return GenerateIsInstance(codegen);
138 case EntrypointId::CHECK_CAST:
139 case EntrypointId::CHECK_CAST_DEOPTIMIZE:
140 return GenerateCheckCast(codegen);
141 case EntrypointId::CREATE_OBJECT_BY_CLASS:
142 return GenerateCreateObject(codegen);
143 case EntrypointId::SAFEPOINT:
144 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, {});
145 return true;
146 default:
147 return false;
148 }
149 }
150
GenerateImpl(Codegen * codegen)151 void SlowPathEntrypoint::GenerateImpl(Codegen *codegen)
152 {
153 if (!GenerateByEntry(codegen)) {
154 switch (GetEntrypoint()) {
155 case EntrypointId::GET_UNKNOWN_CALLEE_METHOD:
156 case EntrypointId::RESOLVE_UNKNOWN_VIRTUAL_CALL:
157 case EntrypointId::GET_FIELD_OFFSET:
158 case EntrypointId::GET_UNKNOWN_STATIC_FIELD_MEMORY_ADDRESS:
159 case EntrypointId::RESOLVE_CLASS_OBJECT:
160 case EntrypointId::RESOLVE_CLASS:
161 case EntrypointId::ABSTRACT_METHOD_ERROR:
162 case EntrypointId::INITIALIZE_CLASS_BY_ID:
163 case EntrypointId::CHECK_STORE_ARRAY_REFERENCE:
164 case EntrypointId::RESOLVE_STRING_AOT:
165 case EntrypointId::CLASS_CAST_EXCEPTION:
166 break;
167 default:
168 LOG(FATAL, COMPILER) << "Unsupported entrypoint!";
169 UNREACHABLE();
170 break;
171 }
172 }
173 }
174
GenerateImpl(Codegen * codegen)175 void SlowPathDeoptimize::GenerateImpl(Codegen *codegen)
176 {
177 uintptr_t value =
178 helpers::ToUnderlying(deoptimizeType_) | (GetInst()->GetId() << MinimumBitsToStore(DeoptimizeType::COUNT));
179 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, RegMask::GetZeroMask(), TypedImm(value));
180 }
181
GenerateImpl(Codegen * codegen)182 void SlowPathIntrinsic::GenerateImpl(Codegen *codegen)
183 {
184 codegen->CreateCallIntrinsic(GetInst()->CastToIntrinsic());
185 }
186
GenerateImpl(Codegen * codegen)187 void SlowPathImplicitNullCheck::GenerateImpl(Codegen *codegen)
188 {
189 ASSERT(!GetInst()->CastToNullCheck()->IsImplicit());
190 SlowPathEntrypoint::GenerateImpl(codegen);
191 }
192
GenerateImpl(Codegen * codegen)193 void SlowPathShared::GenerateImpl(Codegen *codegen)
194 {
195 ASSERT(tmpReg_ != INVALID_REGISTER);
196 [[maybe_unused]] ScopedTmpReg tmpReg(codegen->GetEncoder(), tmpReg_);
197 ASSERT(tmpReg.GetReg().GetId() == tmpReg_.GetId());
198 auto graph = codegen->GetGraph();
199 ASSERT(graph->IsAotMode());
200 auto aotData = graph->GetAotData();
201 aotData->SetSharedSlowPathOffset(GetEntrypoint(), codegen->GetEncoder()->GetCursorOffset());
202 MemRef entry(codegen->ThreadReg(), graph->GetRuntime()->GetEntrypointTlsOffset(graph->GetArch(), GetEntrypoint()));
203 ScopedTmpReg tmp1Reg(codegen->GetEncoder());
204 codegen->GetEncoder()->EncodeLdr(tmp1Reg, false, entry);
205 codegen->GetEncoder()->EncodeJump(tmp1Reg);
206 }
207
GenerateImpl(Codegen * codegen)208 void SlowPathResolveStringAot::GenerateImpl(Codegen *codegen)
209 {
210 ScopedTmpRegU64 tmpAddrReg(codegen->GetEncoder());
211 // Slot address was loaded into temporary register before we jumped into slow path, but it is already released
212 // because temporary registers are scoped. Try to allocate a new one and check that it is the same register
213 // as was allocated in codegen. If it is a different register then copy the slot address into it.
214 if (tmpAddrReg.GetReg() != addrReg_) {
215 codegen->GetEncoder()->EncodeMov(tmpAddrReg, addrReg_);
216 }
217 codegen->CallRuntimeWithMethod(GetInst(), method_, GetEntrypoint(), dstReg_, TypedImm(stringId_), tmpAddrReg);
218 }
219
GenerateImpl(Codegen * codegen)220 void SlowPathRefCheck::GenerateImpl(Codegen *codegen)
221 {
222 ASSERT(arrayReg_ != INVALID_REGISTER);
223 ASSERT(refReg_ != INVALID_REGISTER);
224 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, RegMask::GetZeroMask(), arrayReg_, refReg_);
225 }
226
GenerateImpl(Codegen * codegen)227 void SlowPathAbstract::GenerateImpl(Codegen *codegen)
228 {
229 SCOPED_DISASM_STR(codegen, std::string("SlowPath for Abstract method ") + std::to_string(GetInst()->GetId()));
230 ASSERT(methodReg_ != INVALID_REGISTER);
231 ScopedTmpReg methodReg(codegen->GetEncoder(), methodReg_);
232 ASSERT(methodReg.GetReg().GetId() == methodReg_.GetId());
233 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, RegMask::GetZeroMask(), methodReg.GetReg());
234 }
235
GenerateImpl(Codegen * codegen)236 void SlowPathCheckCast::GenerateImpl(Codegen *codegen)
237 {
238 SCOPED_DISASM_STR(codegen, std::string("SlowPath for CheckCast exception") + std::to_string(GetInst()->GetId()));
239 auto inst = GetInst();
240 auto src = codegen->ConvertRegister(inst->GetSrcReg(0), inst->GetInputType(0));
241
242 codegen->CallRuntime(GetInst(), GetEntrypoint(), INVALID_REGISTER, RegMask::GetZeroMask(), classReg_, src);
243 }
244
GenerateImpl(Codegen * codegen)245 void SlowPathUnresolved::GenerateImpl(Codegen *codegen)
246 {
247 SlowPathEntrypoint::GenerateImpl(codegen);
248
249 ASSERT(method_ != nullptr);
250 ASSERT(typeId_ != 0);
251 ASSERT(slotAddr_ != 0);
252 auto typeImm = TypedImm(typeId_);
253 auto arch = codegen->GetGraph()->GetArch();
254 // On 32-bit architecture slot address requires additional down_cast,
255 // similar to `method` address processing in `CallRuntimeWithMethod`
256 auto slotImm = Is64BitsArch(arch) ? TypedImm(slotAddr_) : TypedImm(down_cast<uint32_t>(slotAddr_));
257
258 ScopedTmpReg valueReg(codegen->GetEncoder());
259 if (GetInst()->GetOpcode() == Opcode::ResolveVirtual) {
260 codegen->CallRuntimeWithMethod(GetInst(), method_, GetEntrypoint(), valueReg, argReg_, typeImm, slotImm);
261 } else if (GetEntrypoint() == EntrypointId::GET_UNKNOWN_CALLEE_METHOD ||
262 GetEntrypoint() == EntrypointId::GET_UNKNOWN_STATIC_FIELD_MEMORY_ADDRESS) {
263 codegen->CallRuntimeWithMethod(GetInst(), method_, GetEntrypoint(), valueReg, typeImm, slotImm);
264 } else {
265 codegen->CallRuntimeWithMethod(GetInst(), method_, GetEntrypoint(), valueReg, typeImm);
266
267 ScopedTmpReg addrReg(codegen->GetEncoder());
268 codegen->GetEncoder()->EncodeMov(addrReg, Imm(slotAddr_));
269 codegen->GetEncoder()->EncodeStr(valueReg, MemRef(addrReg));
270 }
271
272 if (dstReg_.IsValid()) {
273 codegen->GetEncoder()->EncodeMov(dstReg_, valueReg);
274 }
275 }
276
GenerateImpl(Codegen * codegen)277 void SlowPathJsCastDoubleToInt32::GenerateImpl(Codegen *codegen)
278 {
279 ASSERT(dstReg_.IsValid());
280 ASSERT(srcReg_.IsValid());
281
282 auto enc {codegen->GetEncoder()};
283 if (codegen->GetGraph()->GetMode().SupportManagedCode()) {
284 ScopedTmpRegU64 tmp(enc);
285 enc->EncodeMov(tmp, srcReg_);
286 codegen->CallRuntime(GetInst(), EntrypointId::JS_CAST_DOUBLE_TO_INT32, dstReg_, RegMask::GetZeroMask(), tmp);
287 return;
288 }
289
290 auto [live_regs, live_vregs] {codegen->GetLiveRegisters<true>(GetInst())};
291 live_regs.Reset(dstReg_.GetId());
292
293 codegen->SaveCallerRegisters(live_regs, live_vregs, true);
294 codegen->FillCallParams(srcReg_);
295 codegen->EmitCallRuntimeCode(nullptr, EntrypointId::JS_CAST_DOUBLE_TO_INT32_NO_BRIDGE);
296
297 auto retReg {codegen->GetTarget().GetReturnReg(dstReg_.GetType())};
298 if (dstReg_.GetId() != retReg.GetId()) {
299 enc->EncodeMov(dstReg_, retReg);
300 }
301 codegen->LoadCallerRegisters(live_regs, live_vregs, true);
302 }
303
GenerateImpl(Codegen * codegen)304 void SlowPathStringHashCode::GenerateImpl(Codegen *codegen)
305 {
306 ASSERT(dstReg_.IsValid());
307 ASSERT(srcReg_.IsValid());
308 codegen->CallFastPath(GetInst(), GetEntrypoint(), dstReg_, RegMask::GetZeroMask(), srcReg_);
309 }
310
311 } // namespace ark::compiler
312