1 /**
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "runtime/entrypoints/entrypoints.h"
17
18 #include "libpandabase/events/events.h"
19 #include "libpandabase/macros.h"
20 #include "libpandabase/utils/utils.h"
21 #include "compiler/compiler_options.h"
22 #include "runtime/deoptimization.h"
23 #include "runtime/arch/memory_helpers.h"
24 #include "runtime/jit/profiling_data.h"
25 #include "runtime/include/class_linker-inl.h"
26 #include "runtime/include/coretypes/array.h"
27 #include "runtime/include/exceptions.h"
28 #include "runtime/include/method-inl.h"
29 #include "runtime/include/object_header-inl.h"
30 #include "runtime/include/runtime.h"
31 #include "runtime/include/value-inl.h"
32 #include "runtime/include/panda_vm.h"
33 #include "runtime/interpreter/frame.h"
34 #include "runtime/interpreter/interpreter.h"
35 #include "runtime/interpreter/runtime_interface.h"
36 #include "runtime/mem/tlab.h"
37 #include "compiler/optimizer/ir/runtime_interface.h"
38 #include "runtime/handle_base-inl.h"
39 #include "libpandabase/utils/asan_interface.h"
40 #include "libpandabase/utils/tsan_interface.h"
41 #include "utils/cframe_layout.h"
42 #include "intrinsics.h"
43 #include "runtime/interpreter/vregister_iterator.h"
44
45 namespace ark {
46
47 using ark::compiler::TraceId;
48 using ark::coretypes::String;
49
50 #undef LOG_ENTRYPOINTS
51
52 class ScopedLog {
53 public:
54 ScopedLog() = delete;
ScopedLog(const char * function)55 explicit ScopedLog(const char *function) : function_(function)
56 {
57 LOG(DEBUG, INTEROP) << "ENTRYPOINT: " << function;
58 }
~ScopedLog()59 ~ScopedLog()
60 {
61 LOG(DEBUG, INTEROP) << "EXIT ENTRYPOINT: " << function_;
62 }
63 NO_COPY_SEMANTIC(ScopedLog);
64 NO_MOVE_SEMANTIC(ScopedLog);
65
66 private:
67 std::string function_;
68 };
69
70 #ifdef LOG_ENTRYPOINTS
71 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
72 #define LOG_ENTRYPOINT() ScopedLog __log(__FUNCTION__)
73 #else
74 #define LOG_ENTRYPOINT()
75 #endif
76
77 // enable stack walker dry run on each entrypoint to discover stack issues early
78 #ifndef NDEBUG
79 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
80 #define CHECK_STACK_WALKER \
81 if (Runtime::GetOptions().IsVerifyEntrypoints()) { \
82 StackWalker::Create(ManagedThread::GetCurrent()).Verify(); \
83 }
84 #else
85 #define CHECK_STACK_WALKER
86 #endif
87
88 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
89 #define BEGIN_ENTRYPOINT() \
90 CHECK_STACK_WALKER; \
91 LOG_ENTRYPOINT()
92
InterpreterEntryPoint(Method * method,Frame * frame)93 extern "C" NO_ADDRESS_SANITIZE void InterpreterEntryPoint(Method *method, Frame *frame)
94 {
95 auto pc = method->GetInstructions();
96 Method *callee = frame->GetMethod();
97 ASSERT(callee != nullptr);
98
99 if (callee->IsAbstract()) {
100 ASSERT(pc == nullptr);
101 ark::ThrowAbstractMethodError(callee);
102 HandlePendingException();
103 UNREACHABLE();
104 }
105
106 ManagedThread *thread = ManagedThread::GetCurrent();
107 if (!thread->template StackOverflowCheck<true, false>()) {
108 HandlePendingException(UnwindPolicy::SKIP_INLINED);
109 UNREACHABLE();
110 }
111
112 Frame *prevFrame = thread->GetCurrentFrame();
113 thread->SetCurrentFrame(frame);
114
115 auto isCompiledCode = thread->IsCurrentFrameCompiled();
116 thread->SetCurrentFrameIsCompiled(false);
117 interpreter::Execute(thread, pc, frame);
118 thread->SetCurrentFrameIsCompiled(isCompiledCode);
119 if (prevFrame != nullptr && reinterpret_cast<uintptr_t>(prevFrame->GetMethod()) == COMPILED_CODE_TO_INTERPRETER) {
120 thread->SetCurrentFrame(prevFrame->GetPrevFrame());
121 } else {
122 thread->SetCurrentFrame(prevFrame);
123 }
124 }
125
AnnotateSanitizersEntrypoint(void const * addr,size_t size)126 extern "C" void AnnotateSanitizersEntrypoint([[maybe_unused]] void const *addr, [[maybe_unused]] size_t size)
127 {
128 #ifdef PANDA_TSAN_ON
129 TSAN_ANNOTATE_HAPPENS_BEFORE(const_cast<void *>(addr));
130 #endif
131 #ifdef PANDA_ASAN_ON
132 __asan_unpoison_memory_region(addr, size);
133 #endif
134 }
135
WriteTlabStatsEntrypoint(void const * mem,size_t size)136 extern "C" void WriteTlabStatsEntrypoint([[maybe_unused]] void const *mem, size_t size)
137 {
138 LOG_ENTRYPOINT();
139
140 LOG(DEBUG, MM_OBJECT_EVENTS) << "Alloc object in compiled code at " << mem << " size: " << size;
141 auto *thread = ManagedThread::GetCurrent();
142 ASSERT(size <= Runtime::GetOptions().GetMaxTlabSize());
143 // 1. Pointer to TLAB
144 // 2. Pointer to allocated memory
145 // 3. size
146 [[maybe_unused]] auto tlab = reinterpret_cast<size_t>(thread->GetTLAB());
147 EVENT_TLAB_ALLOC(thread->GetId(), tlab, reinterpret_cast<size_t>(mem), size);
148 if (mem::PANDA_TRACK_TLAB_ALLOCATIONS) {
149 auto memStats = thread->GetVM()->GetHeapManager()->GetMemStats();
150 if (memStats == nullptr) {
151 return;
152 }
153 memStats->RecordAllocateObject(size, SpaceType::SPACE_TYPE_OBJECT);
154 }
155 }
156
CreateArraySlowPathEntrypoint(Class * klass,size_t length)157 extern "C" coretypes::Array *CreateArraySlowPathEntrypoint(Class *klass, size_t length)
158 {
159 BEGIN_ENTRYPOINT();
160
161 TSAN_ANNOTATE_HAPPENS_AFTER(klass);
162 auto arr = coretypes::Array::Create(klass, length);
163 if (UNLIKELY(arr == nullptr)) {
164 HandlePendingException();
165 UNREACHABLE();
166 }
167 if (compiler::g_options.IsCompilerEnableTlabEvents()) {
168 EVENT_SLOWPATH_ALLOC(ManagedThread::GetCurrent()->GetId());
169 }
170 return arr;
171 }
172
CreateMultiArrayRecEntrypoint(ManagedThread * thread,Class * klass,uint32_t nargs,size_t * sizes,uint32_t num)173 extern "C" coretypes::Array *CreateMultiArrayRecEntrypoint(ManagedThread *thread, Class *klass, uint32_t nargs,
174 size_t *sizes, uint32_t num)
175 {
176 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic,-warnings-as-errors)
177 auto arrSize = sizes[num];
178
179 [[maybe_unused]] HandleScope<ObjectHeader *> scope(thread);
180 // SUPPRESS_CSA_NEXTLINE(alpha.core.CheckObjHeaderTypeRef)
181 VMHandle<coretypes::Array> handle(thread, coretypes::Array::Create(klass, arrSize));
182 if (handle.GetPtr() == nullptr) {
183 return nullptr;
184 }
185 auto *component = klass->GetComponentType();
186
187 if (component->IsArrayClass() && num + 1 < nargs) {
188 for (size_t idx = 0; idx < arrSize; idx++) {
189 auto *array = CreateMultiArrayRecEntrypoint(thread, component, nargs, sizes, num + 1);
190
191 if (array == nullptr) {
192 return nullptr;
193 }
194 handle.GetPtr()->template Set<coretypes::Array *>(idx, array);
195 }
196 }
197
198 return handle.GetPtr();
199 }
200
CreateEmptyStringEntrypoint()201 extern "C" coretypes::String *CreateEmptyStringEntrypoint()
202 {
203 BEGIN_ENTRYPOINT();
204
205 auto vm = ManagedThread::GetCurrent()->GetVM();
206 auto str = coretypes::String::CreateEmptyString(vm->GetLanguageContext(), vm);
207 if (UNLIKELY(str == nullptr)) {
208 HandlePendingException();
209 UNREACHABLE();
210 }
211 return str;
212 }
213
CreateStringFromStringEntrypoint(ObjectHeader * obj)214 extern "C" coretypes::String *CreateStringFromStringEntrypoint(ObjectHeader *obj)
215 {
216 BEGIN_ENTRYPOINT();
217 auto vm = ManagedThread::GetCurrent()->GetVM();
218 auto str = coretypes::String::CreateFromString(static_cast<coretypes::String *>(obj), vm->GetLanguageContext(), vm);
219 if (UNLIKELY(str == nullptr)) {
220 HandlePendingException();
221 UNREACHABLE();
222 }
223 return str;
224 }
225
CreateStringFromCharsEntrypoint(ObjectHeader * obj)226 extern "C" coretypes::String *CreateStringFromCharsEntrypoint(ObjectHeader *obj)
227 {
228 BEGIN_ENTRYPOINT();
229 auto vm = ManagedThread::GetCurrent()->GetVM();
230 auto array = static_cast<coretypes::Array *>(obj);
231 auto str = coretypes::String::CreateNewStringFromChars(0, array->GetLength(), array, vm->GetLanguageContext(), vm);
232 if (UNLIKELY(str == nullptr)) {
233 HandlePendingException();
234 UNREACHABLE();
235 }
236 return str;
237 }
238
CreateStringFromCharsWithOffsetEntrypoint(uint32_t offset,uint32_t length,ObjectHeader * obj)239 extern "C" coretypes::String *CreateStringFromCharsWithOffsetEntrypoint(uint32_t offset, uint32_t length,
240 ObjectHeader *obj)
241 {
242 BEGIN_ENTRYPOINT();
243 auto vm = ManagedThread::GetCurrent()->GetVM();
244 auto array = static_cast<coretypes::Array *>(obj);
245 auto str = coretypes::String::CreateNewStringFromChars(offset, length, array, vm->GetLanguageContext(), vm);
246 if (UNLIKELY(str == nullptr)) {
247 HandlePendingException();
248 UNREACHABLE();
249 }
250 return str;
251 }
252
CreateStringFromCharsZeroOffsetEntrypoint(uint32_t length,ObjectHeader * obj)253 extern "C" coretypes::String *CreateStringFromCharsZeroOffsetEntrypoint(uint32_t length, ObjectHeader *obj)
254 {
255 BEGIN_ENTRYPOINT();
256 auto vm = ManagedThread::GetCurrent()->GetVM();
257 auto array = static_cast<coretypes::Array *>(obj);
258 auto str = coretypes::String::CreateNewStringFromChars(0, length, array, vm->GetLanguageContext(), vm);
259 if (UNLIKELY(str == nullptr)) {
260 HandlePendingException();
261 UNREACHABLE();
262 }
263 return str;
264 }
265
SubStringFromStringEntrypoint(ObjectHeader * obj,int32_t begin,int32_t end)266 extern "C" coretypes::String *SubStringFromStringEntrypoint(ObjectHeader *obj, int32_t begin, int32_t end)
267 {
268 BEGIN_ENTRYPOINT();
269
270 auto vm = ManagedThread::GetCurrent()->GetVM();
271 auto indexes = coretypes::String::NormalizeSubStringIndexes(begin, end, static_cast<coretypes::String *>(obj));
272 auto substrLength = indexes.second - indexes.first;
273 auto substr = coretypes::String::FastSubString(static_cast<coretypes::String *>(obj), indexes.first, substrLength,
274 vm->GetLanguageContext(), vm);
275 if (UNLIKELY(substr == nullptr)) {
276 HandlePendingException();
277 UNREACHABLE();
278 }
279 return substr;
280 }
281
StringGetCharsEntrypoint(ObjectHeader * obj,int32_t begin,int32_t end)282 extern "C" coretypes::Array *StringGetCharsEntrypoint(ObjectHeader *obj, int32_t begin, int32_t end)
283 {
284 BEGIN_ENTRYPOINT();
285
286 auto length = static_cast<coretypes::String *>(obj)->GetLength();
287 if (UNLIKELY(static_cast<uint32_t>(end) > length)) {
288 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
289 ark::ThrowStringIndexOutOfBoundsException(end, length);
290 HandlePendingException(UnwindPolicy::SKIP_INLINED);
291 UNREACHABLE();
292 }
293 if (UNLIKELY(begin > end)) {
294 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
295 ark::ThrowStringIndexOutOfBoundsException(begin, length);
296 HandlePendingException(UnwindPolicy::SKIP_INLINED);
297 UNREACHABLE();
298 }
299
300 if (UNLIKELY(begin < 0)) {
301 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
302 ark::ThrowStringIndexOutOfBoundsException(begin, length);
303 HandlePendingException(UnwindPolicy::SKIP_INLINED);
304 UNREACHABLE();
305 }
306 auto vm = ManagedThread::GetCurrent()->GetVM();
307 auto arrayLength = end - begin;
308 auto array = coretypes::String::GetChars(static_cast<coretypes::String *>(obj), begin, arrayLength,
309 vm->GetLanguageContext());
310 if (UNLIKELY(array == nullptr)) {
311 HandlePendingException();
312 UNREACHABLE();
313 }
314 return array;
315 }
316
ResolveLiteralArrayEntrypoint(const Method * caller,uint32_t typeId)317 extern "C" coretypes::Array *ResolveLiteralArrayEntrypoint(const Method *caller, uint32_t typeId)
318 {
319 BEGIN_ENTRYPOINT();
320
321 auto arr = Runtime::GetCurrent()->ResolveLiteralArray(ManagedThread::GetCurrent()->GetVM(), *caller, typeId);
322 if (UNLIKELY(arr == nullptr)) {
323 HandlePendingException();
324 UNREACHABLE();
325 }
326 return arr;
327 }
328
CreateMultiArrayEntrypoint(Class * klass,uint32_t nargs,size_t * sizes)329 extern "C" coretypes::Array *CreateMultiArrayEntrypoint(Class *klass, uint32_t nargs, size_t *sizes)
330 {
331 BEGIN_ENTRYPOINT();
332
333 auto arr = CreateMultiArrayRecEntrypoint(ManagedThread::GetCurrent(), klass, nargs, sizes, 0);
334 if (UNLIKELY(arr == nullptr)) {
335 HandlePendingException();
336 UNREACHABLE();
337 }
338 return arr;
339 }
340
CreateObjectByClassEntrypoint(Class * klass)341 extern "C" ObjectHeader *CreateObjectByClassEntrypoint(Class *klass)
342 {
343 BEGIN_ENTRYPOINT();
344
345 // We need annotation here for the FullMemoryBarrier used in InitializeClassByIdEntrypoint
346 TSAN_ANNOTATE_HAPPENS_AFTER(klass);
347 if (klass->IsStringClass()) {
348 LanguageContext ctx = Runtime::GetCurrent()->GetLanguageContext(*klass);
349 auto str = coretypes::String::CreateEmptyString(ctx, Runtime::GetCurrent()->GetPandaVM());
350 if (UNLIKELY(str == nullptr)) {
351 HandlePendingException();
352 UNREACHABLE();
353 }
354 return str;
355 }
356
357 if (LIKELY(klass->IsInstantiable())) {
358 auto obj = ObjectHeader::Create(klass);
359 if (UNLIKELY(obj == nullptr)) {
360 HandlePendingException();
361 UNREACHABLE();
362 }
363 if (compiler::g_options.IsCompilerEnableTlabEvents()) {
364 EVENT_SLOWPATH_ALLOC(ManagedThread::GetCurrent()->GetId());
365 }
366 return obj;
367 }
368
369 ThrowInstantiationErrorEntrypoint(klass);
370 UNREACHABLE();
371 }
372
CloneObjectEntrypoint(ObjectHeader * obj)373 extern "C" ObjectHeader *CloneObjectEntrypoint(ObjectHeader *obj)
374 {
375 BEGIN_ENTRYPOINT();
376
377 uint32_t flags = obj->ClassAddr<Class>()->GetFlags();
378 if (UNLIKELY((flags & Class::IS_CLONEABLE) == 0)) {
379 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
380 ThrowCloneNotSupportedException();
381 HandlePendingException(UnwindPolicy::SKIP_INLINED);
382 return nullptr;
383 }
384 return ObjectHeader::Clone(obj);
385 }
386
PostBarrierWriteEntrypoint(ObjectHeader * obj,size_t size)387 extern "C" ObjectHeader *PostBarrierWriteEntrypoint(ObjectHeader *obj, size_t size)
388 {
389 LOG_ENTRYPOINT();
390 AnnotateSanitizersEntrypoint(obj, size);
391 auto *objectClass = obj->ClassAddr<Class>();
392 auto *barrierSet = ManagedThread::GetCurrent()->GetBarrierSet();
393 if (!objectClass->IsArrayClass() || !objectClass->GetComponentType()->IsPrimitive()) {
394 barrierSet->PostBarrier(obj, 0, size);
395 }
396 return obj;
397 }
398
CheckCastEntrypoint(const ObjectHeader * obj,Class * klass)399 extern "C" void CheckCastEntrypoint(const ObjectHeader *obj, Class *klass)
400 {
401 BEGIN_ENTRYPOINT();
402
403 // Don't use obj after ClassLinker call because GC can move it.
404 // Since we need only class and class in a non-movalble object
405 // it is ok to get it here.
406 Class *objKlass = obj == nullptr ? nullptr : obj->ClassAddr<Class>();
407 if (UNLIKELY(objKlass != nullptr && !klass->IsAssignableFrom(objKlass))) {
408 ark::ThrowClassCastException(klass, objKlass);
409 HandlePendingException();
410 UNREACHABLE();
411 }
412 }
413
CheckCastDeoptimizeEntrypoint(const ObjectHeader * obj,Class * klass)414 extern "C" void CheckCastDeoptimizeEntrypoint(const ObjectHeader *obj, Class *klass)
415 {
416 BEGIN_ENTRYPOINT();
417
418 ASSERT(obj != nullptr);
419 auto *objKlass = obj->ClassAddr<Class>();
420 ASSERT(objKlass != nullptr);
421 if (UNLIKELY(!klass->IsAssignableFrom(objKlass))) {
422 DeoptimizeEntrypoint(static_cast<uint64_t>(compiler::DeoptimizeType::CHECK_CAST));
423 }
424 }
425
IsInstanceEntrypoint(ObjectHeader * obj,Class * klass)426 extern "C" uint8_t IsInstanceEntrypoint(ObjectHeader *obj, Class *klass)
427 {
428 BEGIN_ENTRYPOINT();
429
430 // Don't use obj after ClassLinker call because GC can move it.
431 // Since we need only class and class in a non-movalble object
432 // it is ok to get it here.
433 Class *objKlass = obj == nullptr ? nullptr : obj->ClassAddr<Class>();
434 if (UNLIKELY(objKlass != nullptr && klass->IsAssignableFrom(objKlass))) {
435 return 1;
436 }
437 return 0;
438 }
439
SafepointEntrypoint()440 extern "C" void SafepointEntrypoint()
441 {
442 BEGIN_ENTRYPOINT();
443 interpreter::RuntimeInterface::Safepoint();
444 }
445
ResolveClassObjectEntrypoint(const Method * caller,FileEntityId typeId)446 extern "C" ObjectHeader *ResolveClassObjectEntrypoint(const Method *caller, FileEntityId typeId)
447 {
448 BEGIN_ENTRYPOINT();
449 auto klass = reinterpret_cast<Class *>(ResolveClassEntrypoint(caller, typeId));
450 return klass->GetManagedObject();
451 }
452
ResolveClassEntrypoint(const Method * caller,FileEntityId typeId)453 extern "C" Class *ResolveClassEntrypoint(const Method *caller, FileEntityId typeId)
454 {
455 BEGIN_ENTRYPOINT();
456 ClassLinker *classLinker = Runtime::GetCurrent()->GetClassLinker();
457 Class *klass = classLinker->GetClass(*caller, panda_file::File::EntityId(typeId));
458 if (UNLIKELY(klass == nullptr)) {
459 HandlePendingException();
460 UNREACHABLE();
461 }
462 return klass;
463 }
464
ResolveStringEntrypoint(const Method * caller,FileEntityId id)465 extern "C" coretypes::String *ResolveStringEntrypoint(const Method *caller, FileEntityId id)
466 {
467 BEGIN_ENTRYPOINT();
468 return Runtime::GetCurrent()->ResolveStringFromCompiledCode(ManagedThread::GetCurrent()->GetVM(), *caller,
469 panda_file::File::EntityId(id));
470 }
471
ResolveStringAotEntrypoint(const Method * caller,FileEntityId id,ObjectHeader ** slot)472 extern "C" coretypes::String *ResolveStringAotEntrypoint(const Method *caller, FileEntityId id, ObjectHeader **slot)
473 {
474 BEGIN_ENTRYPOINT();
475 auto runtime = Runtime::GetCurrent();
476 auto aotManager = runtime->GetClassLinker()->GetAotManager();
477 auto vm = ManagedThread::GetCurrent()->GetVM();
478 auto str = runtime->ResolveStringFromCompiledCode(vm, *caller, ark::panda_file::File::EntityId(id));
479 if (UNLIKELY(str == nullptr)) {
480 return nullptr;
481 }
482
483 // to many strings were saved to slots, so simply return the resolved string
484 if (aotManager->GetAotStringRootsCount() >= Runtime::GetOptions().GetAotStringGcRootsLimit()) {
485 return str;
486 }
487
488 auto counter = reinterpret_cast<std::atomic_uint32_t *>(slot);
489
490 // Atomic with acquire order reason: data race with slot with dependecies on reads after the load which should
491 // become visible
492 auto counterVal = counter->load(std::memory_order_acquire);
493 if (counterVal >= compiler::RuntimeInterface::RESOLVE_STRING_AOT_COUNTER_LIMIT - 1) {
494 return str;
495 }
496
497 if (counterVal < Runtime::GetOptions().GetResolveStringAotThreshold()) {
498 // try to update counter, but ignore result - in the worst case we'll save
499 // string's pointer to slot a bit later
500 counter->compare_exchange_strong(counterVal, counterVal + 1, std::memory_order_release,
501 std::memory_order_relaxed);
502 } else {
503 // try to replace the counter with string pointer and register the slot as GC root in case of success
504 if (counter->compare_exchange_strong(counterVal, static_cast<uint32_t>(reinterpret_cast<uint64_t>(str)),
505 std::memory_order_release, std::memory_order_relaxed)) {
506 bool isYoung = vm->GetHeapManager()->IsObjectInYoungSpace(str);
507 aotManager->RegisterAotStringRoot(slot, isYoung);
508 EVENT_AOT_RESOLVE_STRING(ConvertToString(str));
509 }
510 }
511
512 return str;
513 }
514
CreateFrameWithSize(uint32_t size,uint32_t nregs,Method * method,Frame * prev)515 extern "C" Frame *CreateFrameWithSize(uint32_t size, uint32_t nregs, Method *method, Frame *prev)
516 {
517 uint32_t extSz = EMPTY_EXT_FRAME_DATA_SIZE;
518 if (LIKELY(method)) {
519 extSz = Runtime::GetCurrent()->GetLanguageContext(*method).GetFrameExtSize();
520 }
521 size_t allocSz = Frame::GetAllocSize(size, extSz);
522 Frame *frame = Thread::GetCurrent()->GetVM()->GetHeapManager()->AllocateExtFrame(allocSz, extSz);
523 if (UNLIKELY(frame == nullptr)) {
524 return nullptr;
525 }
526 return new (frame) Frame(Frame::ToExt(frame, extSz), method, prev, nregs);
527 }
528
CreateFrameWithActualArgsAndSize(uint32_t size,uint32_t nregs,uint32_t numActualArgs,Method * method,Frame * prev)529 extern "C" Frame *CreateFrameWithActualArgsAndSize(uint32_t size, uint32_t nregs, uint32_t numActualArgs,
530 Method *method, Frame *prev)
531 {
532 auto *thread = ManagedThread::GetCurrent();
533 uint32_t extSz = thread->GetVM()->GetFrameExtSize();
534 size_t allocSz = Frame::GetAllocSize(size, extSz);
535 void *mem = thread->GetStackFrameAllocator()->Alloc(allocSz);
536 if (UNLIKELY(mem == nullptr)) {
537 return nullptr;
538 }
539 return new (Frame::FromExt(mem, extSz)) Frame(mem, method, prev, nregs, numActualArgs);
540 }
541
CreateNativeFrameWithActualArgsAndSize(uint32_t size,uint32_t nregs,uint32_t numActualArgs,Method * method,Frame * prev)542 extern "C" Frame *CreateNativeFrameWithActualArgsAndSize(uint32_t size, uint32_t nregs, uint32_t numActualArgs,
543 Method *method, Frame *prev)
544 {
545 uint32_t extSz = EMPTY_EXT_FRAME_DATA_SIZE;
546 size_t allocSz = Frame::GetAllocSize(size, extSz);
547 void *mem = ManagedThread::GetCurrent()->GetStackFrameAllocator()->Alloc(allocSz);
548 if (UNLIKELY(mem == nullptr)) {
549 return nullptr;
550 }
551 return new (Frame::FromExt(mem, extSz)) Frame(mem, method, prev, nregs, numActualArgs);
552 }
553
554 template <bool IS_DYNAMIC = false>
CreateFrame(uint32_t nregs,Method * method,Frame * prev)555 static Frame *CreateFrame(uint32_t nregs, Method *method, Frame *prev)
556 {
557 return CreateFrameWithSize(Frame::GetActualSize<IS_DYNAMIC>(nregs), nregs, method, prev);
558 }
559
560 template <bool IS_DYNAMIC>
CreateFrameWithActualArgs(uint32_t nregs,uint32_t numActualArgs,Method * method,Frame * prev)561 static Frame *CreateFrameWithActualArgs(uint32_t nregs, uint32_t numActualArgs, Method *method, Frame *prev)
562 {
563 auto frame =
564 CreateFrameWithActualArgsAndSize(Frame::GetActualSize<IS_DYNAMIC>(nregs), nregs, numActualArgs, method, prev);
565 if (IS_DYNAMIC) {
566 LanguageContext ctx = Runtime::GetCurrent()->GetLanguageContext(*method);
567 coretypes::TaggedValue initialValue = ctx.GetInitialTaggedValue();
568 for (size_t i = 0; i < nregs; i++) {
569 frame->GetVReg(i).SetValue(initialValue.GetRawData());
570 }
571 frame->SetDynamic();
572 }
573 return frame;
574 }
575
CreateFrameForMethod(Method * method,Frame * prev)576 extern "C" Frame *CreateFrameForMethod(Method *method, Frame *prev)
577 {
578 auto nregs = method->GetNumArgs() + method->GetNumVregs();
579 return CreateFrame<false>(nregs, method, prev);
580 }
581
CreateFrameForMethodDyn(Method * method,Frame * prev)582 extern "C" Frame *CreateFrameForMethodDyn(Method *method, Frame *prev)
583 {
584 auto nregs = method->GetNumArgs() + method->GetNumVregs();
585 return CreateFrame<true>(nregs, method, prev);
586 }
587
CreateFrameForMethodWithActualArgs(uint32_t numActualArgs,Method * method,Frame * prev)588 extern "C" Frame *CreateFrameForMethodWithActualArgs(uint32_t numActualArgs, Method *method, Frame *prev)
589 {
590 auto nargs = std::max(numActualArgs, method->GetNumArgs());
591 auto nregs = nargs + method->GetNumVregs();
592 return CreateFrameWithActualArgs<false>(nregs, numActualArgs, method, prev);
593 }
594
CreateFrameForMethodWithActualArgsDyn(uint32_t numActualArgs,Method * method,Frame * prev)595 extern "C" Frame *CreateFrameForMethodWithActualArgsDyn(uint32_t numActualArgs, Method *method, Frame *prev)
596 {
597 auto nargs = std::max(numActualArgs, method->GetNumArgs());
598 auto nregs = nargs + method->GetNumVregs();
599 return CreateFrameWithActualArgs<true>(nregs, numActualArgs, method, prev);
600 }
601
FreeFrame(Frame * frame)602 extern "C" void FreeFrame(Frame *frame)
603 {
604 ASSERT(frame->GetExt() != nullptr);
605 ManagedThread::GetCurrent()->GetStackFrameAllocator()->Free(frame->GetExt());
606 }
607
GetStaticFieldAddressEntrypoint(Method * method,uint32_t fieldId)608 extern "C" uintptr_t GetStaticFieldAddressEntrypoint(Method *method, uint32_t fieldId)
609 {
610 BEGIN_ENTRYPOINT();
611 auto *classLinker = Runtime::GetCurrent()->GetClassLinker();
612 auto field = classLinker->GetField(*method, panda_file::File::EntityId(fieldId));
613 if (UNLIKELY(field == nullptr)) {
614 HandlePendingException();
615 UNREACHABLE();
616 }
617 auto *klass = field->GetClass();
618 ASSERT(klass != nullptr);
619 return reinterpret_cast<uintptr_t>(klass) + field->GetOffset();
620 }
621
UnresolvedStoreStaticBarrieredEntrypoint(Method * method,uint32_t fieldId,ObjectHeader * obj)622 extern "C" void UnresolvedStoreStaticBarrieredEntrypoint(Method *method, uint32_t fieldId, ObjectHeader *obj)
623 {
624 BEGIN_ENTRYPOINT();
625 auto thread = ManagedThread::GetCurrent();
626 // One must not use plain ObjectHeader pointer here as GC can move obj.
627 // Wrap it in VMHandle.
628 [[maybe_unused]] HandleScope<ObjectHeader *> objHandleScope(thread);
629 VMHandle<ObjectHeader> objHandle(thread, obj);
630
631 auto field = Runtime::GetCurrent()->GetClassLinker()->GetField(*method, panda_file::File::EntityId(fieldId));
632 if (UNLIKELY(field == nullptr)) {
633 HandlePendingException();
634 UNREACHABLE();
635 }
636 if (UNLIKELY(!field->GetType().IsReference())) {
637 HandlePendingException();
638 UNREACHABLE();
639 }
640
641 auto *classPtr = field->GetClass();
642 ASSERT(classPtr != nullptr);
643 InitializeClassEntrypoint(classPtr);
644
645 // field_addr = class_ptr + field's offset
646 auto fieldAddr = reinterpret_cast<uint32_t *>(reinterpret_cast<uintptr_t>(classPtr) + field->GetOffset());
647 auto *barrierSet = thread->GetBarrierSet();
648 // Pre-barrier
649 if (barrierSet->IsPreBarrierEnabled()) {
650 // Load
651 // Atomic with acquire order reason: assume load can be volatile
652 auto currentReference = __atomic_load_n(fieldAddr, std::memory_order_acquire);
653 barrierSet->PreBarrier(reinterpret_cast<void *>(currentReference));
654 }
655 // Store
656 // Atomic with release order reason: assume store can be volatile
657 auto ref = reinterpret_cast<uintptr_t>(objHandle.GetPtr());
658 ASSERT(sizeof(ref) == 4U || (ref & 0xFFFFFFFF00000000UL) == 0);
659 __atomic_store_n(fieldAddr, static_cast<uint32_t>(ref), std::memory_order_release);
660 // Post-barrier
661 if (!mem::IsEmptyBarrier(barrierSet->GetPostType())) {
662 auto objPtrPtr = reinterpret_cast<uintptr_t>(classPtr) + Class::GetManagedObjectOffset();
663 auto objPtr = *reinterpret_cast<uint32_t *>(objPtrPtr);
664 barrierSet->PostBarrier(reinterpret_cast<void *>(objPtr), 0, reinterpret_cast<void *>(ref));
665 }
666 }
667
GetUnknownStaticFieldMemoryAddressEntrypoint(Method * method,uint32_t fieldId,size_t * slot)668 extern "C" uintptr_t GetUnknownStaticFieldMemoryAddressEntrypoint(Method *method, uint32_t fieldId, size_t *slot)
669 {
670 BEGIN_ENTRYPOINT();
671 auto *classLinker = Runtime::GetCurrent()->GetClassLinker();
672 auto field = classLinker->GetField(*method, panda_file::File::EntityId(fieldId));
673 if (UNLIKELY(field == nullptr)) {
674 HandlePendingException();
675 UNREACHABLE();
676 }
677 auto *klass = field->GetClass();
678 ASSERT(klass != nullptr);
679 InitializeClassEntrypoint(klass);
680
681 uintptr_t addr = reinterpret_cast<uintptr_t>(klass) + field->GetOffset();
682 if (klass->IsInitialized() && slot != nullptr) {
683 *slot = addr;
684 }
685 return addr;
686 }
687
GetFieldOffsetEntrypoint(Method * method,uint32_t fieldId)688 extern "C" size_t GetFieldOffsetEntrypoint(Method *method, uint32_t fieldId)
689 {
690 BEGIN_ENTRYPOINT();
691 auto *classLinker = Runtime::GetCurrent()->GetClassLinker();
692 auto field = classLinker->GetField(*method, panda_file::File::EntityId(fieldId));
693 if (UNLIKELY(field == nullptr)) {
694 HandlePendingException();
695 UNREACHABLE();
696 }
697 return field->GetOffset();
698 }
699
InitializeClassEntrypoint(Class * klass)700 extern "C" void InitializeClassEntrypoint(Class *klass)
701 {
702 BEGIN_ENTRYPOINT();
703 auto *classLinker = Runtime::GetCurrent()->GetClassLinker();
704 if (!klass->IsInitialized() && !classLinker->InitializeClass(ManagedThread::GetCurrent(), klass)) {
705 HandlePendingException();
706 UNREACHABLE();
707 }
708 }
709
InitializeClassByIdEntrypoint(const Method * caller,FileEntityId id)710 extern "C" Class *InitializeClassByIdEntrypoint(const Method *caller, FileEntityId id)
711 {
712 BEGIN_ENTRYPOINT();
713 ClassLinker *classLinker = Runtime::GetCurrent()->GetClassLinker();
714 Class *klass = classLinker->GetClass(*caller, panda_file::File::EntityId(id));
715 if (UNLIKELY(klass == nullptr)) {
716 HandlePendingException();
717 UNREACHABLE();
718 }
719 // Later we store klass pointer into .aot_got section.
720 // Without full memory barrier on the architectures with weak memory order
721 // we can read klass pointer, but fetch klass data before it's set in GetClass and InitializeClass
722 arch::FullMemoryBarrier();
723 // Full barrier is not visible by TSAN so we need annotation here
724 TSAN_ANNOTATE_HAPPENS_BEFORE(klass);
725 InitializeClassEntrypoint(klass);
726 return klass;
727 }
728
ResolveVirtualCallEntrypoint(const Method * callee,ObjectHeader * obj)729 extern "C" uintptr_t NO_ADDRESS_SANITIZE ResolveVirtualCallEntrypoint(const Method *callee, ObjectHeader *obj)
730 {
731 BEGIN_ENTRYPOINT();
732 if (UNLIKELY(callee == nullptr)) {
733 HandlePendingException();
734 UNREACHABLE();
735 }
736 auto *resolved = obj->ClassAddr<Class>()->ResolveVirtualMethod(callee);
737 ASSERT(resolved != nullptr);
738
739 return reinterpret_cast<uintptr_t>(resolved);
740 }
741
ResolveVirtualCallAotEntrypoint(const Method * caller,ObjectHeader * obj,size_t calleeId,uintptr_t cacheAddr)742 extern "C" uintptr_t NO_ADDRESS_SANITIZE ResolveVirtualCallAotEntrypoint(const Method *caller, ObjectHeader *obj,
743 size_t calleeId,
744 [[maybe_unused]] uintptr_t cacheAddr)
745 {
746 BEGIN_ENTRYPOINT();
747 // Don't use obj after ClassLinker call because GC can move it.
748 // Since we need only class and class in a non-movalble object
749 // it is ok to get it here.
750 auto *objKlass = obj->ClassAddr<Class>();
751 Method *method = Runtime::GetCurrent()->GetClassLinker()->GetMethod(*caller, panda_file::File::EntityId(calleeId));
752 if (UNLIKELY(method == nullptr)) {
753 HandlePendingException();
754 UNREACHABLE();
755 }
756 auto *resolved = objKlass->ResolveVirtualMethod(method);
757 ASSERT(resolved != nullptr);
758
759 #if defined(PANDA_TARGET_ARM64)
760 // In arm64, use interface inlineCache
761 // NOTE(liyiming): will support x86_64 in future
762 // issue #7018
763 auto methodHead = objKlass->GetRawFirstMethodAddr();
764 if (cacheAddr == 0 || methodHead == nullptr) {
765 return reinterpret_cast<uintptr_t>(resolved);
766 }
767
768 constexpr uint32_t METHOD_COMPRESS = 3;
769 constexpr uint32_t CACHE_OFFSET_32 = 32;
770 constexpr uint32_t CACHE_OFFSET_34 = 34;
771 auto cache = reinterpret_cast<int64_t *>(cacheAddr);
772 auto methodResolved = reinterpret_cast<int64_t>(resolved);
773 int64_t methodCache = methodResolved - reinterpret_cast<int64_t>(methodHead);
774
775 int64_t methodCacheJudge = methodCache >> CACHE_OFFSET_34; // NOLINT(hicpp-signed-bitwise)
776 if (methodCacheJudge != 0 && methodCacheJudge != -1) {
777 return reinterpret_cast<uintptr_t>(resolved);
778 }
779 methodCache = methodCache >> METHOD_COMPRESS; // NOLINT(hicpp-signed-bitwise)
780 methodCache = methodCache << CACHE_OFFSET_32; // NOLINT(hicpp-signed-bitwise)
781 int64_t saveCache = methodCache | reinterpret_cast<int64_t>(objKlass); // NOLINT(hicpp-signed-bitwise)
782 *cache = saveCache;
783 #endif
784 return reinterpret_cast<uintptr_t>(resolved);
785 }
786
ResolveUnknownVirtualCallEntrypoint(const Method * caller,ObjectHeader * obj,size_t calleeId,size_t * slot)787 extern "C" uintptr_t NO_ADDRESS_SANITIZE ResolveUnknownVirtualCallEntrypoint(const Method *caller, ObjectHeader *obj,
788 size_t calleeId, size_t *slot)
789 {
790 {
791 auto thread = ManagedThread::GetCurrent();
792 [[maybe_unused]] HandleScope<ObjectHeader *> scope(thread);
793 VMHandle<ObjectHeader> handleObj(thread, obj);
794
795 BEGIN_ENTRYPOINT();
796 auto runtime = Runtime::GetCurrent();
797 Method *method = runtime->GetClassLinker()->GetMethod(*caller, panda_file::File::EntityId(calleeId));
798 if (LIKELY(method != nullptr)) {
799 // Cache a method index in vtable
800 if (slot != nullptr && (!method->GetClass()->IsInterface() || method->IsDefaultInterfaceMethod())) {
801 // We save 'vtable index + 1' because 0 value means uninitialized.
802 // Codegen must subtract index after loading from the slot.
803 *slot = method->GetVTableIndex() + 1;
804 }
805
806 auto *resolved = handleObj.GetPtr()->ClassAddr<Class>()->ResolveVirtualMethod(method);
807 ASSERT(resolved != nullptr);
808
809 return reinterpret_cast<uintptr_t>(resolved);
810 }
811 }
812
813 HandlePendingException();
814 UNREACHABLE();
815 }
816
CheckStoreArrayReferenceEntrypoint(coretypes::Array * array,ObjectHeader * storeObj)817 extern "C" void CheckStoreArrayReferenceEntrypoint(coretypes::Array *array, ObjectHeader *storeObj)
818 {
819 BEGIN_ENTRYPOINT();
820 ASSERT(array != nullptr);
821 ASSERT(storeObj != nullptr);
822
823 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
824 auto *arrayClass = array->ClassAddr<Class>();
825 auto *elementClass = arrayClass->GetComponentType();
826 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
827 if (UNLIKELY(!storeObj->IsInstanceOf(elementClass))) {
828 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
829 ark::ThrowArrayStoreException(arrayClass, storeObj->ClassAddr<Class>());
830 HandlePendingException();
831 UNREACHABLE();
832 }
833 }
834
CheckStoreArrayReferenceDeoptimizeEntrypoint(coretypes::Array * array,ObjectHeader * storeObj)835 extern "C" void CheckStoreArrayReferenceDeoptimizeEntrypoint(coretypes::Array *array, ObjectHeader *storeObj)
836 {
837 BEGIN_ENTRYPOINT();
838 ASSERT(array != nullptr);
839 ASSERT(storeObj != nullptr);
840
841 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
842 auto *arrayClass = array->ClassAddr<Class>();
843 auto *elementClass = arrayClass->GetComponentType();
844 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
845 if (UNLIKELY(!storeObj->IsInstanceOf(elementClass))) {
846 DeoptimizeEntrypoint(static_cast<uint64_t>(compiler::DeoptimizeType::CHECK_CAST));
847 }
848 }
849
GetCalleeMethodEntrypoint(const Method * caller,size_t calleeId)850 extern "C" Method *GetCalleeMethodEntrypoint(const Method *caller, size_t calleeId)
851 {
852 BEGIN_ENTRYPOINT();
853 auto *method = Runtime::GetCurrent()->GetClassLinker()->GetMethod(*caller, panda_file::File::EntityId(calleeId));
854 if (UNLIKELY(method == nullptr)) {
855 HandlePendingException();
856 UNREACHABLE();
857 }
858
859 return method;
860 }
861
GetUnknownCalleeMethodEntrypoint(const Method * caller,size_t calleeId,size_t * slot)862 extern "C" Method *GetUnknownCalleeMethodEntrypoint(const Method *caller, size_t calleeId, size_t *slot)
863 {
864 BEGIN_ENTRYPOINT();
865 auto *method = Runtime::GetCurrent()->GetClassLinker()->GetMethod(*caller, panda_file::File::EntityId(calleeId));
866 if (UNLIKELY(method == nullptr)) {
867 HandlePendingException();
868 UNREACHABLE();
869 }
870 auto klass = method->GetClass();
871 ASSERT(klass != nullptr);
872 InitializeClassEntrypoint(klass);
873
874 if (klass->IsInitialized() && slot != nullptr) {
875 *slot = reinterpret_cast<size_t>(method);
876 }
877
878 return method;
879 }
880
ThrowExceptionEntrypoint(ObjectHeader * exception)881 extern "C" NO_ADDRESS_SANITIZE void ThrowExceptionEntrypoint(ObjectHeader *exception)
882 {
883 BEGIN_ENTRYPOINT();
884 LOG(DEBUG, INTEROP) << "ThrowExceptionEntrypoint \n";
885 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
886 if (exception == nullptr) {
887 NullPointerExceptionEntrypoint();
888 UNREACHABLE();
889 }
890 ManagedThread::GetCurrent()->SetException(exception);
891
892 SetExceptionEvent(events::ExceptionType::THROW, ManagedThread::GetCurrent());
893 HandlePendingException(UnwindPolicy::SKIP_INLINED);
894 }
895
ThrowNativeExceptionEntrypoint()896 extern "C" NO_ADDRESS_SANITIZE void ThrowNativeExceptionEntrypoint()
897 {
898 BEGIN_ENTRYPOINT();
899 LOG(DEBUG, INTEROP) << "ThrowNativeExceptionEntrypoint \n";
900 SetExceptionEvent(events::ExceptionType::NATIVE, ManagedThread::GetCurrent());
901 HandlePendingException(UnwindPolicy::SKIP_INLINED);
902 }
903
ArrayIndexOutOfBoundsExceptionEntrypoint(ssize_t idx,size_t length)904 extern "C" NO_ADDRESS_SANITIZE void ArrayIndexOutOfBoundsExceptionEntrypoint([[maybe_unused]] ssize_t idx,
905 [[maybe_unused]] size_t length)
906 {
907 BEGIN_ENTRYPOINT();
908 LOG(DEBUG, INTEROP) << "ArrayIndexOutOfBoundsExceptionEntrypoint \n";
909 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
910 ThrowArrayIndexOutOfBoundsException(idx, length);
911 HandlePendingException(UnwindPolicy::SKIP_INLINED);
912 }
913
StringIndexOutOfBoundsExceptionEntrypoint(ssize_t idx,size_t length)914 extern "C" NO_ADDRESS_SANITIZE void StringIndexOutOfBoundsExceptionEntrypoint([[maybe_unused]] ssize_t idx,
915 [[maybe_unused]] size_t length)
916 {
917 BEGIN_ENTRYPOINT();
918 LOG(DEBUG, INTEROP) << "StringIndexOutOfBoundsExceptionEntrypoint \n";
919 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
920 ThrowStringIndexOutOfBoundsException(idx, length);
921 HandlePendingException(UnwindPolicy::SKIP_INLINED);
922 }
923
NullPointerExceptionEntrypoint()924 extern "C" NO_ADDRESS_SANITIZE void NullPointerExceptionEntrypoint()
925 {
926 BEGIN_ENTRYPOINT();
927 LOG(DEBUG, INTEROP) << "NullPointerExceptionEntrypoint \n";
928 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
929 ThrowNullPointerException();
930 HandlePendingException(UnwindPolicy::SKIP_INLINED);
931 }
932
AbstractMethodErrorEntrypoint(Method * method)933 extern "C" NO_ADDRESS_SANITIZE void AbstractMethodErrorEntrypoint(Method *method)
934 {
935 BEGIN_ENTRYPOINT();
936 LOG(DEBUG, INTEROP) << "AbstractMethodErrorEntrypoint \n";
937 ManagedThread *thread = ManagedThread::GetCurrent();
938 ASSERT(!thread->HasPendingException());
939 auto stack = StackWalker::Create(thread, UnwindPolicy::SKIP_INLINED);
940 ThrowAbstractMethodError(method);
941 ASSERT(thread->HasPendingException());
942 if (stack.IsCFrame()) {
943 FindCatchBlockInCFrames(thread, &stack, nullptr);
944 }
945 }
946
ArithmeticExceptionEntrypoint()947 extern "C" NO_ADDRESS_SANITIZE void ArithmeticExceptionEntrypoint()
948 {
949 BEGIN_ENTRYPOINT();
950 LOG(DEBUG, INTEROP) << "ArithmeticExceptionEntrypoint \n";
951 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
952 ThrowArithmeticException();
953 HandlePendingException(UnwindPolicy::SKIP_INLINED);
954 }
955
NegativeArraySizeExceptionEntrypoint(ssize_t size)956 extern "C" NO_ADDRESS_SANITIZE void NegativeArraySizeExceptionEntrypoint(ssize_t size)
957 {
958 BEGIN_ENTRYPOINT();
959 LOG(DEBUG, INTEROP) << "NegativeArraySizeExceptionEntrypoint \n";
960 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
961 ThrowNegativeArraySizeException(size);
962 HandlePendingException(UnwindPolicy::SKIP_INLINED);
963 }
964
ClassCastExceptionEntrypoint(Class * instClass,ObjectHeader * srcObj)965 extern "C" NO_ADDRESS_SANITIZE void ClassCastExceptionEntrypoint(Class *instClass, ObjectHeader *srcObj)
966 {
967 BEGIN_ENTRYPOINT();
968 LOG(DEBUG, INTEROP) << "ClassCastExceptionEntrypoint \n";
969 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
970 ASSERT(srcObj != nullptr);
971 ThrowClassCastException(instClass, srcObj->ClassAddr<Class>());
972 HandlePendingException(UnwindPolicy::SKIP_INLINED);
973 }
974
StackOverflowExceptionEntrypoint()975 extern "C" NO_ADDRESS_SANITIZE void StackOverflowExceptionEntrypoint()
976 {
977 // WARNING: We should not add any heavy code constructions here, like events or other debug/testing stuff,
978 // because we have small stack here, see ManagedThread::STACK_OVERFLOW_RESERVED_SIZE.
979 auto thread = ManagedThread::GetCurrent();
980
981 ASSERT(!thread->HasPendingException());
982 thread->DisableStackOverflowCheck();
983 ThrowStackOverflowException(thread);
984 thread->EnableStackOverflowCheck();
985 HandlePendingException(UnwindPolicy::SKIP_INLINED);
986 }
987
DeoptimizeEntrypoint(uint64_t deoptimizeType)988 extern "C" NO_ADDRESS_SANITIZE void DeoptimizeEntrypoint(uint64_t deoptimizeType)
989 {
990 BEGIN_ENTRYPOINT();
991 auto thread = ManagedThread::GetCurrent();
992 auto type = static_cast<ark::compiler::DeoptimizeType>(
993 deoptimizeType & ((1U << MinimumBitsToStore(ark::compiler::DeoptimizeType::COUNT)) - 1));
994 [[maybe_unused]] auto instId = deoptimizeType >> MinimumBitsToStore(ark::compiler::DeoptimizeType::COUNT);
995 LOG(INFO, INTEROP) << "DeoptimizeEntrypoint (reason: " << ark::compiler::DeoptimizeTypeToString(type)
996 << ", inst_id: " << instId << ")\n";
997
998 EVENT_DEOPTIMIZATION_REASON(std::string(StackWalker::Create(thread).GetMethod()->GetFullName()),
999 ark::compiler::DeoptimizeTypeToString(type));
1000
1001 ASSERT(!thread->HasPendingException());
1002 auto stack = StackWalker::Create(thread);
1003 Method *destroyMethod = nullptr;
1004 if (type >= ark::compiler::DeoptimizeType::CAUSE_METHOD_DESTRUCTION) {
1005 // Get pointer to top method
1006 destroyMethod = StackWalker::Create(thread, UnwindPolicy::SKIP_INLINED).GetMethod();
1007 }
1008 Deoptimize(&stack, nullptr, false, destroyMethod);
1009 }
1010
ThrowInstantiationErrorEntrypoint(Class * klass)1011 extern "C" NO_ADDRESS_SANITIZE void ThrowInstantiationErrorEntrypoint(Class *klass)
1012 {
1013 BEGIN_ENTRYPOINT();
1014 LOG(DEBUG, INTEROP) << "ThrowInstantiationErrorEntrypoint \n";
1015 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
1016 const auto &name = klass->GetName();
1017 PandaString pname(name.cbegin(), name.cend());
1018 ThrowInstantiationError(pname);
1019 HandlePendingException(UnwindPolicy::SKIP_INLINED);
1020 }
1021
GetInitialTaggedValue(Method * method)1022 coretypes::TaggedValue GetInitialTaggedValue(Method *method)
1023 {
1024 BEGIN_ENTRYPOINT();
1025 return Runtime::GetCurrent()->GetLanguageContext(*method).GetInitialTaggedValue();
1026 }
1027
LockObjectEntrypoint(ObjectHeader * obj)1028 extern "C" void LockObjectEntrypoint(ObjectHeader *obj)
1029 {
1030 BEGIN_ENTRYPOINT();
1031 ark::intrinsics::ObjectMonitorEnter(obj);
1032 }
1033
LockObjectSlowPathEntrypoint(ObjectHeader * obj)1034 extern "C" void LockObjectSlowPathEntrypoint(ObjectHeader *obj)
1035 {
1036 BEGIN_ENTRYPOINT();
1037 ark::intrinsics::ObjectMonitorEnter(obj);
1038 if (!ManagedThread::GetCurrent()->HasPendingException()) {
1039 return;
1040 }
1041 LOG(DEBUG, INTEROP) << "ThrowNativeExceptionEntrypoint after LockObject \n";
1042 SetExceptionEvent(events::ExceptionType::NATIVE, ManagedThread::GetCurrent());
1043 HandlePendingException(UnwindPolicy::SKIP_INLINED);
1044 }
1045
UnlockObjectEntrypoint(ObjectHeader * obj)1046 extern "C" void UnlockObjectEntrypoint(ObjectHeader *obj)
1047 {
1048 BEGIN_ENTRYPOINT();
1049 ark::intrinsics::ObjectMonitorExit(obj);
1050 }
1051
UnlockObjectSlowPathEntrypoint(ObjectHeader * obj)1052 extern "C" void UnlockObjectSlowPathEntrypoint(ObjectHeader *obj)
1053 {
1054 BEGIN_ENTRYPOINT();
1055 ark::intrinsics::ObjectMonitorExit(obj);
1056 if (!ManagedThread::GetCurrent()->HasPendingException()) {
1057 return;
1058 }
1059 LOG(DEBUG, INTEROP) << "ThrowNativeExceptionEntrypoint after UnlockObject \n";
1060 SetExceptionEvent(events::ExceptionType::NATIVE, ManagedThread::GetCurrent());
1061 HandlePendingException(UnwindPolicy::SKIP_INLINED);
1062 }
1063
IncompatibleClassChangeErrorForMethodConflictEntrypoint(Method * method)1064 extern "C" NO_ADDRESS_SANITIZE void IncompatibleClassChangeErrorForMethodConflictEntrypoint(Method *method)
1065 {
1066 BEGIN_ENTRYPOINT();
1067 LOG(DEBUG, INTEROP) << "IncompatibleClassChangeErrorForMethodConflictEntrypoint \n";
1068 ManagedThread *thread = ManagedThread::GetCurrent();
1069 ASSERT(!thread->HasPendingException());
1070 auto stack = StackWalker::Create(thread, UnwindPolicy::SKIP_INLINED);
1071 ThrowIncompatibleClassChangeErrorForMethodConflict(method);
1072 ASSERT(thread->HasPendingException());
1073 if (stack.IsCFrame()) {
1074 FindCatchBlockInCFrames(thread, &stack, nullptr);
1075 }
1076 }
1077
1078 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
TraceEntrypoint(size_t pid,...)1079 extern "C" void TraceEntrypoint(size_t pid, ...)
1080 {
1081 LOG_ENTRYPOINT();
1082 auto id = static_cast<TraceId>(pid);
1083
1084 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1085 va_list args;
1086 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1087 va_start(args, pid);
1088 switch (id) {
1089 case TraceId::METHOD_ENTER: {
1090 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,clang-analyzer-valist.Uninitialized)
1091 [[maybe_unused]] auto method = va_arg(args, const Method *);
1092 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1093 [[maybe_unused]] auto kind = va_arg(args, events::MethodEnterKind);
1094 EVENT_METHOD_ENTER(method->GetFullName(), kind, ManagedThread::GetCurrent()->RecordMethodEnter());
1095 break;
1096 }
1097 case TraceId::METHOD_EXIT: {
1098 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,clang-analyzer-valist.Uninitialized)
1099 [[maybe_unused]] auto method = va_arg(args, const Method *);
1100 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1101 [[maybe_unused]] auto kind = va_arg(args, events::MethodExitKind);
1102 EVENT_METHOD_EXIT(method->GetFullName(), kind, ManagedThread::GetCurrent()->RecordMethodExit());
1103 break;
1104 }
1105 case TraceId::PRINT_ARG: {
1106 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,clang-analyzer-valist.Uninitialized)
1107 size_t argsNum = va_arg(args, size_t);
1108 std::cerr << "[TRACE ARGS] ";
1109 for (size_t i = 0; i < argsNum; i++) {
1110 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1111 std::cerr << i << "=" << va_arg(args, void *) << " ";
1112 }
1113 std::cerr << std::endl;
1114
1115 break;
1116 }
1117 default: {
1118 break;
1119 }
1120 }
1121 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1122 va_end(args);
1123 }
1124
LogEntrypoint(const char * fmt,...)1125 extern "C" void LogEntrypoint(const char *fmt, ...)
1126 {
1127 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1128 va_list args;
1129 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1130 va_start(args, fmt);
1131 // NOLINTNEXTLINE(clang-analyzer-valist.Uninitialized)
1132 vfprintf(stderr, fmt, args);
1133 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1134 va_end(args);
1135 }
1136
1137 // Irtoc Interpreter Entrypoints
1138
FreeFrameInterp(Frame * frame,ManagedThread * current)1139 extern "C" void FreeFrameInterp(Frame *frame, ManagedThread *current)
1140 {
1141 ASSERT(frame->GetExt() != nullptr);
1142 current->GetStackFrameAllocator()->Free(frame->GetExt());
1143 }
1144
AllocFrameInterp(ManagedThread * current,size_t allocSz)1145 extern "C" void *AllocFrameInterp(ManagedThread *current, size_t allocSz)
1146 {
1147 void *mem = current->GetStackFrameAllocator()->Alloc<false>(allocSz);
1148 if (UNLIKELY(mem == nullptr)) {
1149 current->DisableStackOverflowCheck();
1150 ark::ThrowStackOverflowException(current);
1151 current->EnableStackOverflowCheck();
1152 }
1153 return mem;
1154 }
1155
InitializeFrame(void * mem,Method * method,Frame * prev,size_t nregs)1156 extern "C" Frame *InitializeFrame(void *mem, Method *method, Frame *prev, size_t nregs)
1157 {
1158 return new (Frame::FromExt(mem, CORE_EXT_FRAME_DATA_SIZE)) Frame(mem, method, prev, nregs);
1159 }
1160
SafepointEntrypointInterp(ManagedThread * thread)1161 extern "C" void SafepointEntrypointInterp(ManagedThread *thread)
1162 {
1163 BEGIN_ENTRYPOINT();
1164 interpreter::RuntimeInterface::Safepoint(thread);
1165 }
1166
IsCompiled(const void * entrypoint)1167 extern "C" int IsCompiled(const void *entrypoint)
1168 {
1169 return static_cast<int>(entrypoint != reinterpret_cast<const void *>(CompiledCodeToInterpreterBridge));
1170 }
1171
GetClassIdEntrypoint(const Method * caller,uint32_t classId)1172 extern "C" size_t GetClassIdEntrypoint(const Method *caller, uint32_t classId)
1173 {
1174 auto resolvedId = caller->GetClass()->ResolveClassIndex(BytecodeId(classId).AsIndex());
1175 return resolvedId.GetOffset();
1176 }
1177
CreateArrayByIdEntrypoint(ManagedThread * thread,const Method * caller,uint32_t classId,int32_t length)1178 extern "C" coretypes::Array *CreateArrayByIdEntrypoint(ManagedThread *thread, const Method *caller, uint32_t classId,
1179 int32_t length)
1180 {
1181 CHECK_STACK_WALKER;
1182 auto *klass = interpreter::RuntimeInterface::ResolveClass<true>(thread, *caller, BytecodeId(classId));
1183 if (UNLIKELY(klass == nullptr)) {
1184 return nullptr;
1185 }
1186 return interpreter::RuntimeInterface::CreateArray(klass, length);
1187 }
1188
1189 template <BytecodeInstruction::Format FORMAT>
CreateMultiDimArray(ManagedThread * thread,Frame * frame,Class * klass,Method * caller,uint32_t methodId,const uint8_t * pc)1190 static coretypes::Array *CreateMultiDimArray(ManagedThread *thread, Frame *frame, Class *klass, Method *caller,
1191 uint32_t methodId, const uint8_t *pc)
1192 {
1193 interpreter::DimIterator<FORMAT> dimIter {BytecodeInstruction(pc), frame};
1194 auto nargs = interpreter::RuntimeInterface::GetMethodArgumentsCount(caller, BytecodeId(methodId));
1195 auto obj =
1196 coretypes::Array::CreateMultiDimensionalArray<interpreter::DimIterator<FORMAT>>(thread, klass, nargs, dimIter);
1197 return obj;
1198 }
1199
CreateMultiDimensionalArrayById(ManagedThread * thread,Frame * frame,Class * klass,Method * caller,uint32_t methodId,const uint8_t * pc,int32_t formatIdx)1200 extern "C" coretypes::Array *CreateMultiDimensionalArrayById(ManagedThread *thread, Frame *frame, Class *klass,
1201 Method *caller, uint32_t methodId, const uint8_t *pc,
1202 int32_t formatIdx)
1203 {
1204 switch (formatIdx) {
1205 case 0U:
1206 return CreateMultiDimArray<BytecodeInstruction::Format::V4_V4_ID16>(thread, frame, klass, caller, methodId,
1207 pc);
1208 case 1U:
1209 return CreateMultiDimArray<BytecodeInstruction::Format::V4_V4_V4_V4_ID16>(thread, frame, klass, caller,
1210 methodId, pc);
1211 case 2U:
1212 return CreateMultiDimArray<BytecodeInstruction::Format::V8_ID16>(thread, frame, klass, caller, methodId,
1213 pc);
1214 default:
1215 UNREACHABLE();
1216 }
1217 return nullptr;
1218 }
1219
CreateObjectByClassInterpreter(ManagedThread * thread,Class * klass)1220 extern "C" ObjectHeader *CreateObjectByClassInterpreter(ManagedThread *thread, Class *klass)
1221 {
1222 ASSERT(klass != nullptr);
1223 ASSERT(!klass->IsArrayClass());
1224
1225 if (!klass->IsInitialized()) {
1226 auto *classLinker = Runtime::GetCurrent()->GetClassLinker();
1227 if (!classLinker->InitializeClass(thread, klass)) {
1228 return nullptr;
1229 }
1230 }
1231
1232 return interpreter::RuntimeInterface::CreateObject(thread, klass);
1233 }
1234
CheckCastByIdEntrypoint(ObjectHeader * obj,Class * klass)1235 extern "C" uint32_t CheckCastByIdEntrypoint(ObjectHeader *obj, Class *klass)
1236 {
1237 CHECK_STACK_WALKER;
1238 ASSERT(IsAddressInObjectsHeapOrNull(obj));
1239 Class *objKlass = obj == nullptr ? nullptr : obj->ClassAddr<Class>();
1240 if (UNLIKELY(objKlass != nullptr && !klass->IsAssignableFrom(objKlass))) {
1241 ark::ThrowClassCastException(klass, objKlass);
1242 return 1;
1243 }
1244 return 0;
1245 }
1246
IsInstanceByIdEntrypoint(ObjectHeader * obj,Class * klass)1247 extern "C" uint32_t IsInstanceByIdEntrypoint(ObjectHeader *obj, Class *klass)
1248 {
1249 CHECK_STACK_WALKER;
1250 ASSERT(IsAddressInObjectsHeapOrNull(obj));
1251
1252 return IsInstanceEntrypoint(obj, klass);
1253 }
1254
ResolveStringByIdEntrypoint(ManagedThread * thread,Frame * frame,FileEntityId id)1255 extern "C" coretypes::String *ResolveStringByIdEntrypoint(ManagedThread *thread, Frame *frame, FileEntityId id)
1256 {
1257 BEGIN_ENTRYPOINT();
1258 return thread->GetVM()->ResolveString(frame, panda_file::File::EntityId(id));
1259 }
1260
ResolveLiteralArrayByIdEntrypoint(ManagedThread * thread,const Method * caller,uint32_t typeId)1261 extern "C" coretypes::Array *ResolveLiteralArrayByIdEntrypoint(ManagedThread *thread, const Method *caller,
1262 uint32_t typeId)
1263 {
1264 BEGIN_ENTRYPOINT();
1265 return interpreter::RuntimeInterface::ResolveLiteralArray(thread->GetVM(), *caller, BytecodeId(typeId));
1266 }
1267
ResolveTypeByIdEntrypoint(ManagedThread * thread,Method * caller,uint32_t id,InterpreterCache::Entry * entry,const uint8_t * pc)1268 extern "C" Class *ResolveTypeByIdEntrypoint(ManagedThread *thread, Method *caller, uint32_t id,
1269 InterpreterCache::Entry *entry, const uint8_t *pc)
1270 {
1271 CHECK_STACK_WALKER;
1272 auto klass = interpreter::RuntimeInterface::ResolveClass<false>(thread, *caller, BytecodeId(id));
1273 if (klass == nullptr) {
1274 return nullptr;
1275 }
1276 *entry = {pc, caller, static_cast<void *>(klass)};
1277 return klass;
1278 }
1279
GetFieldByIdEntrypoint(ManagedThread * thread,Method * caller,uint32_t fieldId,InterpreterCache::Entry * entry,const uint8_t * pc)1280 extern "C" Field *GetFieldByIdEntrypoint([[maybe_unused]] ManagedThread *thread, Method *caller, uint32_t fieldId,
1281 InterpreterCache::Entry *entry, const uint8_t *pc)
1282 {
1283 CHECK_STACK_WALKER;
1284 auto resolvedId = caller->GetClass()->ResolveFieldIndex(BytecodeId(fieldId).AsIndex());
1285 auto *classLinker = Runtime::GetCurrent()->GetClassLinker();
1286 auto *field = classLinker->GetField(*caller, panda_file::File::EntityId(resolvedId));
1287 if (field != nullptr) {
1288 *entry = {pc, caller, static_cast<void *>(field)};
1289 }
1290 return field;
1291 }
1292
GetStaticFieldByIdEntrypoint(ManagedThread * thread,Method * caller,uint32_t fieldId,InterpreterCache::Entry * entry,const uint8_t * pc)1293 extern "C" Field *GetStaticFieldByIdEntrypoint(ManagedThread *thread, Method *caller, uint32_t fieldId,
1294 InterpreterCache::Entry *entry, const uint8_t *pc)
1295 {
1296 CHECK_STACK_WALKER;
1297 auto *field = interpreter::RuntimeInterface::ResolveField(thread, *caller, BytecodeId(fieldId));
1298 if (field == nullptr) {
1299 return field;
1300 }
1301 ASSERT(field->IsStatic());
1302 *entry = {pc, caller, static_cast<void *>(field)};
1303 return field;
1304 }
1305
GetCalleeMethodFromBytecodeId(ManagedThread * thread,Method * caller,uint32_t calleeId,InterpreterCache::Entry * entry,const uint8_t * pc)1306 extern "C" Method *GetCalleeMethodFromBytecodeId(ManagedThread *thread, Method *caller, uint32_t calleeId,
1307 InterpreterCache::Entry *entry, const uint8_t *pc)
1308 {
1309 CHECK_STACK_WALKER;
1310 auto *method = interpreter::RuntimeInterface::ResolveMethod(thread, *caller, BytecodeId(calleeId));
1311 if (method == nullptr) {
1312 return nullptr; // if nullptr we don't need to cache
1313 }
1314 *entry = {pc, caller, static_cast<void *>(method)};
1315 return method;
1316 }
1317
GetMethodClassById(Method * method,uint32_t methodId)1318 extern "C" Class *GetMethodClassById(Method *method, uint32_t methodId)
1319 {
1320 CHECK_STACK_WALKER;
1321 Class *klass = interpreter::RuntimeInterface::GetMethodClass(method, BytecodeId(methodId));
1322 return klass; // May be nullptr if exception occured
1323 }
1324
ResolveVirtualMethod(const Method * callee,Frame * frame,const ObjectPointerType objPtr,const uint8_t * pc,Method * caller)1325 extern "C" Method *ResolveVirtualMethod(const Method *callee, Frame *frame, const ObjectPointerType objPtr,
1326 const uint8_t *pc, Method *caller)
1327 {
1328 auto *obj = reinterpret_cast<const ObjectHeader *>(objPtr);
1329 ASSERT(IsAddressInObjectsHeap(obj));
1330 auto *cls = obj->ClassAddr<Class>();
1331 ASSERT(cls != nullptr);
1332 auto *resolved = cls->ResolveVirtualMethod(callee);
1333 ASSERT(resolved != nullptr);
1334
1335 ProfilingData *profData = caller->GetProfilingData();
1336 auto bytecodeOffset = pc - frame->GetInstruction();
1337 if (profData != nullptr) {
1338 profData->UpdateInlineCaches(bytecodeOffset, cls);
1339 }
1340
1341 return resolved;
1342 }
1343
Verify(Method * method)1344 extern "C" bool Verify(Method *method)
1345 {
1346 if (UNLIKELY(!method->Verify())) {
1347 interpreter::RuntimeInterface::ThrowVerificationException(method->GetFullName());
1348 return false;
1349 }
1350 return true;
1351 }
1352
DecrementHotnessCounter(Method * method,ManagedThread * thread)1353 extern "C" bool DecrementHotnessCounter(Method *method, ManagedThread *thread)
1354 {
1355 method->DecrementHotnessCounter<true>(thread, 0, nullptr);
1356 if (thread->HasPendingException()) {
1357 return false;
1358 }
1359 return method->GetCompiledEntryPoint() != GetCompiledCodeToInterpreterBridge(method);
1360 }
1361
DecrementHotnessCounterDyn(Method * method,TaggedValue funcObj,ManagedThread * thread)1362 extern "C" bool DecrementHotnessCounterDyn(Method *method, TaggedValue funcObj, ManagedThread *thread)
1363 {
1364 method->DecrementHotnessCounter<true>(thread, 0, nullptr, false, funcObj);
1365 if (thread->HasPendingException()) {
1366 return false;
1367 }
1368 return method->GetCompiledEntryPoint() != GetCompiledCodeToInterpreterBridge(method);
1369 }
1370
CallCompilerSlowPath(ManagedThread * thread,Method * method)1371 extern "C" void CallCompilerSlowPath(ManagedThread *thread, Method *method)
1372 {
1373 CHECK_STACK_WALKER;
1374 method->DecrementHotnessCounter<false>(thread, 0, nullptr);
1375 }
1376
CallCompilerSlowPathOSR(ManagedThread * thread,Method * method,Frame * frame,uint64_t acc,uint64_t accTag,uint32_t insOffset,int offset)1377 extern "C" bool CallCompilerSlowPathOSR(ManagedThread *thread, Method *method, Frame *frame, uint64_t acc,
1378 uint64_t accTag, uint32_t insOffset, int offset)
1379 {
1380 CHECK_STACK_WALKER;
1381 if constexpr (ArchTraits<RUNTIME_ARCH>::SUPPORT_OSR) {
1382 ASSERT(ArchTraits<RUNTIME_ARCH>::SUPPORT_OSR);
1383 if (!frame->IsDeoptimized() && Runtime::GetOptions().IsCompilerEnableOsr()) {
1384 frame->SetAcc(ark::interpreter::AccVRegister(acc, accTag));
1385 return method->DecrementHotnessCounter<false>(thread, insOffset + offset, &frame->GetAcc(), true);
1386 }
1387 }
1388 method->DecrementHotnessCounter<false>(thread, 0, nullptr);
1389 return false;
1390 }
1391
UpdateBranchTaken(Method * method,Frame * frame,const uint8_t * pc,ProfilingData * profDataIrtoc)1392 extern "C" void UpdateBranchTaken([[maybe_unused]] Method *method, Frame *frame, const uint8_t *pc,
1393 ProfilingData *profDataIrtoc)
1394 {
1395 // Add a second prof_data loading because without it THREAD_SANITIZER crashes
1396 // NOTE(aantipina): investigate and delete the second loading (issue I6DTAA)
1397 [[maybe_unused]] ProfilingData *profData = method->GetProfilingDataWithoutCheck();
1398 profDataIrtoc->UpdateBranchTaken(pc - frame->GetInstruction());
1399 }
1400
UpdateBranchUntaken(Method * method,Frame * frame,const uint8_t * pc,ProfilingData * profDataIrtoc)1401 extern "C" void UpdateBranchUntaken([[maybe_unused]] Method *method, Frame *frame, const uint8_t *pc,
1402 ProfilingData *profDataIrtoc)
1403 {
1404 // Add a second prof_data loading because without it THREAD_SANITIZER crashes
1405 // NOTE(aantipina): investigate and delete the second loading
1406 [[maybe_unused]] ProfilingData *profData = method->GetProfilingDataWithoutCheck();
1407 profDataIrtoc->UpdateBranchNotTaken(pc - frame->GetInstruction());
1408 }
1409
ReadUlebEntrypoint(const uint8_t * ptr)1410 extern "C" uint32_t ReadUlebEntrypoint(const uint8_t *ptr)
1411 {
1412 uint32_t result;
1413 [[maybe_unused]] size_t n;
1414 [[maybe_unused]] bool isFull;
1415 std::tie(result, n, isFull) = leb128::DecodeUnsigned<uint32_t>(ptr);
1416 ASSERT(isFull);
1417 return result;
1418 }
1419
GetInstructionsByMethod(const Method * method)1420 extern "C" const uint8_t *GetInstructionsByMethod(const Method *method)
1421 {
1422 return method->GetInstructions();
1423 }
1424
GetNumVregsByMethod(const Method * method)1425 extern "C" size_t GetNumVregsByMethod(const Method *method)
1426 {
1427 return method->GetNumVregs();
1428 }
1429
ThrowExceptionFromInterpreter(ManagedThread * thread,ObjectHeader * exception,Frame * frame,const uint8_t * pc)1430 extern "C" void ThrowExceptionFromInterpreter(ManagedThread *thread, ObjectHeader *exception, Frame *frame,
1431 const uint8_t *pc)
1432 {
1433 CHECK_STACK_WALKER;
1434 ASSERT(!thread->HasPendingException());
1435 ASSERT(IsAddressInObjectsHeap(exception));
1436 Method *method = frame->GetMethod();
1437 ProfilingData *profData = method->GetProfilingDataWithoutCheck();
1438 if (profData != nullptr) {
1439 profData->UpdateThrowTaken(pc - frame->GetInstruction());
1440 }
1441 thread->SetException(exception);
1442 }
1443
ThrowArithmeticExceptionFromInterpreter()1444 extern "C" void ThrowArithmeticExceptionFromInterpreter()
1445 {
1446 CHECK_STACK_WALKER;
1447 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
1448 interpreter::RuntimeInterface::ThrowArithmeticException();
1449 }
1450
ThrowNullPointerExceptionFromInterpreter()1451 extern "C" void ThrowNullPointerExceptionFromInterpreter()
1452 {
1453 CHECK_STACK_WALKER;
1454 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
1455 interpreter::RuntimeInterface::ThrowNullPointerException();
1456 }
1457
ThrowNegativeArraySizeExceptionFromInterpreter(int32_t size)1458 extern "C" void ThrowNegativeArraySizeExceptionFromInterpreter(int32_t size)
1459 {
1460 CHECK_STACK_WALKER;
1461 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
1462 interpreter::RuntimeInterface::ThrowNegativeArraySizeException(size);
1463 }
1464
ThrowArrayIndexOutOfBoundsExceptionFromInterpreter(ssize_t idx,size_t length)1465 extern "C" void ThrowArrayIndexOutOfBoundsExceptionFromInterpreter(ssize_t idx, size_t length)
1466 {
1467 CHECK_STACK_WALKER;
1468 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
1469 interpreter::RuntimeInterface::ThrowArrayIndexOutOfBoundsException(idx, length);
1470 }
1471
CheckStoreArrayReferenceFromInterpreter(coretypes::Array * array,ObjectHeader * storeObj)1472 extern "C" uint8_t CheckStoreArrayReferenceFromInterpreter(coretypes::Array *array, ObjectHeader *storeObj)
1473 {
1474 CHECK_STACK_WALKER;
1475 ASSERT(array != nullptr);
1476 ASSERT(IsAddressInObjectsHeapOrNull(storeObj));
1477
1478 if (storeObj == nullptr) { // NOTE: may be moved to IRTOC
1479 return 0;
1480 }
1481
1482 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
1483 auto *arrayClass = array->ClassAddr<Class>();
1484 auto *elementClass = arrayClass->GetComponentType();
1485 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
1486 if (!storeObj->IsInstanceOf(elementClass)) {
1487 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
1488 interpreter::RuntimeInterface::ThrowArrayStoreException(arrayClass, storeObj->ClassAddr<Class>());
1489 return 1;
1490 }
1491 return 0;
1492 }
1493
FindCatchBlockInIFramesStackless(ManagedThread ** currThread,Frame ** currFrame,const uint8_t * pc)1494 extern "C" uint32_t FindCatchBlockInIFramesStackless(ManagedThread **currThread, Frame **currFrame, const uint8_t *pc)
1495 {
1496 uint32_t inst = pc - (*currFrame)->GetInstruction();
1497 Frame *frame = *currFrame;
1498
1499 while (frame != nullptr) {
1500 ManagedThread *thread = *currThread;
1501 Frame *prev = frame->GetPrevFrame();
1502 ASSERT(thread->HasPendingException());
1503
1504 Method *method = frame->GetMethod();
1505 uint32_t pcOffset = interpreter::RuntimeInterface::FindCatchBlock(*method, thread->GetException(), inst);
1506 if (pcOffset != panda_file::INVALID_OFFSET) {
1507 return pcOffset;
1508 }
1509
1510 if (!frame->IsStackless() || prev == nullptr || StackWalker::IsBoundaryFrame<FrameKind::INTERPRETER>(prev)) {
1511 return pcOffset;
1512 }
1513
1514 EVENT_METHOD_EXIT(method->GetFullName(), events::MethodExitKind::INTERP, thread->RecordMethodExit());
1515
1516 Runtime::GetCurrent()->GetNotificationManager()->MethodExitEvent(thread, method);
1517
1518 inst = prev->GetBytecodeOffset();
1519 *currFrame = prev;
1520
1521 thread->GetVM()->HandleReturnFrame();
1522
1523 interpreter::RuntimeInterface::SetCurrentFrame(thread, prev);
1524
1525 ASSERT(thread->HasPendingException());
1526
1527 interpreter::RuntimeInterface::FreeFrame(*currThread, frame);
1528
1529 LOG(DEBUG, INTERPRETER) << "Exit: Runtime Call.";
1530
1531 frame = prev;
1532 }
1533
1534 return panda_file::INVALID_OFFSET;
1535 }
1536
FindCatchBlockInIFrames(ManagedThread * currThread,Frame * currFrame,const uint8_t * pc)1537 extern "C" const uint8_t *FindCatchBlockInIFrames(ManagedThread *currThread, Frame *currFrame, const uint8_t *pc)
1538 {
1539 CHECK_STACK_WALKER;
1540
1541 uint32_t pcOffset = panda_file::INVALID_OFFSET;
1542
1543 pcOffset = FindCatchBlockInIFramesStackless(&currThread, &currFrame, pc);
1544 if (pcOffset == panda_file::INVALID_OFFSET) {
1545 if constexpr (RUNTIME_ARCH == Arch::AARCH64 || RUNTIME_ARCH == Arch::AARCH32 || RUNTIME_ARCH == Arch::X86_64) {
1546 ark::FindCatchBlockInCallStack(currThread);
1547 }
1548 return pc;
1549 }
1550
1551 Method *method = currFrame->GetMethod();
1552 ASSERT(method != nullptr);
1553 LanguageContext ctx = interpreter::RuntimeInterface::GetLanguageContext(*method);
1554 ObjectHeader *exceptionObject = currThread->GetException();
1555 ctx.SetExceptionToVReg(currFrame->GetAcc(), exceptionObject);
1556
1557 currThread->ClearException();
1558 Span<const uint8_t> sp(currFrame->GetMethod()->GetInstructions(), pcOffset);
1559 return sp.cend();
1560 }
1561
VmCreateString(ManagedThread * thread,Method * method,ObjectHeader * ctorArg)1562 extern "C" coretypes::String *VmCreateString(ManagedThread *thread, Method *method, ObjectHeader *ctorArg)
1563 {
1564 CHECK_STACK_WALKER;
1565 // If ctor has the only argument (object itself), ctor_arg is allowed to contain garbage
1566 ASSERT(method->GetNumArgs() == 1 || IsAddressInObjectsHeapOrNull(ctorArg));
1567
1568 return thread->GetVM()->CreateString(method, ctorArg);
1569 }
1570
DebugPrintEntrypoint(ark::Frame * frame,const uint8_t * pc,uint64_t accPayload,uint64_t accTag)1571 extern "C" void DebugPrintEntrypoint([[maybe_unused]] ark::Frame *frame, [[maybe_unused]] const uint8_t *pc,
1572 [[maybe_unused]] uint64_t accPayload, [[maybe_unused]] uint64_t accTag)
1573 {
1574 CHECK_STACK_WALKER;
1575 #ifndef NDEBUG
1576 if (!Logger::IsLoggingOn(Logger::Level::DEBUG, Logger::Component::INTERPRETER)) {
1577 return;
1578 }
1579
1580 constexpr uint64_t STANDARD_DEBUG_INDENT = 5;
1581 PandaString accDump;
1582 auto accVreg = ark::interpreter::VRegister(accPayload);
1583 auto accTagVreg = ark::interpreter::VRegister(accTag);
1584 if (frame->IsDynamic()) {
1585 accDump = ark::interpreter::DynamicVRegisterRef(&accVreg).DumpVReg();
1586 LOG(DEBUG, INTERPRETER) << PandaString(STANDARD_DEBUG_INDENT, ' ') << "acc." << accDump;
1587
1588 DynamicFrameHandler frameHandler(frame);
1589 for (size_t i = 0; i < frame->GetSize(); ++i) {
1590 LOG(DEBUG, INTERPRETER) << PandaString(STANDARD_DEBUG_INDENT, ' ') << "v" << i << "."
1591 << frameHandler.GetVReg(i).DumpVReg();
1592 }
1593 } else {
1594 accDump = ark::interpreter::StaticVRegisterRef(&accVreg, &accTagVreg).DumpVReg();
1595 LOG(DEBUG, INTERPRETER) << PandaString(STANDARD_DEBUG_INDENT, ' ') << "acc." << accDump;
1596
1597 StaticFrameHandler frameHandler(frame);
1598 for (size_t i = 0; i < frame->GetSize(); ++i) {
1599 LOG(DEBUG, INTERPRETER) << PandaString(STANDARD_DEBUG_INDENT, ' ') << "v" << i << "."
1600 << frameHandler.GetVReg(i).DumpVReg();
1601 }
1602 }
1603 LOG(DEBUG, INTERPRETER) << " pc: " << (void *)pc << " ---> " << BytecodeInstruction(pc);
1604 #endif
1605 }
1606
JsCastDoubleToInt32(double d)1607 extern "C" PANDA_PUBLIC_API int32_t JsCastDoubleToInt32(double d)
1608 {
1609 return coretypes::JsCastDoubleToInt(d, coretypes::INT32_BITS);
1610 }
1611
JsCastDoubleToInt32Entrypoint(uint64_t d)1612 extern "C" int32_t JsCastDoubleToInt32Entrypoint(uint64_t d)
1613 {
1614 return coretypes::JsCastDoubleToInt(bit_cast<double>(d), coretypes::INT32_BITS);
1615 }
1616
1617 /* The shared implementation of the StringBuilder.append() intrinsics */
1618 template <class T>
ToClassPtr(uintptr_t obj)1619 static T *ToClassPtr(uintptr_t obj)
1620 {
1621 return reinterpret_cast<T *>(*reinterpret_cast<ObjectPointerType *>(obj));
1622 }
1623
1624 /* offset values to be aligned with the StringBuilder class layout */
1625 static constexpr uint32_t SB_COUNT_OFFSET = 12;
1626 static constexpr uint32_t SB_VALUE_OFFSET = 8;
1627
GetCountValue(ObjectHeader * sb)1628 static uint32_t GetCountValue(ObjectHeader *sb)
1629 {
1630 /* StringBuilder is not thread safe, we are supposed to
1631 * provide just the regular access to it's fields */
1632 return sb->GetFieldPrimitive<uint32_t>(SB_COUNT_OFFSET);
1633 }
1634
GetCapacity(ObjectHeader * self)1635 static uint32_t GetCapacity(ObjectHeader *self)
1636 {
1637 return ToClassPtr<coretypes::Array>(ToUintPtr(self) + SB_VALUE_OFFSET)->GetLength();
1638 }
1639
GetStorageAddress(ObjectHeader * sb,uint32_t count)1640 static uint16_t *GetStorageAddress(ObjectHeader *sb, uint32_t count)
1641 {
1642 auto storage = ToClassPtr<coretypes::Array>(ToUintPtr(sb) + SB_VALUE_OFFSET);
1643 return reinterpret_cast<uint16_t *>(ToUintPtr(storage) + coretypes::Array::GetDataOffset() + (count << 1U));
1644 }
1645
RecalculateCapacity(uint32_t capacity,uint32_t newsize)1646 static inline uint32_t RecalculateCapacity(uint32_t capacity, uint32_t newsize)
1647 {
1648 capacity = (capacity + 1) << 1U;
1649 capacity = newsize > capacity ? newsize : capacity;
1650 return capacity;
1651 }
1652
MoveStorageData(ObjectHeader * sb,void * newstorage)1653 static void MoveStorageData(ObjectHeader *sb, void *newstorage)
1654 {
1655 auto size = GetCountValue(sb) << 1U;
1656 auto newbuf = ToVoidPtr(ToUintPtr(newstorage) + coretypes::Array::GetDataOffset());
1657 auto oldbuf = ToVoidPtr(ToUintPtr(GetStorageAddress(sb, 0U)));
1658 std::copy_n(reinterpret_cast<uint8_t *>(oldbuf), size, reinterpret_cast<uint8_t *>(newbuf));
1659 }
1660
AssureCapacity(ObjectHeader * sb,uint32_t newsize,coretypes::String * str)1661 static std::tuple<bool, ObjectHeader *, coretypes::String *> AssureCapacity(ObjectHeader *sb, uint32_t newsize,
1662 coretypes::String *str)
1663 {
1664 if (GetCapacity(sb) >= newsize) {
1665 return std::make_tuple(true, sb, str);
1666 }
1667
1668 auto thread = ManagedThread::GetCurrent();
1669 HandleScope<ObjectHeader *> scope(thread);
1670 VMHandle<ObjectHeader> handle(thread, sb);
1671 VMHandle<coretypes::String> strHandle {};
1672 strHandle = str != nullptr ? VMHandle<coretypes::String>(thread, str) : strHandle;
1673
1674 auto *vm = Runtime::GetCurrent()->GetPandaVM();
1675 auto klass = Runtime::GetCurrent()
1676 ->GetClassLinker()
1677 ->GetExtension(vm->GetLanguageContext())
1678 ->GetClassRoot(ClassRoot::ARRAY_U16);
1679
1680 auto capacity = RecalculateCapacity(GetCapacity(sb), newsize);
1681 auto *newstorage = ark::coretypes::Array::Create(klass, capacity);
1682 sb = reinterpret_cast<ObjectHeader *>(handle.GetPtr());
1683 str = str != nullptr ? strHandle.GetPtr() : nullptr;
1684
1685 if (newstorage == nullptr) {
1686 return std::make_tuple(false, sb, str);
1687 }
1688
1689 MoveStorageData(sb, newstorage);
1690 handle->SetFieldObject(SB_VALUE_OFFSET, newstorage);
1691
1692 return std::make_tuple(true, sb, str);
1693 }
1694
StoreNumber(ObjectHeader * sb,int64_t n)1695 static ObjectHeader *StoreNumber(ObjectHeader *sb, int64_t n)
1696 {
1697 auto num = n < 0 ? -static_cast<uint64_t>(n) : static_cast<uint64_t>(n);
1698 auto size = CountDigits(num) + static_cast<uint32_t>(n < 0);
1699 auto count = GetCountValue(sb);
1700 auto newsize = count + size;
1701
1702 auto success = false;
1703 std::tie(success, sb, std::ignore) = AssureCapacity(sb, newsize, nullptr);
1704 if (!success) {
1705 return sb;
1706 }
1707 utf::UInt64ToUtf16Array(num, GetStorageAddress(sb, count), size, n < 0);
1708 sb->SetFieldPrimitive<uint32_t>(SB_COUNT_OFFSET, newsize);
1709 return sb;
1710 }
1711
CoreStringBuilderLong(ObjectHeader * sb,int64_t n)1712 extern "C" ObjectHeader *CoreStringBuilderLong(ObjectHeader *sb, int64_t n)
1713 {
1714 return StoreNumber(sb, n);
1715 }
1716
CoreStringBuilderInt(ObjectHeader * sb,int32_t n)1717 extern "C" ObjectHeader *CoreStringBuilderInt(ObjectHeader *sb, int32_t n)
1718 {
1719 return StoreNumber(sb, static_cast<int64_t>(n));
1720 }
1721
CoreStringBuilderBool(ObjectHeader * sb,const uint8_t v)1722 extern "C" ObjectHeader *CoreStringBuilderBool(ObjectHeader *sb, const uint8_t v)
1723 {
1724 const uint64_t truestr = 0x0065007500720074;
1725 const uint64_t falsestr = 0x0073006c00610066;
1726 const uint16_t falsestr2 = 0x0065;
1727
1728 auto count = GetCountValue(sb);
1729 auto size = 4U + (1U - v); // v is actually u1
1730 auto newsize = count + size;
1731
1732 auto success = false;
1733 std::tie(success, sb, std::ignore) = AssureCapacity(sb, newsize, nullptr);
1734 if (!success) {
1735 return sb;
1736 }
1737 auto dst = reinterpret_cast<uint64_t *>(GetStorageAddress(sb, count));
1738
1739 if (v != 0) {
1740 *dst = truestr;
1741 } else {
1742 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
1743 auto dst2 = reinterpret_cast<uint16_t *>(dst + 1);
1744 *dst = falsestr;
1745 *dst2 = falsestr2;
1746 }
1747
1748 sb->SetFieldPrimitive<uint32_t>(SB_COUNT_OFFSET, newsize);
1749 return sb;
1750 }
1751
CoreStringBuilderChar(ObjectHeader * sb,const uint16_t c)1752 extern "C" ObjectHeader *CoreStringBuilderChar(ObjectHeader *sb, const uint16_t c)
1753 {
1754 auto count = GetCountValue(sb);
1755 auto newsize = count + 1;
1756
1757 auto success = false;
1758 std::tie(success, sb, std::ignore) = AssureCapacity(sb, newsize, nullptr);
1759 if (!success) {
1760 return sb;
1761 }
1762 auto dst = GetStorageAddress(sb, count);
1763
1764 *dst = c;
1765 sb->SetFieldPrimitive<uint32_t>(SB_COUNT_OFFSET, newsize);
1766 return sb;
1767 }
1768
CoreStringBuilderString(ObjectHeader * sb,void * s)1769 extern "C" ObjectHeader *CoreStringBuilderString(ObjectHeader *sb, void *s)
1770 {
1771 const uint64_t nullstr = 0x006c006c0075006e;
1772 auto str = static_cast<coretypes::String *>(s);
1773 auto isnull = str == nullptr;
1774 auto size = isnull ? sizeof(nullstr) / sizeof(uint16_t) : str->GetLength();
1775 if (size == 0) {
1776 return sb;
1777 }
1778
1779 auto count = GetCountValue(sb);
1780 auto newsize = count + size;
1781
1782 auto success = false;
1783 std::tie(success, sb, str) = AssureCapacity(sb, newsize, str);
1784 if (!success) {
1785 return sb;
1786 }
1787
1788 auto dst = GetStorageAddress(sb, count);
1789
1790 if (!isnull) {
1791 auto src = ToVoidPtr(ToUintPtr(str) + coretypes::String::GetDataOffset());
1792 str->IsUtf16() ? std::copy_n(reinterpret_cast<uint16_t *>(src), size, dst)
1793 : std::copy_n(reinterpret_cast<uint8_t *>(src), size, dst);
1794 } else {
1795 /* store the 8-bytes chunk right away */
1796 *reinterpret_cast<uint64_t *>(dst) = nullstr;
1797 }
1798
1799 sb->SetFieldPrimitive<uint32_t>(SB_COUNT_OFFSET, newsize);
1800 return sb;
1801 }
1802
CoreStringConcat2(coretypes::String * str1,coretypes::String * str2)1803 extern "C" coretypes::String *CoreStringConcat2(coretypes::String *str1, coretypes::String *str2)
1804 {
1805 auto *vm = Runtime::GetCurrent()->GetPandaVM();
1806 auto ctx = vm->GetLanguageContext();
1807 return coretypes::String::Concat(str1, str2, ctx, vm);
1808 }
1809
CoreStringConcat3(coretypes::String * str1,coretypes::String * str2,coretypes::String * str3)1810 extern "C" coretypes::String *CoreStringConcat3(coretypes::String *str1, coretypes::String *str2,
1811 coretypes::String *str3)
1812 {
1813 auto *vm = Runtime::GetCurrent()->GetPandaVM();
1814 auto ctx = vm->GetLanguageContext();
1815 auto thread = ManagedThread::GetCurrent();
1816 [[maybe_unused]] HandleScope<ObjectHeader *> scope(thread);
1817 VMHandle<String> str3Handle(thread, str3);
1818 auto str = coretypes::String::Concat(str1, str2, ctx, vm);
1819 if (UNLIKELY(str == nullptr)) {
1820 HandlePendingException();
1821 UNREACHABLE();
1822 }
1823 str = coretypes::String::Concat(str, str3Handle.GetPtr(), ctx, vm);
1824 return str;
1825 }
1826
CoreStringConcat4(coretypes::String * str1,coretypes::String * str2,coretypes::String * str3,coretypes::String * str4)1827 extern "C" coretypes::String *CoreStringConcat4(coretypes::String *str1, coretypes::String *str2,
1828 coretypes::String *str3, coretypes::String *str4)
1829 {
1830 auto *vm = Runtime::GetCurrent()->GetPandaVM();
1831 auto ctx = vm->GetLanguageContext();
1832 auto thread = ManagedThread::GetCurrent();
1833 [[maybe_unused]] HandleScope<ObjectHeader *> scope(thread);
1834 VMHandle<String> str3Handle(thread, str3);
1835 VMHandle<String> str4Handle(thread, str4);
1836 auto str = coretypes::String::Concat(str1, str2, ctx, vm);
1837 if (UNLIKELY(str == nullptr)) {
1838 HandlePendingException();
1839 UNREACHABLE();
1840 }
1841 str3 = str3Handle.GetPtr();
1842 str = coretypes::String::Concat(str, str3, ctx, vm);
1843 if (UNLIKELY(str == nullptr)) {
1844 HandlePendingException();
1845 UNREACHABLE();
1846 }
1847 str4 = str4Handle.GetPtr();
1848 str = coretypes::String::Concat(str, str4, ctx, vm);
1849 return str;
1850 }
1851
1852 } // namespace ark
1853