1 /**
2 * Copyright (c) 2021-2025 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "runtime/entrypoints/entrypoints.h"
17
18 #include "libpandabase/events/events.h"
19 #include "libpandabase/macros.h"
20 #include "libpandabase/utils/utils.h"
21 #include "compiler/compiler_options.h"
22 #include "runtime/deoptimization.h"
23 #include "runtime/arch/memory_helpers.h"
24 #include "runtime/jit/profiling_data.h"
25 #include "runtime/include/class_linker-inl.h"
26 #include "runtime/include/coretypes/array.h"
27 #include "runtime/include/exceptions.h"
28 #include "runtime/include/method-inl.h"
29 #include "runtime/include/object_header-inl.h"
30 #include "runtime/include/runtime.h"
31 #include "runtime/include/value-inl.h"
32 #include "runtime/include/panda_vm.h"
33 #include "runtime/interpreter/frame.h"
34 #include "runtime/interpreter/interpreter.h"
35 #include "runtime/interpreter/runtime_interface.h"
36 #include "runtime/mem/tlab.h"
37 #include "compiler/optimizer/ir/runtime_interface.h"
38 #include "runtime/handle_base-inl.h"
39 #include "libpandabase/utils/asan_interface.h"
40 #include "libpandabase/utils/tsan_interface.h"
41 #include "utils/cframe_layout.h"
42 #include "intrinsics.h"
43 #include "runtime/interpreter/vregister_iterator.h"
44
45 namespace ark {
46
47 using ark::compiler::TraceId;
48 using ark::coretypes::String;
49
50 #undef LOG_ENTRYPOINTS
51
52 class ScopedLog {
53 public:
54 ScopedLog() = delete;
ScopedLog(const char * function)55 explicit ScopedLog(const char *function) : function_(function)
56 {
57 LOG(DEBUG, INTEROP) << "ENTRYPOINT: " << function;
58 }
~ScopedLog()59 ~ScopedLog()
60 {
61 LOG(DEBUG, INTEROP) << "EXIT ENTRYPOINT: " << function_;
62 }
63 NO_COPY_SEMANTIC(ScopedLog);
64 NO_MOVE_SEMANTIC(ScopedLog);
65
66 private:
67 std::string function_;
68 };
69
70 #ifdef LOG_ENTRYPOINTS
71 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
72 #define LOG_ENTRYPOINT() ScopedLog __log(__FUNCTION__)
73 #else
74 #define LOG_ENTRYPOINT()
75 #endif
76
77 // enable stack walker dry run on each entrypoint to discover stack issues early
78 #ifndef NDEBUG
79 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
80 #define CHECK_STACK_WALKER \
81 if (Runtime::GetOptions().IsVerifyEntrypoints()) { \
82 StackWalker::Create(ManagedThread::GetCurrent()).Verify(); \
83 }
84 #else
85 #define CHECK_STACK_WALKER
86 #endif
87
88 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
89 #define BEGIN_ENTRYPOINT() \
90 CHECK_STACK_WALKER; \
91 LOG_ENTRYPOINT()
92
InterpreterEntryPoint(Method * method,Frame * frame)93 extern "C" NO_ADDRESS_SANITIZE void InterpreterEntryPoint(Method *method, Frame *frame)
94 {
95 auto pc = method->GetInstructions();
96 Method *callee = frame->GetMethod();
97 ASSERT(callee != nullptr);
98
99 if (callee->IsAbstract()) {
100 ASSERT(pc == nullptr);
101 ark::ThrowAbstractMethodError(callee);
102 HandlePendingException();
103 UNREACHABLE();
104 }
105
106 ManagedThread *thread = ManagedThread::GetCurrent();
107 ASSERT(thread != nullptr);
108 if (!thread->template StackOverflowCheck<true, false>()) {
109 HandlePendingException(UnwindPolicy::SKIP_INLINED);
110 UNREACHABLE();
111 }
112
113 Frame *prevFrame = thread->GetCurrentFrame();
114 thread->SetCurrentFrame(frame);
115
116 auto isCompiledCode = thread->IsCurrentFrameCompiled();
117 thread->SetCurrentFrameIsCompiled(false);
118 interpreter::Execute(thread, pc, frame);
119 thread->SetCurrentFrameIsCompiled(isCompiledCode);
120 if (prevFrame != nullptr && reinterpret_cast<uintptr_t>(prevFrame->GetMethod()) == COMPILED_CODE_TO_INTERPRETER) {
121 thread->SetCurrentFrame(prevFrame->GetPrevFrame());
122 } else {
123 thread->SetCurrentFrame(prevFrame);
124 }
125 }
126
AnnotateSanitizersEntrypoint(void const * addr,size_t size)127 extern "C" void AnnotateSanitizersEntrypoint([[maybe_unused]] void const *addr, [[maybe_unused]] size_t size)
128 {
129 #ifdef PANDA_TSAN_ON
130 TSAN_ANNOTATE_HAPPENS_BEFORE(const_cast<void *>(addr));
131 #endif
132 #ifdef PANDA_ASAN_ON
133 ASAN_UNPOISON_MEMORY_REGION(addr, size);
134 #endif
135 }
136
WriteTlabStatsEntrypoint(void const * mem,size_t size)137 extern "C" void WriteTlabStatsEntrypoint([[maybe_unused]] void const *mem, size_t size)
138 {
139 LOG_ENTRYPOINT();
140
141 LOG(DEBUG, MM_OBJECT_EVENTS) << "Alloc object in compiled code at " << mem << " size: " << size;
142 auto *thread = ManagedThread::GetCurrent();
143 ASSERT(thread != nullptr);
144 ASSERT(size <= Runtime::GetOptions().GetMaxTlabSize());
145 // 1. Pointer to TLAB
146 // 2. Pointer to allocated memory
147 // 3. size
148 [[maybe_unused]] auto tlab = reinterpret_cast<size_t>(thread->GetTLAB());
149 EVENT_TLAB_ALLOC(thread->GetId(), tlab, reinterpret_cast<size_t>(mem), size);
150 if (mem::PANDA_TRACK_TLAB_ALLOCATIONS) {
151 auto memStats = thread->GetVM()->GetHeapManager()->GetMemStats();
152 if (memStats == nullptr) {
153 return;
154 }
155 memStats->RecordAllocateObject(size, SpaceType::SPACE_TYPE_OBJECT);
156 }
157 }
158
CreateArraySlowPathEntrypoint(Class * klass,size_t length)159 extern "C" coretypes::Array *CreateArraySlowPathEntrypoint(Class *klass, size_t length)
160 {
161 BEGIN_ENTRYPOINT();
162
163 TSAN_ANNOTATE_HAPPENS_AFTER(klass);
164 auto arr = coretypes::Array::Create(klass, length);
165 if (UNLIKELY(arr == nullptr)) {
166 HandlePendingException();
167 UNREACHABLE();
168 }
169 if (compiler::g_options.IsCompilerEnableTlabEvents()) {
170 EVENT_SLOWPATH_ALLOC(ManagedThread::GetCurrent()->GetId());
171 }
172 return arr;
173 }
174
CreateMultiArrayRecEntrypoint(ManagedThread * thread,Class * klass,uint32_t nargs,size_t * sizes,uint32_t num)175 extern "C" coretypes::Array *CreateMultiArrayRecEntrypoint(ManagedThread *thread, Class *klass, uint32_t nargs,
176 size_t *sizes, uint32_t num)
177 {
178 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic,-warnings-as-errors)
179 auto arrSize = sizes[num];
180
181 [[maybe_unused]] HandleScope<ObjectHeader *> scope(thread);
182 // SUPPRESS_CSA_NEXTLINE(alpha.core.CheckObjHeaderTypeRef)
183 VMHandle<coretypes::Array> handle(thread, coretypes::Array::Create(klass, arrSize));
184 if (handle.GetPtr() == nullptr) {
185 return nullptr;
186 }
187 auto *component = klass->GetComponentType();
188
189 if (component->IsArrayClass() && num + 1 < nargs) {
190 for (size_t idx = 0; idx < arrSize; idx++) {
191 auto *array = CreateMultiArrayRecEntrypoint(thread, component, nargs, sizes, num + 1);
192
193 if (array == nullptr) {
194 return nullptr;
195 }
196 handle.GetPtr()->template Set<coretypes::Array *>(idx, array);
197 }
198 }
199
200 return handle.GetPtr();
201 }
202
CreateEmptyStringEntrypoint()203 extern "C" coretypes::String *CreateEmptyStringEntrypoint()
204 {
205 BEGIN_ENTRYPOINT();
206
207 auto thread = ManagedThread::GetCurrent();
208 ASSERT(thread != nullptr);
209 auto vm = thread->GetVM();
210 auto str = coretypes::String::CreateEmptyString(vm->GetLanguageContext(), vm);
211 if (UNLIKELY(str == nullptr)) {
212 HandlePendingException();
213 UNREACHABLE();
214 }
215 return str;
216 }
217
CreateStringFromStringEntrypoint(ObjectHeader * obj)218 extern "C" coretypes::String *CreateStringFromStringEntrypoint(ObjectHeader *obj)
219 {
220 BEGIN_ENTRYPOINT();
221 auto thread = ManagedThread::GetCurrent();
222 ASSERT(thread != nullptr);
223 auto vm = thread->GetVM();
224 auto str = coretypes::String::CreateFromString(static_cast<coretypes::String *>(obj), vm->GetLanguageContext(), vm);
225 if (UNLIKELY(str == nullptr)) {
226 HandlePendingException();
227 UNREACHABLE();
228 }
229 return str;
230 }
231
CreateStringFromCharsEntrypoint(ObjectHeader * obj)232 extern "C" coretypes::String *CreateStringFromCharsEntrypoint(ObjectHeader *obj)
233 {
234 BEGIN_ENTRYPOINT();
235 auto thread = ManagedThread::GetCurrent();
236 ASSERT(thread != nullptr);
237 auto vm = thread->GetVM();
238 auto array = static_cast<coretypes::Array *>(obj);
239 auto str = coretypes::String::CreateNewStringFromChars(0, array->GetLength(), array, vm->GetLanguageContext(), vm);
240 if (UNLIKELY(str == nullptr)) {
241 HandlePendingException();
242 UNREACHABLE();
243 }
244 return str;
245 }
246
CreateStringFromCharsWithOffsetEntrypoint(uint32_t offset,uint32_t length,ObjectHeader * obj)247 extern "C" coretypes::String *CreateStringFromCharsWithOffsetEntrypoint(uint32_t offset, uint32_t length,
248 ObjectHeader *obj)
249 {
250 BEGIN_ENTRYPOINT();
251 auto thread = ManagedThread::GetCurrent();
252 ASSERT(thread != nullptr);
253 auto vm = thread->GetVM();
254 auto array = static_cast<coretypes::Array *>(obj);
255 auto str = coretypes::String::CreateNewStringFromChars(offset, length, array, vm->GetLanguageContext(), vm);
256 if (UNLIKELY(str == nullptr)) {
257 HandlePendingException();
258 UNREACHABLE();
259 }
260 return str;
261 }
262
CreateStringFromCharsZeroOffsetEntrypoint(uint32_t length,ObjectHeader * obj)263 extern "C" coretypes::String *CreateStringFromCharsZeroOffsetEntrypoint(uint32_t length, ObjectHeader *obj)
264 {
265 BEGIN_ENTRYPOINT();
266 auto thread = ManagedThread::GetCurrent();
267 ASSERT(thread != nullptr);
268 auto vm = thread->GetVM();
269 auto array = static_cast<coretypes::Array *>(obj);
270 auto str = coretypes::String::CreateNewStringFromChars(0, length, array, vm->GetLanguageContext(), vm);
271 if (UNLIKELY(str == nullptr)) {
272 HandlePendingException();
273 UNREACHABLE();
274 }
275 return str;
276 }
277
SubStringFromStringEntrypoint(ObjectHeader * obj,int32_t begin,int32_t end)278 extern "C" coretypes::String *SubStringFromStringEntrypoint(ObjectHeader *obj, int32_t begin, int32_t end)
279 {
280 BEGIN_ENTRYPOINT();
281
282 auto thread = ManagedThread::GetCurrent();
283 ASSERT(thread != nullptr);
284 auto vm = thread->GetVM();
285 auto indexes = coretypes::String::NormalizeSubStringIndexes(begin, end, static_cast<coretypes::String *>(obj));
286 auto substrLength = indexes.second - indexes.first;
287 auto substr = coretypes::String::FastSubString(static_cast<coretypes::String *>(obj), indexes.first, substrLength,
288 vm->GetLanguageContext(), vm);
289 if (UNLIKELY(substr == nullptr)) {
290 HandlePendingException();
291 UNREACHABLE();
292 }
293 return substr;
294 }
295
StringGetCharsEntrypoint(ObjectHeader * obj,int32_t begin,int32_t end)296 extern "C" coretypes::Array *StringGetCharsEntrypoint(ObjectHeader *obj, int32_t begin, int32_t end)
297 {
298 BEGIN_ENTRYPOINT();
299
300 auto length = static_cast<coretypes::String *>(obj)->GetLength();
301 if (UNLIKELY(static_cast<uint32_t>(end) > length)) {
302 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
303 ark::ThrowStringIndexOutOfBoundsException(end, length);
304 HandlePendingException(UnwindPolicy::SKIP_INLINED);
305 UNREACHABLE();
306 }
307 if (UNLIKELY(begin > end)) {
308 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
309 ark::ThrowStringIndexOutOfBoundsException(begin, length);
310 HandlePendingException(UnwindPolicy::SKIP_INLINED);
311 UNREACHABLE();
312 }
313
314 if (UNLIKELY(begin < 0)) {
315 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
316 ark::ThrowStringIndexOutOfBoundsException(begin, length);
317 HandlePendingException(UnwindPolicy::SKIP_INLINED);
318 UNREACHABLE();
319 }
320 auto thread = ManagedThread::GetCurrent();
321 ASSERT(thread != nullptr);
322 auto vm = thread->GetVM();
323 auto arrayLength = end - begin;
324 auto array = coretypes::String::GetChars(static_cast<coretypes::String *>(obj), begin, arrayLength,
325 vm->GetLanguageContext());
326 if (UNLIKELY(array == nullptr)) {
327 HandlePendingException();
328 UNREACHABLE();
329 }
330 return array;
331 }
332
ResolveLiteralArrayEntrypoint(const Method * caller,uint32_t typeId)333 extern "C" coretypes::Array *ResolveLiteralArrayEntrypoint(const Method *caller, uint32_t typeId)
334 {
335 BEGIN_ENTRYPOINT();
336
337 auto thread = Runtime::GetCurrent();
338 ASSERT(thread != nullptr);
339 auto arr = thread->ResolveLiteralArray(ManagedThread::GetCurrent()->GetVM(), *caller, typeId);
340 if (UNLIKELY(arr == nullptr)) {
341 HandlePendingException();
342 UNREACHABLE();
343 }
344 return arr;
345 }
346
CreateMultiArrayEntrypoint(Class * klass,uint32_t nargs,size_t * sizes)347 extern "C" coretypes::Array *CreateMultiArrayEntrypoint(Class *klass, uint32_t nargs, size_t *sizes)
348 {
349 BEGIN_ENTRYPOINT();
350
351 auto arr = CreateMultiArrayRecEntrypoint(ManagedThread::GetCurrent(), klass, nargs, sizes, 0);
352 if (UNLIKELY(arr == nullptr)) {
353 HandlePendingException();
354 UNREACHABLE();
355 }
356 return arr;
357 }
358
CreateObjectByClassEntrypoint(Class * klass)359 extern "C" ObjectHeader *CreateObjectByClassEntrypoint(Class *klass)
360 {
361 BEGIN_ENTRYPOINT();
362
363 // We need annotation here for the FullMemoryBarrier used in InitializeClassByIdEntrypoint
364 TSAN_ANNOTATE_HAPPENS_AFTER(klass);
365 if (klass->IsStringClass()) {
366 LanguageContext ctx = Runtime::GetCurrent()->GetLanguageContext(*klass);
367 auto str = coretypes::String::CreateEmptyString(ctx, Runtime::GetCurrent()->GetPandaVM());
368 if (UNLIKELY(str == nullptr)) {
369 HandlePendingException();
370 UNREACHABLE();
371 }
372 return str;
373 }
374
375 if (LIKELY(klass->IsInstantiable())) {
376 auto obj = ObjectHeader::Create(klass);
377 if (UNLIKELY(obj == nullptr)) {
378 HandlePendingException();
379 UNREACHABLE();
380 }
381 if (compiler::g_options.IsCompilerEnableTlabEvents()) {
382 EVENT_SLOWPATH_ALLOC(ManagedThread::GetCurrent()->GetId());
383 }
384 return obj;
385 }
386
387 ThrowInstantiationErrorEntrypoint(klass);
388 UNREACHABLE();
389 }
390
CloneObjectEntrypoint(ObjectHeader * obj)391 extern "C" ObjectHeader *CloneObjectEntrypoint(ObjectHeader *obj)
392 {
393 BEGIN_ENTRYPOINT();
394
395 uint32_t flags = obj->ClassAddr<Class>()->GetFlags();
396 if (UNLIKELY((flags & Class::IS_CLONEABLE) == 0)) {
397 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
398 ThrowCloneNotSupportedException();
399 HandlePendingException(UnwindPolicy::SKIP_INLINED);
400 return nullptr;
401 }
402 return ObjectHeader::Clone(obj);
403 }
404
PostBarrierWriteEntrypoint(ObjectHeader * obj,size_t size)405 extern "C" ObjectHeader *PostBarrierWriteEntrypoint(ObjectHeader *obj, size_t size)
406 {
407 LOG_ENTRYPOINT();
408 AnnotateSanitizersEntrypoint(obj, size);
409 auto *objectClass = obj->ClassAddr<Class>();
410 auto *thread = ManagedThread::GetCurrent();
411 ASSERT(thread != nullptr);
412 auto *barrierSet = thread->GetBarrierSet();
413 if (!objectClass->IsArrayClass() || !objectClass->GetComponentType()->IsPrimitive()) {
414 barrierSet->PostBarrier(obj, 0, size);
415 }
416 return obj;
417 }
418
CheckCastEntrypoint(const ObjectHeader * obj,Class * klass)419 extern "C" void CheckCastEntrypoint(const ObjectHeader *obj, Class *klass)
420 {
421 BEGIN_ENTRYPOINT();
422
423 // Don't use obj after ClassLinker call because GC can move it.
424 // Since we need only class and class in a non-movalble object
425 // it is ok to get it here.
426 Class *objKlass = obj == nullptr ? nullptr : obj->ClassAddr<Class>();
427 if (UNLIKELY(objKlass != nullptr && !klass->IsAssignableFrom(objKlass))) {
428 ark::ThrowClassCastException(klass, objKlass);
429 HandlePendingException();
430 UNREACHABLE();
431 }
432 }
433
CheckCastDeoptimizeEntrypoint(const ObjectHeader * obj,Class * klass)434 extern "C" void CheckCastDeoptimizeEntrypoint(const ObjectHeader *obj, Class *klass)
435 {
436 BEGIN_ENTRYPOINT();
437
438 ASSERT(obj != nullptr);
439 auto *objKlass = obj->ClassAddr<Class>();
440 ASSERT(objKlass != nullptr);
441 if (UNLIKELY(!klass->IsAssignableFrom(objKlass))) {
442 DeoptimizeEntrypoint(static_cast<uint64_t>(compiler::DeoptimizeType::CHECK_CAST));
443 }
444 }
445
IsInstanceEntrypoint(ObjectHeader * obj,Class * klass)446 extern "C" uint8_t IsInstanceEntrypoint(ObjectHeader *obj, Class *klass)
447 {
448 BEGIN_ENTRYPOINT();
449
450 // Don't use obj after ClassLinker call because GC can move it.
451 // Since we need only class and class in a non-movalble object
452 // it is ok to get it here.
453 Class *objKlass = obj == nullptr ? nullptr : obj->ClassAddr<Class>();
454 if (UNLIKELY(objKlass != nullptr && klass->IsAssignableFrom(objKlass))) {
455 return 1;
456 }
457 return 0;
458 }
459
SafepointEntrypoint()460 extern "C" void SafepointEntrypoint()
461 {
462 BEGIN_ENTRYPOINT();
463 interpreter::RuntimeInterface::Safepoint();
464 }
465
ResolveClassObjectEntrypoint(const Method * caller,FileEntityId typeId)466 extern "C" ObjectHeader *ResolveClassObjectEntrypoint(const Method *caller, FileEntityId typeId)
467 {
468 BEGIN_ENTRYPOINT();
469 auto klass = reinterpret_cast<Class *>(ResolveClassEntrypoint(caller, typeId));
470 return klass->GetManagedObject();
471 }
472
ResolveClassEntrypoint(const Method * caller,FileEntityId typeId)473 extern "C" Class *ResolveClassEntrypoint(const Method *caller, FileEntityId typeId)
474 {
475 BEGIN_ENTRYPOINT();
476 ClassLinker *classLinker = Runtime::GetCurrent()->GetClassLinker();
477 Class *klass = classLinker->GetClass(*caller, panda_file::File::EntityId(typeId));
478 if (UNLIKELY(klass == nullptr)) {
479 HandlePendingException();
480 UNREACHABLE();
481 }
482 return klass;
483 }
484
ResolveStringEntrypoint(const Method * caller,FileEntityId id)485 extern "C" coretypes::String *ResolveStringEntrypoint(const Method *caller, FileEntityId id)
486 {
487 BEGIN_ENTRYPOINT();
488 auto *thread = Runtime::GetCurrent();
489 ASSERT(thread != nullptr);
490 return thread->ResolveStringFromCompiledCode(ManagedThread::GetCurrent()->GetVM(), *caller,
491 panda_file::File::EntityId(id));
492 }
493
ResolveStringAotEntrypoint(const Method * caller,FileEntityId id,ObjectHeader ** slot)494 extern "C" coretypes::String *ResolveStringAotEntrypoint(const Method *caller, FileEntityId id, ObjectHeader **slot)
495 {
496 BEGIN_ENTRYPOINT();
497 auto runtime = Runtime::GetCurrent();
498 auto aotManager = runtime->GetClassLinker()->GetAotManager();
499 auto thread = ManagedThread::GetCurrent();
500 ASSERT(thread != nullptr);
501 auto vm = thread->GetVM();
502 auto str = runtime->ResolveStringFromCompiledCode(vm, *caller, ark::panda_file::File::EntityId(id));
503 if (UNLIKELY(str == nullptr)) {
504 return nullptr;
505 }
506
507 // to many strings were saved to slots, so simply return the resolved string
508 if (aotManager->GetAotStringRootsCount() >= Runtime::GetOptions().GetAotStringGcRootsLimit()) {
509 return str;
510 }
511
512 auto counter = reinterpret_cast<std::atomic_uint32_t *>(slot);
513
514 // Atomic with acquire order reason: data race with slot with dependecies on reads after the load which should
515 // become visible
516 auto counterVal = counter->load(std::memory_order_acquire);
517 if (counterVal >= compiler::RuntimeInterface::RESOLVE_STRING_AOT_COUNTER_LIMIT - 1) {
518 return str;
519 }
520
521 if (counterVal < Runtime::GetOptions().GetResolveStringAotThreshold()) {
522 // try to update counter, but ignore result - in the worst case we'll save
523 // string's pointer to slot a bit later
524 counter->compare_exchange_strong(counterVal, counterVal + 1, std::memory_order_release,
525 std::memory_order_relaxed);
526 } else {
527 // try to replace the counter with string pointer and register the slot as GC root in case of success
528 if (counter->compare_exchange_strong(counterVal, static_cast<uint32_t>(reinterpret_cast<uint64_t>(str)),
529 std::memory_order_release, std::memory_order_relaxed)) {
530 bool isYoung = vm->GetHeapManager()->IsObjectInYoungSpace(str);
531 aotManager->RegisterAotStringRoot(slot, isYoung);
532 EVENT_AOT_RESOLVE_STRING(ConvertToString(str));
533 }
534 }
535
536 return str;
537 }
538
CreateFrameWithSize(uint32_t size,uint32_t nregs,Method * method,Frame * prev)539 extern "C" Frame *CreateFrameWithSize(uint32_t size, uint32_t nregs, Method *method, Frame *prev)
540 {
541 uint32_t extSz = EMPTY_EXT_FRAME_DATA_SIZE;
542 if (LIKELY(method)) {
543 extSz = Runtime::GetCurrent()->GetLanguageContext(*method).GetFrameExtSize();
544 }
545 size_t allocSz = Frame::GetAllocSize(size, extSz);
546 Frame *frame = Thread::GetCurrent()->GetVM()->GetHeapManager()->AllocateExtFrame(allocSz, extSz);
547 if (UNLIKELY(frame == nullptr)) {
548 return nullptr;
549 }
550 return new (frame) Frame(Frame::ToExt(frame, extSz), method, prev, nregs);
551 }
552
CreateFrameWithActualArgsAndSize(uint32_t size,uint32_t nregs,uint32_t numActualArgs,Method * method,Frame * prev)553 extern "C" Frame *CreateFrameWithActualArgsAndSize(uint32_t size, uint32_t nregs, uint32_t numActualArgs,
554 Method *method, Frame *prev)
555 {
556 auto *thread = ManagedThread::GetCurrent();
557 ASSERT(thread != nullptr);
558 uint32_t extSz = thread->GetVM()->GetFrameExtSize();
559 size_t allocSz = Frame::GetAllocSize(size, extSz);
560 void *mem = thread->GetStackFrameAllocator()->Alloc(allocSz);
561 if (UNLIKELY(mem == nullptr)) {
562 return nullptr;
563 }
564 return new (Frame::FromExt(mem, extSz)) Frame(mem, method, prev, nregs, numActualArgs);
565 }
566
CreateNativeFrameWithActualArgsAndSize(uint32_t size,uint32_t nregs,uint32_t numActualArgs,Method * method,Frame * prev)567 extern "C" Frame *CreateNativeFrameWithActualArgsAndSize(uint32_t size, uint32_t nregs, uint32_t numActualArgs,
568 Method *method, Frame *prev)
569 {
570 uint32_t extSz = EMPTY_EXT_FRAME_DATA_SIZE;
571 size_t allocSz = Frame::GetAllocSize(size, extSz);
572 auto *thread = ManagedThread::GetCurrent();
573 ASSERT(thread != nullptr);
574 void *mem = thread->GetStackFrameAllocator()->Alloc(allocSz);
575 if (UNLIKELY(mem == nullptr)) {
576 return nullptr;
577 }
578 return new (Frame::FromExt(mem, extSz)) Frame(mem, method, prev, nregs, numActualArgs);
579 }
580
581 template <bool IS_DYNAMIC = false>
CreateFrame(uint32_t nregs,Method * method,Frame * prev)582 static Frame *CreateFrame(uint32_t nregs, Method *method, Frame *prev)
583 {
584 return CreateFrameWithSize(Frame::GetActualSize<IS_DYNAMIC>(nregs), nregs, method, prev);
585 }
586
587 template <bool IS_DYNAMIC>
CreateFrameWithActualArgs(uint32_t nregs,uint32_t numActualArgs,Method * method,Frame * prev)588 static Frame *CreateFrameWithActualArgs(uint32_t nregs, uint32_t numActualArgs, Method *method, Frame *prev)
589 {
590 auto frame =
591 CreateFrameWithActualArgsAndSize(Frame::GetActualSize<IS_DYNAMIC>(nregs), nregs, numActualArgs, method, prev);
592 if (UNLIKELY(frame == nullptr)) {
593 return nullptr;
594 }
595 if (IS_DYNAMIC) {
596 LanguageContext ctx = Runtime::GetCurrent()->GetLanguageContext(*method);
597 coretypes::TaggedValue initialValue = ctx.GetInitialTaggedValue();
598 for (size_t i = 0; i < nregs; i++) {
599 frame->GetVReg(i).SetValue(initialValue.GetRawData());
600 }
601 frame->SetDynamic();
602 }
603 return frame;
604 }
605
CreateFrameForMethod(Method * method,Frame * prev)606 extern "C" Frame *CreateFrameForMethod(Method *method, Frame *prev)
607 {
608 auto nregs = method->GetNumArgs() + method->GetNumVregs();
609 return CreateFrame<false>(nregs, method, prev);
610 }
611
CreateFrameForMethodDyn(Method * method,Frame * prev)612 extern "C" Frame *CreateFrameForMethodDyn(Method *method, Frame *prev)
613 {
614 auto nregs = method->GetNumArgs() + method->GetNumVregs();
615 return CreateFrame<true>(nregs, method, prev);
616 }
617
CreateFrameForMethodWithActualArgs(uint32_t numActualArgs,Method * method,Frame * prev)618 extern "C" Frame *CreateFrameForMethodWithActualArgs(uint32_t numActualArgs, Method *method, Frame *prev)
619 {
620 auto nargs = std::max(numActualArgs, method->GetNumArgs());
621 auto nregs = nargs + method->GetNumVregs();
622 return CreateFrameWithActualArgs<false>(nregs, numActualArgs, method, prev);
623 }
624
CreateFrameForMethodWithActualArgsDyn(uint32_t numActualArgs,Method * method,Frame * prev)625 extern "C" Frame *CreateFrameForMethodWithActualArgsDyn(uint32_t numActualArgs, Method *method, Frame *prev)
626 {
627 auto nargs = std::max(numActualArgs, method->GetNumArgs());
628 auto nregs = nargs + method->GetNumVregs();
629 return CreateFrameWithActualArgs<true>(nregs, numActualArgs, method, prev);
630 }
631
FreeFrame(Frame * frame)632 extern "C" void FreeFrame(Frame *frame)
633 {
634 ASSERT(frame->GetExt() != nullptr);
635 auto *thread = ManagedThread::GetCurrent();
636 ASSERT(thread != nullptr);
637 thread->GetStackFrameAllocator()->Free(frame->GetExt());
638 }
639
GetStaticFieldAddressEntrypoint(Method * method,uint32_t fieldId)640 extern "C" uintptr_t GetStaticFieldAddressEntrypoint(Method *method, uint32_t fieldId)
641 {
642 BEGIN_ENTRYPOINT();
643 auto *classLinker = Runtime::GetCurrent()->GetClassLinker();
644 auto field = classLinker->GetField(*method, panda_file::File::EntityId(fieldId), true);
645 if (UNLIKELY(field == nullptr)) {
646 HandlePendingException();
647 UNREACHABLE();
648 }
649 auto *klass = field->GetClass();
650 ASSERT(klass != nullptr);
651 return reinterpret_cast<uintptr_t>(klass) + field->GetOffset();
652 }
653
UnresolvedStoreStaticBarrieredEntrypoint(Method * method,uint32_t fieldId,ObjectHeader * obj)654 extern "C" void UnresolvedStoreStaticBarrieredEntrypoint(Method *method, uint32_t fieldId, ObjectHeader *obj)
655 {
656 BEGIN_ENTRYPOINT();
657 auto thread = ManagedThread::GetCurrent();
658 // One must not use plain ObjectHeader pointer here as GC can move obj.
659 // Wrap it in VMHandle.
660 [[maybe_unused]] HandleScope<ObjectHeader *> objHandleScope(thread);
661 VMHandle<ObjectHeader> objHandle(thread, obj);
662
663 auto field = Runtime::GetCurrent()->GetClassLinker()->GetField(*method, panda_file::File::EntityId(fieldId), true);
664 if (UNLIKELY(field == nullptr)) {
665 HandlePendingException();
666 UNREACHABLE();
667 }
668 if (UNLIKELY(!field->GetType().IsReference())) {
669 HandlePendingException();
670 UNREACHABLE();
671 }
672
673 auto *classPtr = field->GetClass();
674 ASSERT(classPtr != nullptr);
675 InitializeClassEntrypoint(classPtr);
676
677 // field_addr = class_ptr + field's offset
678 auto fieldAddr = reinterpret_cast<uint32_t *>(reinterpret_cast<uintptr_t>(classPtr) + field->GetOffset());
679 auto *barrierSet = thread->GetBarrierSet();
680 // Pre-barrier
681 if (barrierSet->IsPreBarrierEnabled()) {
682 // Load
683 // Atomic with acquire order reason: assume load can be volatile
684 auto currentReference = __atomic_load_n(fieldAddr, std::memory_order_acquire);
685 barrierSet->PreBarrier(reinterpret_cast<void *>(currentReference));
686 }
687 // Store
688 // Atomic with release order reason: assume store can be volatile
689 auto ref = reinterpret_cast<uintptr_t>(objHandle.GetPtr());
690 ASSERT(sizeof(ref) == 4U || (ref & 0xFFFFFFFF00000000UL) == 0);
691 __atomic_store_n(fieldAddr, static_cast<uint32_t>(ref), std::memory_order_release);
692 // Post-barrier
693 if (!mem::IsEmptyBarrier(barrierSet->GetPostType())) {
694 auto objPtrPtr = reinterpret_cast<uintptr_t>(classPtr) + Class::GetManagedObjectOffset();
695 auto objPtr = *reinterpret_cast<uint32_t *>(objPtrPtr);
696 barrierSet->PostBarrier(reinterpret_cast<void *>(objPtr), 0, reinterpret_cast<void *>(ref));
697 }
698 }
699
GetUnknownStaticFieldMemoryAddressEntrypoint(Method * method,uint32_t fieldId,size_t * slot)700 extern "C" uintptr_t GetUnknownStaticFieldMemoryAddressEntrypoint(Method *method, uint32_t fieldId, size_t *slot)
701 {
702 BEGIN_ENTRYPOINT();
703 auto *classLinker = Runtime::GetCurrent()->GetClassLinker();
704 auto field = classLinker->GetField(*method, panda_file::File::EntityId(fieldId), true);
705 if (UNLIKELY(field == nullptr)) {
706 HandlePendingException();
707 UNREACHABLE();
708 }
709 auto *klass = field->GetClass();
710 ASSERT(klass != nullptr);
711 InitializeClassEntrypoint(klass);
712
713 uintptr_t addr = reinterpret_cast<uintptr_t>(klass) + field->GetOffset();
714 if (klass->IsInitialized() && slot != nullptr) {
715 *slot = addr;
716 }
717 return addr;
718 }
719
GetFieldOffsetEntrypoint(Method * method,uint32_t fieldId)720 extern "C" size_t GetFieldOffsetEntrypoint(Method *method, uint32_t fieldId)
721 {
722 BEGIN_ENTRYPOINT();
723 auto *classLinker = Runtime::GetCurrent()->GetClassLinker();
724 auto field = classLinker->GetField(*method, panda_file::File::EntityId(fieldId), false);
725 if (UNLIKELY(field == nullptr)) {
726 HandlePendingException();
727 UNREACHABLE();
728 }
729 return field->GetOffset();
730 }
731
InitializeClassEntrypoint(Class * klass)732 extern "C" void InitializeClassEntrypoint(Class *klass)
733 {
734 BEGIN_ENTRYPOINT();
735 auto *classLinker = Runtime::GetCurrent()->GetClassLinker();
736 if (!klass->IsInitialized() && !classLinker->InitializeClass(ManagedThread::GetCurrent(), klass)) {
737 HandlePendingException();
738 UNREACHABLE();
739 }
740 }
741
InitializeClassByIdEntrypoint(const Method * caller,FileEntityId id)742 extern "C" Class *InitializeClassByIdEntrypoint(const Method *caller, FileEntityId id)
743 {
744 BEGIN_ENTRYPOINT();
745 ClassLinker *classLinker = Runtime::GetCurrent()->GetClassLinker();
746 Class *klass = classLinker->GetClass(*caller, panda_file::File::EntityId(id));
747 if (UNLIKELY(klass == nullptr)) {
748 HandlePendingException();
749 UNREACHABLE();
750 }
751 // Later we store klass pointer into .aot_got section.
752 // Without full memory barrier on the architectures with weak memory order
753 // we can read klass pointer, but fetch klass data before it's set in GetClass and InitializeClass
754 arch::FullMemoryBarrier();
755 // Full barrier is not visible by TSAN so we need annotation here
756 TSAN_ANNOTATE_HAPPENS_BEFORE(klass);
757 InitializeClassEntrypoint(klass);
758 return klass;
759 }
760
ResolveVirtualCallEntrypoint(const Method * callee,ObjectHeader * obj)761 extern "C" uintptr_t NO_ADDRESS_SANITIZE ResolveVirtualCallEntrypoint(const Method *callee, ObjectHeader *obj)
762 {
763 BEGIN_ENTRYPOINT();
764 if (UNLIKELY(callee == nullptr)) {
765 HandlePendingException();
766 UNREACHABLE();
767 }
768 auto *resolved = obj->ClassAddr<Class>()->ResolveVirtualMethod(callee);
769 ASSERT(resolved != nullptr);
770
771 return reinterpret_cast<uintptr_t>(resolved);
772 }
773
ResolveVirtualCallAotEntrypoint(const Method * caller,ObjectHeader * obj,size_t calleeId,uintptr_t cacheAddr)774 extern "C" uintptr_t NO_ADDRESS_SANITIZE ResolveVirtualCallAotEntrypoint(const Method *caller, ObjectHeader *obj,
775 size_t calleeId,
776 [[maybe_unused]] uintptr_t cacheAddr)
777 {
778 BEGIN_ENTRYPOINT();
779 // Don't use obj after ClassLinker call because GC can move it.
780 // Since we need only class and class in a non-movalble object
781 // it is ok to get it here.
782 auto *objKlass = obj->ClassAddr<Class>();
783 Method *method = Runtime::GetCurrent()->GetClassLinker()->GetMethod(*caller, panda_file::File::EntityId(calleeId));
784 if (UNLIKELY(method == nullptr)) {
785 HandlePendingException();
786 UNREACHABLE();
787 }
788 auto *resolved = objKlass->ResolveVirtualMethod(method);
789 ASSERT(resolved != nullptr);
790
791 #if defined(PANDA_TARGET_ARM64)
792 // In arm64, use interface inlineCache
793 // NOTE(liyiming): will support x86_64 in future
794 // issue #7018
795 auto methodHead = objKlass->GetRawFirstMethodAddr();
796 if (cacheAddr == 0 || methodHead == nullptr) {
797 return reinterpret_cast<uintptr_t>(resolved);
798 }
799
800 constexpr uint32_t METHOD_COMPRESS = 3;
801 constexpr uint32_t CACHE_OFFSET_32 = 32;
802 constexpr uint32_t CACHE_OFFSET_34 = 34;
803 auto cache = reinterpret_cast<int64_t *>(cacheAddr);
804 auto methodResolved = reinterpret_cast<int64_t>(resolved);
805 int64_t methodCache = methodResolved - reinterpret_cast<int64_t>(methodHead);
806
807 int64_t methodCacheJudge = methodCache >> CACHE_OFFSET_34; // NOLINT(hicpp-signed-bitwise)
808 if (methodCacheJudge != 0 && methodCacheJudge != -1) {
809 return reinterpret_cast<uintptr_t>(resolved);
810 }
811 methodCache = methodCache >> METHOD_COMPRESS; // NOLINT(hicpp-signed-bitwise)
812 methodCache = methodCache << CACHE_OFFSET_32; // NOLINT(hicpp-signed-bitwise)
813 int64_t saveCache = methodCache | reinterpret_cast<int64_t>(objKlass); // NOLINT(hicpp-signed-bitwise)
814 *cache = saveCache;
815 #endif
816 return reinterpret_cast<uintptr_t>(resolved);
817 }
818
ResolveUnknownVirtualCallEntrypoint(const Method * caller,ObjectHeader * obj,size_t calleeId,size_t * slot)819 extern "C" uintptr_t NO_ADDRESS_SANITIZE ResolveUnknownVirtualCallEntrypoint(const Method *caller, ObjectHeader *obj,
820 size_t calleeId, size_t *slot)
821 {
822 {
823 auto thread = ManagedThread::GetCurrent();
824 [[maybe_unused]] HandleScope<ObjectHeader *> scope(thread);
825 ASSERT(obj != nullptr);
826 VMHandle<ObjectHeader> handleObj(thread, obj);
827
828 BEGIN_ENTRYPOINT();
829 auto runtime = Runtime::GetCurrent();
830 Method *method = runtime->GetClassLinker()->GetMethod(*caller, panda_file::File::EntityId(calleeId));
831 if (LIKELY(method != nullptr)) {
832 // Cache a method index in vtable
833 if (slot != nullptr && (!method->GetClass()->IsInterface() || method->IsDefaultInterfaceMethod())) {
834 // We save 'vtable index + 1' because 0 value means uninitialized.
835 // Codegen must subtract index after loading from the slot.
836 *slot = method->GetVTableIndex() + 1;
837 }
838 auto objPtr = handleObj.GetPtr();
839 ASSERT(objPtr != nullptr);
840 auto *resolved = objPtr->ClassAddr<Class>()->ResolveVirtualMethod(method);
841 ASSERT(resolved != nullptr);
842
843 return reinterpret_cast<uintptr_t>(resolved);
844 }
845 }
846
847 HandlePendingException();
848 UNREACHABLE();
849 }
850
CheckStoreArrayReferenceEntrypoint(coretypes::Array * array,ObjectHeader * storeObj)851 extern "C" void CheckStoreArrayReferenceEntrypoint(coretypes::Array *array, ObjectHeader *storeObj)
852 {
853 BEGIN_ENTRYPOINT();
854 ASSERT(array != nullptr);
855 ASSERT(storeObj != nullptr);
856
857 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
858 auto *arrayClass = array->ClassAddr<Class>();
859 auto *elementClass = arrayClass->GetComponentType();
860 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
861 if (UNLIKELY(!storeObj->IsInstanceOf(elementClass))) {
862 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
863 ark::ThrowArrayStoreException(arrayClass, storeObj->ClassAddr<Class>());
864 HandlePendingException();
865 UNREACHABLE();
866 }
867 }
868
CheckStoreArrayReferenceDeoptimizeEntrypoint(coretypes::Array * array,ObjectHeader * storeObj)869 extern "C" void CheckStoreArrayReferenceDeoptimizeEntrypoint(coretypes::Array *array, ObjectHeader *storeObj)
870 {
871 BEGIN_ENTRYPOINT();
872 ASSERT(array != nullptr);
873 ASSERT(storeObj != nullptr);
874
875 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
876 auto *arrayClass = array->ClassAddr<Class>();
877 auto *elementClass = arrayClass->GetComponentType();
878 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
879 if (UNLIKELY(!storeObj->IsInstanceOf(elementClass))) {
880 DeoptimizeEntrypoint(static_cast<uint64_t>(compiler::DeoptimizeType::CHECK_CAST));
881 }
882 }
883
GetCalleeMethodEntrypoint(const Method * caller,size_t calleeId)884 extern "C" Method *GetCalleeMethodEntrypoint(const Method *caller, size_t calleeId)
885 {
886 BEGIN_ENTRYPOINT();
887 auto *method = Runtime::GetCurrent()->GetClassLinker()->GetMethod(*caller, panda_file::File::EntityId(calleeId));
888 if (UNLIKELY(method == nullptr)) {
889 HandlePendingException();
890 UNREACHABLE();
891 }
892
893 return method;
894 }
895
GetUnknownCalleeMethodEntrypoint(const Method * caller,size_t calleeId,size_t * slot)896 extern "C" Method *GetUnknownCalleeMethodEntrypoint(const Method *caller, size_t calleeId, size_t *slot)
897 {
898 BEGIN_ENTRYPOINT();
899 auto *method = Runtime::GetCurrent()->GetClassLinker()->GetMethod(*caller, panda_file::File::EntityId(calleeId));
900 if (UNLIKELY(method == nullptr)) {
901 HandlePendingException();
902 UNREACHABLE();
903 }
904 auto klass = method->GetClass();
905 ASSERT(klass != nullptr);
906 InitializeClassEntrypoint(klass);
907
908 if (klass->IsInitialized() && slot != nullptr) {
909 *slot = reinterpret_cast<size_t>(method);
910 }
911
912 return method;
913 }
914
ThrowExceptionEntrypoint(ObjectHeader * exception)915 extern "C" NO_ADDRESS_SANITIZE void ThrowExceptionEntrypoint(ObjectHeader *exception)
916 {
917 BEGIN_ENTRYPOINT();
918 LOG(DEBUG, INTEROP) << "ThrowExceptionEntrypoint \n";
919 auto *thread = ManagedThread::GetCurrent();
920 ASSERT(thread != nullptr);
921 ASSERT(!thread->HasPendingException());
922 if (exception == nullptr) {
923 NullPointerExceptionEntrypoint();
924 UNREACHABLE();
925 }
926 thread->SetException(exception);
927
928 SetExceptionEvent(events::ExceptionType::THROW, thread);
929 HandlePendingException(UnwindPolicy::SKIP_INLINED);
930 }
931
ThrowNativeExceptionEntrypoint()932 extern "C" NO_ADDRESS_SANITIZE void ThrowNativeExceptionEntrypoint()
933 {
934 BEGIN_ENTRYPOINT();
935 LOG(DEBUG, INTEROP) << "ThrowNativeExceptionEntrypoint \n";
936 SetExceptionEvent(events::ExceptionType::NATIVE, ManagedThread::GetCurrent());
937 HandlePendingException(UnwindPolicy::SKIP_INLINED);
938 }
939
ArrayIndexOutOfBoundsExceptionEntrypoint(ssize_t idx,size_t length)940 extern "C" NO_ADDRESS_SANITIZE void ArrayIndexOutOfBoundsExceptionEntrypoint([[maybe_unused]] ssize_t idx,
941 [[maybe_unused]] size_t length)
942 {
943 BEGIN_ENTRYPOINT();
944 LOG(DEBUG, INTEROP) << "ArrayIndexOutOfBoundsExceptionEntrypoint \n";
945 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
946 ThrowArrayIndexOutOfBoundsException(idx, length);
947 HandlePendingException(UnwindPolicy::SKIP_INLINED);
948 }
949
StringIndexOutOfBoundsExceptionEntrypoint(ssize_t idx,size_t length)950 extern "C" NO_ADDRESS_SANITIZE void StringIndexOutOfBoundsExceptionEntrypoint([[maybe_unused]] ssize_t idx,
951 [[maybe_unused]] size_t length)
952 {
953 BEGIN_ENTRYPOINT();
954 LOG(DEBUG, INTEROP) << "StringIndexOutOfBoundsExceptionEntrypoint \n";
955 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
956 ThrowStringIndexOutOfBoundsException(idx, length);
957 HandlePendingException(UnwindPolicy::SKIP_INLINED);
958 }
959
NullPointerExceptionEntrypoint()960 extern "C" NO_ADDRESS_SANITIZE void NullPointerExceptionEntrypoint()
961 {
962 BEGIN_ENTRYPOINT();
963 LOG(DEBUG, INTEROP) << "NullPointerExceptionEntrypoint \n";
964 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
965 ThrowNullPointerException();
966 HandlePendingException(UnwindPolicy::SKIP_INLINED);
967 }
968
AbstractMethodErrorEntrypoint(Method * method)969 extern "C" NO_ADDRESS_SANITIZE void AbstractMethodErrorEntrypoint(Method *method)
970 {
971 BEGIN_ENTRYPOINT();
972 LOG(DEBUG, INTEROP) << "AbstractMethodErrorEntrypoint \n";
973 ManagedThread *thread = ManagedThread::GetCurrent();
974 ASSERT(!thread->HasPendingException());
975 auto stack = StackWalker::Create(thread, UnwindPolicy::SKIP_INLINED);
976 ThrowAbstractMethodError(method);
977 ASSERT(thread->HasPendingException());
978 if (stack.IsCFrame()) {
979 FindCatchBlockInCFrames(thread, &stack, nullptr);
980 }
981 }
982
ArithmeticExceptionEntrypoint()983 extern "C" NO_ADDRESS_SANITIZE void ArithmeticExceptionEntrypoint()
984 {
985 BEGIN_ENTRYPOINT();
986 LOG(DEBUG, INTEROP) << "ArithmeticExceptionEntrypoint \n";
987 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
988 ThrowArithmeticException();
989 HandlePendingException(UnwindPolicy::SKIP_INLINED);
990 }
991
NegativeArraySizeExceptionEntrypoint(ssize_t size)992 extern "C" NO_ADDRESS_SANITIZE void NegativeArraySizeExceptionEntrypoint(ssize_t size)
993 {
994 BEGIN_ENTRYPOINT();
995 LOG(DEBUG, INTEROP) << "NegativeArraySizeExceptionEntrypoint \n";
996 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
997 ThrowNegativeArraySizeException(size);
998 HandlePendingException(UnwindPolicy::SKIP_INLINED);
999 }
1000
ClassCastExceptionEntrypoint(Class * instClass,ObjectHeader * srcObj)1001 extern "C" NO_ADDRESS_SANITIZE void ClassCastExceptionEntrypoint(Class *instClass, ObjectHeader *srcObj)
1002 {
1003 BEGIN_ENTRYPOINT();
1004 LOG(DEBUG, INTEROP) << "ClassCastExceptionEntrypoint \n";
1005 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
1006 ASSERT(srcObj != nullptr);
1007 ThrowClassCastException(instClass, srcObj->ClassAddr<Class>());
1008 HandlePendingException(UnwindPolicy::SKIP_INLINED);
1009 }
1010
StackOverflowExceptionEntrypoint()1011 extern "C" NO_ADDRESS_SANITIZE void StackOverflowExceptionEntrypoint()
1012 {
1013 // WARNING: We should not add any heavy code constructions here, like events or other debug/testing stuff,
1014 // because we have small stack here, see ManagedThread::STACK_OVERFLOW_RESERVED_SIZE.
1015 auto thread = ManagedThread::GetCurrent();
1016 ASSERT(thread != nullptr);
1017 ASSERT(!thread->HasPendingException());
1018 thread->DisableStackOverflowCheck();
1019 ThrowStackOverflowException(thread);
1020 thread->EnableStackOverflowCheck();
1021 HandlePendingException(UnwindPolicy::SKIP_INLINED);
1022 }
1023
DeoptimizeEntrypoint(uint64_t deoptimizeType)1024 extern "C" NO_ADDRESS_SANITIZE void DeoptimizeEntrypoint(uint64_t deoptimizeType)
1025 {
1026 BEGIN_ENTRYPOINT();
1027 auto thread = ManagedThread::GetCurrent();
1028 auto type = static_cast<ark::compiler::DeoptimizeType>(
1029 deoptimizeType & ((1U << MinimumBitsToStore(ark::compiler::DeoptimizeType::COUNT)) - 1));
1030 [[maybe_unused]] auto instId = deoptimizeType >> MinimumBitsToStore(ark::compiler::DeoptimizeType::COUNT);
1031 LOG(INFO, INTEROP) << "DeoptimizeEntrypoint (reason: " << ark::compiler::DeoptimizeTypeToString(type)
1032 << ", inst_id: " << instId << ")\n";
1033
1034 EVENT_DEOPTIMIZATION_REASON(std::string(StackWalker::Create(thread).GetMethod()->GetFullName()),
1035 ark::compiler::DeoptimizeTypeToString(type));
1036
1037 ASSERT(!thread->HasPendingException());
1038 auto stack = StackWalker::Create(thread);
1039 Method *destroyMethod = nullptr;
1040 if (type >= ark::compiler::DeoptimizeType::CAUSE_METHOD_DESTRUCTION) {
1041 // Get pointer to top method
1042 destroyMethod = StackWalker::Create(thread, UnwindPolicy::SKIP_INLINED).GetMethod();
1043 }
1044 Deoptimize(&stack, nullptr, false, destroyMethod);
1045 }
1046
ThrowInstantiationErrorEntrypoint(Class * klass)1047 extern "C" NO_ADDRESS_SANITIZE void ThrowInstantiationErrorEntrypoint(Class *klass)
1048 {
1049 BEGIN_ENTRYPOINT();
1050 LOG(DEBUG, INTEROP) << "ThrowInstantiationErrorEntrypoint \n";
1051 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
1052 const auto &name = klass->GetName();
1053 PandaString pname(name.cbegin(), name.cend());
1054 ThrowInstantiationError(pname);
1055 HandlePendingException(UnwindPolicy::SKIP_INLINED);
1056 }
1057
GetInitialTaggedValue(Method * method)1058 coretypes::TaggedValue GetInitialTaggedValue(Method *method)
1059 {
1060 BEGIN_ENTRYPOINT();
1061 return Runtime::GetCurrent()->GetLanguageContext(*method).GetInitialTaggedValue();
1062 }
1063
LockObjectEntrypoint(ObjectHeader * obj)1064 extern "C" void LockObjectEntrypoint(ObjectHeader *obj)
1065 {
1066 BEGIN_ENTRYPOINT();
1067 ark::intrinsics::ObjectMonitorEnter(obj);
1068 }
1069
LockObjectSlowPathEntrypoint(ObjectHeader * obj)1070 extern "C" void LockObjectSlowPathEntrypoint(ObjectHeader *obj)
1071 {
1072 BEGIN_ENTRYPOINT();
1073 ark::intrinsics::ObjectMonitorEnter(obj);
1074 auto *thread = ManagedThread::GetCurrent();
1075 ASSERT(thread != nullptr);
1076 if (!thread->HasPendingException()) {
1077 return;
1078 }
1079 LOG(DEBUG, INTEROP) << "ThrowNativeExceptionEntrypoint after LockObject \n";
1080 SetExceptionEvent(events::ExceptionType::NATIVE, thread);
1081 HandlePendingException(UnwindPolicy::SKIP_INLINED);
1082 }
1083
UnlockObjectEntrypoint(ObjectHeader * obj)1084 extern "C" void UnlockObjectEntrypoint(ObjectHeader *obj)
1085 {
1086 BEGIN_ENTRYPOINT();
1087 ark::intrinsics::ObjectMonitorExit(obj);
1088 }
1089
UnlockObjectSlowPathEntrypoint(ObjectHeader * obj)1090 extern "C" void UnlockObjectSlowPathEntrypoint(ObjectHeader *obj)
1091 {
1092 BEGIN_ENTRYPOINT();
1093 ark::intrinsics::ObjectMonitorExit(obj);
1094 auto *thread = ManagedThread::GetCurrent();
1095 ASSERT(thread != nullptr);
1096 if (!thread->HasPendingException()) {
1097 return;
1098 }
1099 LOG(DEBUG, INTEROP) << "ThrowNativeExceptionEntrypoint after UnlockObject \n";
1100 SetExceptionEvent(events::ExceptionType::NATIVE, ManagedThread::GetCurrent());
1101 HandlePendingException(UnwindPolicy::SKIP_INLINED);
1102 }
1103
IncompatibleClassChangeErrorForMethodConflictEntrypoint(Method * method)1104 extern "C" NO_ADDRESS_SANITIZE void IncompatibleClassChangeErrorForMethodConflictEntrypoint(Method *method)
1105 {
1106 BEGIN_ENTRYPOINT();
1107 LOG(DEBUG, INTEROP) << "IncompatibleClassChangeErrorForMethodConflictEntrypoint \n";
1108 ManagedThread *thread = ManagedThread::GetCurrent();
1109 ASSERT(!thread->HasPendingException());
1110 auto stack = StackWalker::Create(thread, UnwindPolicy::SKIP_INLINED);
1111 ThrowIncompatibleClassChangeErrorForMethodConflict(method);
1112 ASSERT(thread->HasPendingException());
1113 if (stack.IsCFrame()) {
1114 FindCatchBlockInCFrames(thread, &stack, nullptr);
1115 }
1116 }
1117
1118 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
TraceEntrypoint(size_t pid,...)1119 extern "C" void TraceEntrypoint(size_t pid, ...)
1120 {
1121 LOG_ENTRYPOINT();
1122 auto id = static_cast<TraceId>(pid);
1123
1124 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1125 va_list args;
1126 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1127 va_start(args, pid);
1128 switch (id) {
1129 case TraceId::METHOD_ENTER: {
1130 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,clang-analyzer-valist.Uninitialized)
1131 [[maybe_unused]] auto method = va_arg(args, const Method *);
1132 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1133 [[maybe_unused]] auto kind = va_arg(args, events::MethodEnterKind);
1134 EVENT_METHOD_ENTER(method->GetFullName(), kind, ManagedThread::GetCurrent()->RecordMethodEnter());
1135 break;
1136 }
1137 case TraceId::METHOD_EXIT: {
1138 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,clang-analyzer-valist.Uninitialized)
1139 [[maybe_unused]] auto method = va_arg(args, const Method *);
1140 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1141 [[maybe_unused]] auto kind = va_arg(args, events::MethodExitKind);
1142 EVENT_METHOD_EXIT(method->GetFullName(), kind, ManagedThread::GetCurrent()->RecordMethodExit());
1143 break;
1144 }
1145 case TraceId::PRINT_ARG: {
1146 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,clang-analyzer-valist.Uninitialized)
1147 size_t argsNum = va_arg(args, size_t);
1148 std::cerr << "[TRACE ARGS] ";
1149 for (size_t i = 0; i < argsNum; i++) {
1150 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1151 std::cerr << i << "=" << va_arg(args, void *) << " ";
1152 }
1153 std::cerr << std::endl;
1154
1155 break;
1156 }
1157 default: {
1158 break;
1159 }
1160 }
1161 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1162 va_end(args);
1163 }
1164
LogEntrypoint(const char * fmt,...)1165 extern "C" void LogEntrypoint(const char *fmt, ...)
1166 {
1167 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1168 va_list args;
1169 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1170 va_start(args, fmt);
1171 // NOLINTNEXTLINE(clang-analyzer-valist.Uninitialized)
1172 vfprintf(stderr, fmt, args);
1173 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
1174 va_end(args);
1175 }
1176
1177 // Irtoc Interpreter Entrypoints
1178
FreeFrameInterp(Frame * frame,ManagedThread * current)1179 extern "C" void FreeFrameInterp(Frame *frame, ManagedThread *current)
1180 {
1181 ASSERT(frame->GetExt() != nullptr);
1182 current->GetStackFrameAllocator()->Free(frame->GetExt());
1183 }
1184
AllocFrameInterp(ManagedThread * current,size_t allocSz)1185 extern "C" void *AllocFrameInterp(ManagedThread *current, size_t allocSz)
1186 {
1187 void *mem = current->GetStackFrameAllocator()->Alloc<false>(allocSz);
1188 if (UNLIKELY(mem == nullptr)) {
1189 current->DisableStackOverflowCheck();
1190 ark::ThrowStackOverflowException(current);
1191 current->EnableStackOverflowCheck();
1192 }
1193 return mem;
1194 }
1195
InitializeFrame(void * mem,Method * method,Frame * prev,size_t nregs)1196 extern "C" Frame *InitializeFrame(void *mem, Method *method, Frame *prev, size_t nregs)
1197 {
1198 return new (Frame::FromExt(mem, CORE_EXT_FRAME_DATA_SIZE)) Frame(mem, method, prev, nregs);
1199 }
1200
SafepointEntrypointInterp(ManagedThread * thread)1201 extern "C" void SafepointEntrypointInterp(ManagedThread *thread)
1202 {
1203 BEGIN_ENTRYPOINT();
1204 interpreter::RuntimeInterface::Safepoint(thread);
1205 }
1206
IsCompiled(const void * entrypoint)1207 extern "C" int IsCompiled(const void *entrypoint)
1208 {
1209 return static_cast<int>(entrypoint != reinterpret_cast<const void *>(CompiledCodeToInterpreterBridge));
1210 }
1211
GetClassIdEntrypoint(const Method * caller,uint32_t classId)1212 extern "C" size_t GetClassIdEntrypoint(const Method *caller, uint32_t classId)
1213 {
1214 auto resolvedId = caller->GetClass()->ResolveClassIndex(BytecodeId(classId).AsIndex());
1215 return resolvedId.GetOffset();
1216 }
1217
CreateArrayByIdEntrypoint(ManagedThread * thread,const Method * caller,uint32_t classId,int32_t length)1218 extern "C" coretypes::Array *CreateArrayByIdEntrypoint(ManagedThread *thread, const Method *caller, uint32_t classId,
1219 int32_t length)
1220 {
1221 CHECK_STACK_WALKER;
1222 auto *klass = interpreter::RuntimeInterface::ResolveClass<true>(thread, *caller, BytecodeId(classId));
1223 if (UNLIKELY(klass == nullptr)) {
1224 return nullptr;
1225 }
1226 return interpreter::RuntimeInterface::CreateArray(klass, length);
1227 }
1228
1229 template <BytecodeInstruction::Format FORMAT>
CreateMultiDimArray(ManagedThread * thread,Frame * frame,Class * klass,Method * caller,uint32_t methodId,const uint8_t * pc)1230 static coretypes::Array *CreateMultiDimArray(ManagedThread *thread, Frame *frame, Class *klass, Method *caller,
1231 uint32_t methodId, const uint8_t *pc)
1232 {
1233 interpreter::DimIterator<FORMAT> dimIter {BytecodeInstruction(pc), frame};
1234 auto nargs = interpreter::RuntimeInterface::GetMethodArgumentsCount(caller, BytecodeId(methodId));
1235 auto obj =
1236 coretypes::Array::CreateMultiDimensionalArray<interpreter::DimIterator<FORMAT>>(thread, klass, nargs, dimIter);
1237 return obj;
1238 }
1239
CreateMultiDimensionalArrayById(ManagedThread * thread,Frame * frame,Class * klass,Method * caller,uint32_t methodId,const uint8_t * pc,int32_t formatIdx)1240 extern "C" coretypes::Array *CreateMultiDimensionalArrayById(ManagedThread *thread, Frame *frame, Class *klass,
1241 Method *caller, uint32_t methodId, const uint8_t *pc,
1242 int32_t formatIdx)
1243 {
1244 switch (formatIdx) {
1245 case 0U:
1246 return CreateMultiDimArray<BytecodeInstruction::Format::V4_V4_ID16>(thread, frame, klass, caller, methodId,
1247 pc);
1248 case 1U:
1249 return CreateMultiDimArray<BytecodeInstruction::Format::V4_V4_V4_V4_ID16>(thread, frame, klass, caller,
1250 methodId, pc);
1251 case 2U:
1252 return CreateMultiDimArray<BytecodeInstruction::Format::V8_ID16>(thread, frame, klass, caller, methodId,
1253 pc);
1254 default:
1255 UNREACHABLE();
1256 }
1257 return nullptr;
1258 }
1259
CreateObjectByClassInterpreter(ManagedThread * thread,Class * klass)1260 extern "C" ObjectHeader *CreateObjectByClassInterpreter(ManagedThread *thread, Class *klass)
1261 {
1262 ASSERT(klass != nullptr);
1263 ASSERT(!klass->IsArrayClass());
1264
1265 if (!klass->IsInitialized()) {
1266 auto *classLinker = Runtime::GetCurrent()->GetClassLinker();
1267 if (!classLinker->InitializeClass(thread, klass)) {
1268 return nullptr;
1269 }
1270 }
1271
1272 return interpreter::RuntimeInterface::CreateObject(thread, klass);
1273 }
1274
CheckCastByIdEntrypoint(ObjectHeader * obj,Class * klass)1275 extern "C" uint32_t CheckCastByIdEntrypoint(ObjectHeader *obj, Class *klass)
1276 {
1277 CHECK_STACK_WALKER;
1278 ASSERT(IsAddressInObjectsHeapOrNull(obj));
1279 Class *objKlass = obj == nullptr ? nullptr : obj->ClassAddr<Class>();
1280 if (UNLIKELY(objKlass != nullptr && !klass->IsAssignableFrom(objKlass))) {
1281 ark::ThrowClassCastException(klass, objKlass);
1282 return 1;
1283 }
1284 return 0;
1285 }
1286
IsInstanceByIdEntrypoint(ObjectHeader * obj,Class * klass)1287 extern "C" uint32_t IsInstanceByIdEntrypoint(ObjectHeader *obj, Class *klass)
1288 {
1289 CHECK_STACK_WALKER;
1290 ASSERT(IsAddressInObjectsHeapOrNull(obj));
1291
1292 return IsInstanceEntrypoint(obj, klass);
1293 }
1294
ResolveStringByIdEntrypoint(ManagedThread * thread,Frame * frame,FileEntityId id)1295 extern "C" coretypes::String *ResolveStringByIdEntrypoint(ManagedThread *thread, Frame *frame, FileEntityId id)
1296 {
1297 BEGIN_ENTRYPOINT();
1298 return thread->GetVM()->ResolveString(frame, panda_file::File::EntityId(id));
1299 }
1300
ResolveLiteralArrayByIdEntrypoint(ManagedThread * thread,const Method * caller,uint32_t typeId)1301 extern "C" coretypes::Array *ResolveLiteralArrayByIdEntrypoint(ManagedThread *thread, const Method *caller,
1302 uint32_t typeId)
1303 {
1304 BEGIN_ENTRYPOINT();
1305 return interpreter::RuntimeInterface::ResolveLiteralArray(thread->GetVM(), *caller, BytecodeId(typeId));
1306 }
1307
ResolveTypeByIdEntrypoint(ManagedThread * thread,Method * caller,uint32_t id,InterpreterCache::Entry * entry,const uint8_t * pc)1308 extern "C" Class *ResolveTypeByIdEntrypoint(ManagedThread *thread, Method *caller, uint32_t id,
1309 InterpreterCache::Entry *entry, const uint8_t *pc)
1310 {
1311 CHECK_STACK_WALKER;
1312 auto klass = interpreter::RuntimeInterface::ResolveClass<false>(thread, *caller, BytecodeId(id));
1313 if (klass == nullptr) {
1314 return nullptr;
1315 }
1316 *entry = {pc, caller, static_cast<void *>(klass)};
1317 return klass;
1318 }
1319
GetFieldByIdEntrypoint(ManagedThread * thread,Method * caller,uint32_t fieldId,InterpreterCache::Entry * entry,const uint8_t * pc)1320 extern "C" Field *GetFieldByIdEntrypoint([[maybe_unused]] ManagedThread *thread, Method *caller, uint32_t fieldId,
1321 InterpreterCache::Entry *entry, const uint8_t *pc)
1322 {
1323 CHECK_STACK_WALKER;
1324 auto resolvedId = caller->GetClass()->ResolveFieldIndex(BytecodeId(fieldId).AsIndex());
1325 auto *classLinker = Runtime::GetCurrent()->GetClassLinker();
1326 auto *field = classLinker->GetField(*caller, panda_file::File::EntityId(resolvedId), false);
1327 if (field != nullptr) {
1328 *entry = {pc, caller, static_cast<void *>(field)};
1329 }
1330 return field;
1331 }
1332
GetStaticFieldByIdEntrypoint(ManagedThread * thread,Method * caller,uint32_t fieldId,InterpreterCache::Entry * entry,const uint8_t * pc)1333 extern "C" Field *GetStaticFieldByIdEntrypoint(ManagedThread *thread, Method *caller, uint32_t fieldId,
1334 InterpreterCache::Entry *entry, const uint8_t *pc)
1335 {
1336 CHECK_STACK_WALKER;
1337 auto *field = interpreter::RuntimeInterface::ResolveField(thread, *caller, BytecodeId(fieldId), true);
1338 if (field == nullptr) {
1339 return field;
1340 }
1341 ASSERT(field->IsStatic());
1342 *entry = {pc, caller, static_cast<void *>(field)};
1343 return field;
1344 }
1345
GetCalleeMethodFromBytecodeId(ManagedThread * thread,Method * caller,uint32_t calleeId,InterpreterCache::Entry * entry,const uint8_t * pc)1346 extern "C" Method *GetCalleeMethodFromBytecodeId(ManagedThread *thread, Method *caller, uint32_t calleeId,
1347 InterpreterCache::Entry *entry, const uint8_t *pc)
1348 {
1349 CHECK_STACK_WALKER;
1350 auto *method = interpreter::RuntimeInterface::ResolveMethod(thread, *caller, BytecodeId(calleeId));
1351 if (method == nullptr) {
1352 return nullptr; // if nullptr we don't need to cache
1353 }
1354 *entry = {pc, caller, static_cast<void *>(method)};
1355 return method;
1356 }
1357
GetMethodClassById(Method * method,uint32_t methodId)1358 extern "C" Class *GetMethodClassById(Method *method, uint32_t methodId)
1359 {
1360 CHECK_STACK_WALKER;
1361 Class *klass = interpreter::RuntimeInterface::GetMethodClass(method, BytecodeId(methodId));
1362 return klass; // May be nullptr if exception occured
1363 }
1364
ResolveVirtualMethod(const Method * callee,Frame * frame,const ObjectPointerType objPtr,const uint8_t * pc,Method * caller)1365 extern "C" Method *ResolveVirtualMethod(const Method *callee, Frame *frame, const ObjectPointerType objPtr,
1366 const uint8_t *pc, Method *caller)
1367 {
1368 auto *obj = reinterpret_cast<const ObjectHeader *>(objPtr);
1369 ASSERT(IsAddressInObjectsHeap(obj));
1370 auto *cls = obj->ClassAddr<Class>();
1371 ASSERT(cls != nullptr);
1372 auto *resolved = cls->ResolveVirtualMethod(callee);
1373 ASSERT(resolved != nullptr);
1374
1375 ProfilingData *profData = caller->GetProfilingData();
1376 auto bytecodeOffset = pc - frame->GetInstruction();
1377 if (profData != nullptr) {
1378 profData->UpdateInlineCaches(bytecodeOffset, cls);
1379 }
1380
1381 return resolved;
1382 }
1383
Verify(Method * method)1384 extern "C" bool Verify(Method *method)
1385 {
1386 if (UNLIKELY(!method->Verify())) {
1387 interpreter::RuntimeInterface::ThrowVerificationException(method->GetFullName());
1388 return false;
1389 }
1390 return true;
1391 }
1392
DecrementHotnessCounter(Method * method,ManagedThread * thread)1393 extern "C" bool DecrementHotnessCounter(Method *method, ManagedThread *thread)
1394 {
1395 method->DecrementHotnessCounter<true>(thread, 0, nullptr);
1396 if (thread->HasPendingException()) {
1397 return false;
1398 }
1399 return method->GetCompiledEntryPoint() != GetCompiledCodeToInterpreterBridge(method);
1400 }
1401
DecrementHotnessCounterDyn(Method * method,TaggedValue funcObj,ManagedThread * thread)1402 extern "C" bool DecrementHotnessCounterDyn(Method *method, TaggedValue funcObj, ManagedThread *thread)
1403 {
1404 method->DecrementHotnessCounter<true>(thread, 0, nullptr, false, funcObj);
1405 if (thread->HasPendingException()) {
1406 return false;
1407 }
1408 return method->GetCompiledEntryPoint() != GetCompiledCodeToInterpreterBridge(method);
1409 }
1410
CallCompilerSlowPath(ManagedThread * thread,Method * method)1411 extern "C" void CallCompilerSlowPath(ManagedThread *thread, Method *method)
1412 {
1413 CHECK_STACK_WALKER;
1414 method->DecrementHotnessCounter<false>(thread, 0, nullptr);
1415 }
1416
CallCompilerSlowPathOSR(ManagedThread * thread,Method * method,Frame * frame,uint64_t acc,uint64_t accTag,uint32_t insOffset,int offset)1417 extern "C" bool CallCompilerSlowPathOSR(ManagedThread *thread, Method *method, Frame *frame, uint64_t acc,
1418 uint64_t accTag, uint32_t insOffset, int offset)
1419 {
1420 CHECK_STACK_WALKER;
1421 if constexpr (ArchTraits<RUNTIME_ARCH>::SUPPORT_OSR) {
1422 ASSERT(ArchTraits<RUNTIME_ARCH>::SUPPORT_OSR);
1423 if (!frame->IsDeoptimized() && Runtime::GetOptions().IsCompilerEnableOsr()) {
1424 frame->SetAcc(ark::interpreter::AccVRegister(acc, accTag));
1425 return method->DecrementHotnessCounter<false>(thread, insOffset + offset, &frame->GetAcc(), true);
1426 }
1427 }
1428 method->DecrementHotnessCounter<false>(thread, 0, nullptr);
1429 return false;
1430 }
1431
UpdateBranchTaken(Method * method,Frame * frame,const uint8_t * pc,ProfilingData * profDataIrtoc)1432 extern "C" void UpdateBranchTaken([[maybe_unused]] Method *method, Frame *frame, const uint8_t *pc,
1433 ProfilingData *profDataIrtoc)
1434 {
1435 // Add a second prof_data loading because without it THREAD_SANITIZER crashes
1436 // NOTE(aantipina): investigate and delete the second loading (issue I6DTAA)
1437 [[maybe_unused]] ProfilingData *profData = method->GetProfilingDataWithoutCheck();
1438 profDataIrtoc->UpdateBranchTaken(pc - frame->GetInstruction());
1439 }
1440
UpdateBranchUntaken(Method * method,Frame * frame,const uint8_t * pc,ProfilingData * profDataIrtoc)1441 extern "C" void UpdateBranchUntaken([[maybe_unused]] Method *method, Frame *frame, const uint8_t *pc,
1442 ProfilingData *profDataIrtoc)
1443 {
1444 // Add a second prof_data loading because without it THREAD_SANITIZER crashes
1445 // NOTE(aantipina): investigate and delete the second loading
1446 [[maybe_unused]] ProfilingData *profData = method->GetProfilingDataWithoutCheck();
1447 profDataIrtoc->UpdateBranchNotTaken(pc - frame->GetInstruction());
1448 }
1449
ReadUlebEntrypoint(const uint8_t * ptr)1450 extern "C" uint32_t ReadUlebEntrypoint(const uint8_t *ptr)
1451 {
1452 uint32_t result;
1453 [[maybe_unused]] size_t n;
1454 [[maybe_unused]] bool isFull;
1455 std::tie(result, n, isFull) = leb128::DecodeUnsigned<uint32_t>(ptr);
1456 ASSERT(isFull);
1457 return result;
1458 }
1459
GetInstructionsByMethod(const Method * method)1460 extern "C" const uint8_t *GetInstructionsByMethod(const Method *method)
1461 {
1462 return method->GetInstructions();
1463 }
1464
GetNumVregsByMethod(const Method * method)1465 extern "C" size_t GetNumVregsByMethod(const Method *method)
1466 {
1467 return method->GetNumVregs();
1468 }
1469
ThrowExceptionFromInterpreter(ManagedThread * thread,ObjectHeader * exception,Frame * frame,const uint8_t * pc)1470 extern "C" void ThrowExceptionFromInterpreter(ManagedThread *thread, ObjectHeader *exception, Frame *frame,
1471 const uint8_t *pc)
1472 {
1473 CHECK_STACK_WALKER;
1474 ASSERT(!thread->HasPendingException());
1475 ASSERT(IsAddressInObjectsHeap(exception));
1476 Method *method = frame->GetMethod();
1477 ProfilingData *profData = method->GetProfilingDataWithoutCheck();
1478 if (profData != nullptr) {
1479 profData->UpdateThrowTaken(pc - frame->GetInstruction());
1480 }
1481 thread->SetException(exception);
1482 }
1483
ThrowArithmeticExceptionFromInterpreter()1484 extern "C" void ThrowArithmeticExceptionFromInterpreter()
1485 {
1486 CHECK_STACK_WALKER;
1487 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
1488 interpreter::RuntimeInterface::ThrowArithmeticException();
1489 }
1490
ThrowNullPointerExceptionFromInterpreter()1491 extern "C" void ThrowNullPointerExceptionFromInterpreter()
1492 {
1493 CHECK_STACK_WALKER;
1494 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
1495 interpreter::RuntimeInterface::ThrowNullPointerException();
1496 }
1497
ThrowNegativeArraySizeExceptionFromInterpreter(int32_t size)1498 extern "C" void ThrowNegativeArraySizeExceptionFromInterpreter(int32_t size)
1499 {
1500 CHECK_STACK_WALKER;
1501 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
1502 interpreter::RuntimeInterface::ThrowNegativeArraySizeException(size);
1503 }
1504
ThrowArrayIndexOutOfBoundsExceptionFromInterpreter(ssize_t idx,size_t length)1505 extern "C" void ThrowArrayIndexOutOfBoundsExceptionFromInterpreter(ssize_t idx, size_t length)
1506 {
1507 CHECK_STACK_WALKER;
1508 ASSERT(!ManagedThread::GetCurrent()->HasPendingException());
1509 interpreter::RuntimeInterface::ThrowArrayIndexOutOfBoundsException(idx, length);
1510 }
1511
CheckStoreArrayReferenceFromInterpreter(coretypes::Array * array,ObjectHeader * storeObj)1512 extern "C" uint8_t CheckStoreArrayReferenceFromInterpreter(coretypes::Array *array, ObjectHeader *storeObj)
1513 {
1514 CHECK_STACK_WALKER;
1515 ASSERT(array != nullptr);
1516 ASSERT(IsAddressInObjectsHeapOrNull(storeObj));
1517
1518 if (storeObj == nullptr) { // NOTE: may be moved to IRTOC
1519 return 0;
1520 }
1521
1522 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
1523 auto *arrayClass = array->ClassAddr<Class>();
1524 auto *elementClass = arrayClass->GetComponentType();
1525 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
1526 if (!storeObj->IsInstanceOf(elementClass)) {
1527 // SUPPRESS_CSA_NEXTLINE(alpha.core.WasteObjHeader)
1528 interpreter::RuntimeInterface::ThrowArrayStoreException(arrayClass, storeObj->ClassAddr<Class>());
1529 return 1;
1530 }
1531 return 0;
1532 }
1533
FindCatchBlockInIFramesStackless(ManagedThread ** currThread,Frame ** currFrame,const uint8_t * pc)1534 extern "C" uint32_t FindCatchBlockInIFramesStackless(ManagedThread **currThread, Frame **currFrame, const uint8_t *pc)
1535 {
1536 uint32_t inst = pc - (*currFrame)->GetInstruction();
1537 Frame *frame = *currFrame;
1538
1539 while (frame != nullptr) {
1540 ManagedThread *thread = *currThread;
1541 Frame *prev = frame->GetPrevFrame();
1542 ASSERT(thread->HasPendingException());
1543
1544 Method *method = frame->GetMethod();
1545 uint32_t pcOffset = interpreter::RuntimeInterface::FindCatchBlock(*method, thread->GetException(), inst);
1546 if (pcOffset != panda_file::INVALID_OFFSET) {
1547 return pcOffset;
1548 }
1549
1550 if (!frame->IsStackless() || prev == nullptr || StackWalker::IsBoundaryFrame<FrameKind::INTERPRETER>(prev)) {
1551 return pcOffset;
1552 }
1553
1554 EVENT_METHOD_EXIT(method->GetFullName(), events::MethodExitKind::INTERP, thread->RecordMethodExit());
1555
1556 Runtime::GetCurrent()->GetNotificationManager()->MethodExitEvent(thread, method);
1557
1558 inst = prev->GetBytecodeOffset();
1559 *currFrame = prev;
1560
1561 thread->GetVM()->HandleReturnFrame();
1562
1563 interpreter::RuntimeInterface::SetCurrentFrame(thread, prev);
1564
1565 ASSERT(thread->HasPendingException());
1566
1567 interpreter::RuntimeInterface::FreeFrame(*currThread, frame);
1568
1569 LOG(DEBUG, INTERPRETER) << "Exit: Runtime Call.";
1570
1571 frame = prev;
1572 }
1573
1574 return panda_file::INVALID_OFFSET;
1575 }
1576
FindCatchBlockInIFrames(ManagedThread * currThread,Frame * currFrame,const uint8_t * pc)1577 extern "C" const uint8_t *FindCatchBlockInIFrames(ManagedThread *currThread, Frame *currFrame, const uint8_t *pc)
1578 {
1579 CHECK_STACK_WALKER;
1580
1581 uint32_t pcOffset = panda_file::INVALID_OFFSET;
1582
1583 pcOffset = FindCatchBlockInIFramesStackless(&currThread, &currFrame, pc);
1584 if (pcOffset == panda_file::INVALID_OFFSET) {
1585 if constexpr (RUNTIME_ARCH == Arch::AARCH64 || RUNTIME_ARCH == Arch::AARCH32 || RUNTIME_ARCH == Arch::X86_64) {
1586 ark::FindCatchBlockInCallStack(currThread);
1587 }
1588 return pc;
1589 }
1590
1591 Method *method = currFrame->GetMethod();
1592 ASSERT(method != nullptr);
1593 LanguageContext ctx = interpreter::RuntimeInterface::GetLanguageContext(*method);
1594 ObjectHeader *exceptionObject = currThread->GetException();
1595 ctx.SetExceptionToVReg(currFrame->GetAcc(), exceptionObject);
1596
1597 currThread->ClearException();
1598 Span<const uint8_t> sp(currFrame->GetMethod()->GetInstructions(), pcOffset);
1599 return sp.cend();
1600 }
1601
VmCreateString(ManagedThread * thread,Method * method,ObjectHeader * ctorArg)1602 extern "C" coretypes::String *VmCreateString(ManagedThread *thread, Method *method, ObjectHeader *ctorArg)
1603 {
1604 CHECK_STACK_WALKER;
1605 // If ctor has the only argument (object itself), ctor_arg is allowed to contain garbage
1606 ASSERT(method->GetNumArgs() == 1 || IsAddressInObjectsHeapOrNull(ctorArg));
1607
1608 return thread->GetVM()->CreateString(method, ctorArg);
1609 }
1610
DebugPrintEntrypoint(ark::Frame * frame,const uint8_t * pc,uint64_t accPayload,uint64_t accTag)1611 extern "C" void DebugPrintEntrypoint([[maybe_unused]] ark::Frame *frame, [[maybe_unused]] const uint8_t *pc,
1612 [[maybe_unused]] uint64_t accPayload, [[maybe_unused]] uint64_t accTag)
1613 {
1614 CHECK_STACK_WALKER;
1615 #ifndef NDEBUG
1616 if (!Logger::IsLoggingOn(Logger::Level::DEBUG, Logger::Component::INTERPRETER)) {
1617 return;
1618 }
1619
1620 constexpr uint64_t STANDARD_DEBUG_INDENT = 5;
1621 PandaString accDump;
1622 auto accVreg = ark::interpreter::VRegister(accPayload);
1623 auto accTagVreg = ark::interpreter::VRegister(accTag);
1624 if (frame->IsDynamic()) {
1625 accDump = ark::interpreter::DynamicVRegisterRef(&accVreg).DumpVReg();
1626 LOG(DEBUG, INTERPRETER) << PandaString(STANDARD_DEBUG_INDENT, ' ') << "acc." << accDump;
1627
1628 DynamicFrameHandler frameHandler(frame);
1629 for (size_t i = 0; i < frame->GetSize(); ++i) {
1630 LOG(DEBUG, INTERPRETER) << PandaString(STANDARD_DEBUG_INDENT, ' ') << "v" << i << "."
1631 << frameHandler.GetVReg(i).DumpVReg();
1632 }
1633 } else {
1634 accDump = ark::interpreter::StaticVRegisterRef(&accVreg, &accTagVreg).DumpVReg();
1635 LOG(DEBUG, INTERPRETER) << PandaString(STANDARD_DEBUG_INDENT, ' ') << "acc." << accDump;
1636
1637 StaticFrameHandler frameHandler(frame);
1638 for (size_t i = 0; i < frame->GetSize(); ++i) {
1639 LOG(DEBUG, INTERPRETER) << PandaString(STANDARD_DEBUG_INDENT, ' ') << "v" << i << "."
1640 << frameHandler.GetVReg(i).DumpVReg();
1641 }
1642 }
1643 LOG(DEBUG, INTERPRETER) << " pc: " << (void *)pc << " ---> " << BytecodeInstruction(pc);
1644 #endif
1645 }
1646
JsCastDoubleToInt32(double d)1647 extern "C" PANDA_PUBLIC_API int32_t JsCastDoubleToInt32(double d)
1648 {
1649 return coretypes::JsCastDoubleToInt(d, coretypes::INT32_BITS);
1650 }
1651
JsCastDoubleToInt32Entrypoint(uint64_t d)1652 extern "C" int32_t JsCastDoubleToInt32Entrypoint(uint64_t d)
1653 {
1654 return coretypes::JsCastDoubleToInt(bit_cast<double>(d), coretypes::INT32_BITS);
1655 }
1656
1657 /* The shared implementation of the StringBuilder.append() intrinsics */
1658 template <class T>
ToClassPtr(uintptr_t obj)1659 static T *ToClassPtr(uintptr_t obj)
1660 {
1661 return reinterpret_cast<T *>(*reinterpret_cast<ObjectPointerType *>(obj));
1662 }
1663
1664 /* offset values to be aligned with the StringBuilder class layout */
1665 static constexpr uint32_t SB_COUNT_OFFSET = 12;
1666 static constexpr uint32_t SB_VALUE_OFFSET = 8;
1667
GetCountValue(ObjectHeader * sb)1668 static uint32_t GetCountValue(ObjectHeader *sb)
1669 {
1670 /* StringBuilder is not thread safe, we are supposed to
1671 * provide just the regular access to it's fields */
1672 return sb->GetFieldPrimitive<uint32_t>(SB_COUNT_OFFSET);
1673 }
1674
GetCapacity(ObjectHeader * self)1675 static uint32_t GetCapacity(ObjectHeader *self)
1676 {
1677 return ToClassPtr<coretypes::Array>(ToUintPtr(self) + SB_VALUE_OFFSET)->GetLength();
1678 }
1679
GetStorageAddress(ObjectHeader * sb,uint32_t count)1680 static uint16_t *GetStorageAddress(ObjectHeader *sb, uint32_t count)
1681 {
1682 auto storage = ToClassPtr<coretypes::Array>(ToUintPtr(sb) + SB_VALUE_OFFSET);
1683 return reinterpret_cast<uint16_t *>(ToUintPtr(storage) + coretypes::Array::GetDataOffset() + (count << 1U));
1684 }
1685
RecalculateCapacity(uint32_t capacity,uint32_t newsize)1686 static inline uint32_t RecalculateCapacity(uint32_t capacity, uint32_t newsize)
1687 {
1688 capacity = (capacity + 1) << 1U;
1689 capacity = newsize > capacity ? newsize : capacity;
1690 return capacity;
1691 }
1692
MoveStorageData(ObjectHeader * sb,void * newstorage)1693 static void MoveStorageData(ObjectHeader *sb, void *newstorage)
1694 {
1695 auto size = GetCountValue(sb) << 1U;
1696 auto newbuf = ToVoidPtr(ToUintPtr(newstorage) + coretypes::Array::GetDataOffset());
1697 auto oldbuf = ToVoidPtr(ToUintPtr(GetStorageAddress(sb, 0U)));
1698 std::copy_n(reinterpret_cast<uint8_t *>(oldbuf), size, reinterpret_cast<uint8_t *>(newbuf));
1699 }
1700
AssureCapacity(ObjectHeader * sb,uint32_t newsize,coretypes::String * str)1701 static std::tuple<bool, ObjectHeader *, coretypes::String *> AssureCapacity(ObjectHeader *sb, uint32_t newsize,
1702 coretypes::String *str)
1703 {
1704 if (GetCapacity(sb) >= newsize) {
1705 return std::make_tuple(true, sb, str);
1706 }
1707
1708 auto thread = ManagedThread::GetCurrent();
1709 HandleScope<ObjectHeader *> scope(thread);
1710 VMHandle<ObjectHeader> handle(thread, sb);
1711 VMHandle<coretypes::String> strHandle {};
1712 strHandle = str != nullptr ? VMHandle<coretypes::String>(thread, str) : strHandle;
1713
1714 auto *vm = Runtime::GetCurrent()->GetPandaVM();
1715 auto klass = Runtime::GetCurrent()
1716 ->GetClassLinker()
1717 ->GetExtension(vm->GetLanguageContext())
1718 ->GetClassRoot(ClassRoot::ARRAY_U16);
1719
1720 auto capacity = RecalculateCapacity(GetCapacity(sb), newsize);
1721 auto *newstorage = ark::coretypes::Array::Create(klass, capacity);
1722 sb = reinterpret_cast<ObjectHeader *>(handle.GetPtr());
1723 str = str != nullptr ? strHandle.GetPtr() : nullptr;
1724
1725 if (newstorage == nullptr) {
1726 return std::make_tuple(false, sb, str);
1727 }
1728
1729 MoveStorageData(sb, newstorage);
1730 handle->SetFieldObject(SB_VALUE_OFFSET, newstorage);
1731
1732 return std::make_tuple(true, sb, str);
1733 }
1734
StoreNumber(ObjectHeader * sb,int64_t n)1735 static ObjectHeader *StoreNumber(ObjectHeader *sb, int64_t n)
1736 {
1737 auto num = n < 0 ? -static_cast<uint64_t>(n) : static_cast<uint64_t>(n);
1738 auto size = CountDigits(num) + static_cast<uint32_t>(n < 0);
1739 auto count = GetCountValue(sb);
1740 auto newsize = count + size;
1741
1742 auto success = false;
1743 std::tie(success, sb, std::ignore) = AssureCapacity(sb, newsize, nullptr);
1744 if (!success) {
1745 return sb;
1746 }
1747 utf::UInt64ToUtf16Array(num, GetStorageAddress(sb, count), size, n < 0);
1748 sb->SetFieldPrimitive<uint32_t>(SB_COUNT_OFFSET, newsize);
1749 return sb;
1750 }
1751
CoreStringBuilderLong(ObjectHeader * sb,int64_t n)1752 extern "C" ObjectHeader *CoreStringBuilderLong(ObjectHeader *sb, int64_t n)
1753 {
1754 return StoreNumber(sb, n);
1755 }
1756
CoreStringBuilderInt(ObjectHeader * sb,int32_t n)1757 extern "C" ObjectHeader *CoreStringBuilderInt(ObjectHeader *sb, int32_t n)
1758 {
1759 return StoreNumber(sb, static_cast<int64_t>(n));
1760 }
1761
CoreStringBuilderBool(ObjectHeader * sb,const uint8_t v)1762 extern "C" ObjectHeader *CoreStringBuilderBool(ObjectHeader *sb, const uint8_t v)
1763 {
1764 const uint64_t truestr = 0x0065007500720074;
1765 const uint64_t falsestr = 0x0073006c00610066;
1766 const uint16_t falsestr2 = 0x0065;
1767
1768 auto count = GetCountValue(sb);
1769 auto size = 4U + (1U - v); // v is actually u1
1770 auto newsize = count + size;
1771
1772 auto success = false;
1773 std::tie(success, sb, std::ignore) = AssureCapacity(sb, newsize, nullptr);
1774 if (!success) {
1775 return sb;
1776 }
1777 auto dst = reinterpret_cast<uint64_t *>(GetStorageAddress(sb, count));
1778
1779 if (v != 0) {
1780 *dst = truestr;
1781 } else {
1782 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
1783 auto dst2 = reinterpret_cast<uint16_t *>(dst + 1);
1784 *dst = falsestr;
1785 *dst2 = falsestr2;
1786 }
1787
1788 sb->SetFieldPrimitive<uint32_t>(SB_COUNT_OFFSET, newsize);
1789 return sb;
1790 }
1791
CoreStringBuilderChar(ObjectHeader * sb,const uint16_t c)1792 extern "C" ObjectHeader *CoreStringBuilderChar(ObjectHeader *sb, const uint16_t c)
1793 {
1794 auto count = GetCountValue(sb);
1795 auto newsize = count + 1;
1796
1797 auto success = false;
1798 std::tie(success, sb, std::ignore) = AssureCapacity(sb, newsize, nullptr);
1799 if (!success) {
1800 return sb;
1801 }
1802 auto dst = GetStorageAddress(sb, count);
1803
1804 *dst = c;
1805 sb->SetFieldPrimitive<uint32_t>(SB_COUNT_OFFSET, newsize);
1806 return sb;
1807 }
1808
CoreStringBuilderString(ObjectHeader * sb,void * s)1809 extern "C" ObjectHeader *CoreStringBuilderString(ObjectHeader *sb, void *s)
1810 {
1811 const uint64_t nullstr = 0x006c006c0075006e;
1812 auto str = static_cast<coretypes::String *>(s);
1813 auto isnull = str == nullptr;
1814 auto size = isnull ? sizeof(nullstr) / sizeof(uint16_t) : str->GetLength();
1815 if (size == 0) {
1816 return sb;
1817 }
1818
1819 auto count = GetCountValue(sb);
1820 auto newsize = count + size;
1821
1822 auto success = false;
1823 std::tie(success, sb, str) = AssureCapacity(sb, newsize, str);
1824 if (!success) {
1825 return sb;
1826 }
1827
1828 auto dst = GetStorageAddress(sb, count);
1829
1830 if (!isnull) {
1831 auto src = ToVoidPtr(ToUintPtr(str) + coretypes::String::GetDataOffset());
1832 str->IsUtf16() ? std::copy_n(reinterpret_cast<uint16_t *>(src), size, dst)
1833 : std::copy_n(reinterpret_cast<uint8_t *>(src), size, dst);
1834 } else {
1835 /* store the 8-bytes chunk right away */
1836 *reinterpret_cast<uint64_t *>(dst) = nullstr;
1837 }
1838
1839 sb->SetFieldPrimitive<uint32_t>(SB_COUNT_OFFSET, newsize);
1840 return sb;
1841 }
1842
CoreStringConcat2(coretypes::String * str1,coretypes::String * str2)1843 extern "C" coretypes::String *CoreStringConcat2(coretypes::String *str1, coretypes::String *str2)
1844 {
1845 auto *vm = Runtime::GetCurrent()->GetPandaVM();
1846 auto ctx = vm->GetLanguageContext();
1847 return coretypes::String::Concat(str1, str2, ctx, vm);
1848 }
1849
CoreStringConcat3(coretypes::String * str1,coretypes::String * str2,coretypes::String * str3)1850 extern "C" coretypes::String *CoreStringConcat3(coretypes::String *str1, coretypes::String *str2,
1851 coretypes::String *str3)
1852 {
1853 auto *vm = Runtime::GetCurrent()->GetPandaVM();
1854 auto ctx = vm->GetLanguageContext();
1855 auto thread = ManagedThread::GetCurrent();
1856 [[maybe_unused]] HandleScope<ObjectHeader *> scope(thread);
1857 VMHandle<String> str3Handle(thread, str3);
1858 auto str = coretypes::String::Concat(str1, str2, ctx, vm);
1859 if (UNLIKELY(str == nullptr)) {
1860 HandlePendingException();
1861 UNREACHABLE();
1862 }
1863 str = coretypes::String::Concat(str, str3Handle.GetPtr(), ctx, vm);
1864 return str;
1865 }
1866
CoreStringConcat4(coretypes::String * str1,coretypes::String * str2,coretypes::String * str3,coretypes::String * str4)1867 extern "C" coretypes::String *CoreStringConcat4(coretypes::String *str1, coretypes::String *str2,
1868 coretypes::String *str3, coretypes::String *str4)
1869 {
1870 auto *vm = Runtime::GetCurrent()->GetPandaVM();
1871 auto ctx = vm->GetLanguageContext();
1872 auto thread = ManagedThread::GetCurrent();
1873 [[maybe_unused]] HandleScope<ObjectHeader *> scope(thread);
1874 VMHandle<String> str3Handle(thread, str3);
1875 VMHandle<String> str4Handle(thread, str4);
1876 auto str = coretypes::String::Concat(str1, str2, ctx, vm);
1877 if (UNLIKELY(str == nullptr)) {
1878 HandlePendingException();
1879 UNREACHABLE();
1880 }
1881 str3 = str3Handle.GetPtr();
1882 str = coretypes::String::Concat(str, str3, ctx, vm);
1883 if (UNLIKELY(str == nullptr)) {
1884 HandlePendingException();
1885 UNREACHABLE();
1886 }
1887 str4 = str4Handle.GetPtr();
1888 str = coretypes::String::Concat(str, str4, ctx, vm);
1889 return str;
1890 }
1891
1892 } // namespace ark
1893