/** * Copyright (c) 2021-2024 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "runtime/deoptimization.h" #include "include/cframe.h" #include "include/managed_thread.h" #include "include/stack_walker.h" #include "libpandabase/events/events.h" #include "libpandafile/file_items.h" #include "macros.h" #include "runtime/include/locks.h" #include "runtime/include/runtime.h" #include "runtime/include/panda_vm.h" #include "runtime/profiling/profiling-inl.h" #include "runtime/mem/rendezvous.h" namespace ark { /** * @brief Deoptimize CFrame that lies after another CFrame. * @param thread Pointer to thread * @param pc PC from which interpreter starts execution * @param frame Pointer to the first interpreter frame to that CFrame will be converted. It will be released via * FreeFrame inside this function * @param cframe_fp Pointer to Cframe to be deoptimized * @param last_frame Pointer to the last interpreter frame to that CFrame will be converted. It will be released via * FreeFrame inside this function * @param callee_regs Pointer to a callee-saved registers buffer from StackWalker */ // CC-OFFNXT(readability-function-size_parameters) asm bridge extern "C" [[noreturn]] void DeoptimizeAfterCFrame(ManagedThread *thread, const uint8_t *pc, Frame *frame, void *cframeFp, Frame *lastFrame, void *calleeRegs); /** * @brief Deoptimize CFrame that lies after interpreter frame. * @param thread Pointer to thread * @param pc PC from which interpreter starts execution * @param frame Pointer to the first interpreter frame to that CFrame will be converted. It will be released via * FreeFrame inside this function * @param cframe_fp Pointer to Cframe to be deoptimized * @param last_frame Pointer to the last interpreter frame to that CFrame will be converted. It will be released via * FreeFrame inside this function * @param callee_regs Pointer to a callee-saved registers buffer from StackWalker */ // CC-OFFNXT(readability-function-size_parameters) asm bridge extern "C" [[noreturn]] void DeoptimizeAfterIFrame(ManagedThread *thread, const uint8_t *pc, Frame *frame, void *cframeFp, Frame *lastFrame, void *calleeRegs); /** * @brief Drop given CFrame and return to its caller. * Drop means that we set stack pointer to the top of given CFrame, restores its return address and invoke `return` * instruction. * @param cframe_fp Pointer to Cframe to be dropped * @param callee_regs Pointer to a callee-saved registers buffer from StackWalker */ extern "C" [[noreturn]] void DropCompiledFrameAndReturn(void *cframeFp, void *calleeVregs); static void UnpoisonAsanStack([[maybe_unused]] void *ptr) { #ifdef PANDA_ASAN_ON uint8_t sp; ASAN_UNPOISON_MEMORY_REGION(&sp, reinterpret_cast(ptr) - &sp); #endif // PANDA_ASAN_ON } #ifdef PANDA_EVENTS_ENABLED static bool InvalidateCompiledMethod(ManagedThread *thread, Method *method, bool isCha, size_t &inStackCount) NO_THREAD_SAFETY_ANALYSIS #else static bool InvalidateCompiledMethod(ManagedThread *thread, Method *method, bool isCha) NO_THREAD_SAFETY_ANALYSIS #endif { ASSERT(thread != nullptr); // NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) for (auto stack = StackWalker::Create(thread); stack.HasFrame(); stack.NextFrame()) { if (stack.IsCFrame() && stack.GetMethod() == method) { auto &cframe = stack.GetCFrame(); cframe.SetShouldDeoptimize(true); cframe.SetDeoptCodeEntry(stack.GetCompiledCodeEntry()); if (isCha) { LOG(DEBUG, CLASS_LINKER) << "[CHA] Set ShouldDeoptimize for method: " << cframe.GetMethod()->GetFullName(); } else { LOG(DEBUG, CLASS_LINKER) << "[IC] Set ShouldDeoptimize for method: " << cframe.GetMethod()->GetFullName(); } #ifdef PANDA_EVENTS_ENABLED inStackCount++; #endif } } return true; } // NO_THREAD_SAFETY_ANALYSIS because it doesn't know about mutator_lock status in this scope void InvalidateCompiledEntryPoint(const PandaSet &methods, bool isCha) NO_THREAD_SAFETY_ANALYSIS { PandaVM *vm = Thread::GetCurrent()->GetVM(); ScopedSuspendAllThreadsRunning ssat(vm->GetRendezvous()); // NOTE(msherstennikov): remove this loop and check `methods` contains frame's method in stack traversing for (const auto &method : methods) { #ifdef PANDA_EVENTS_ENABLED size_t inStackCount = 0; vm->GetThreadManager()->EnumerateThreads([method, &inStackCount, isCha](ManagedThread *thread) { return InvalidateCompiledMethod(thread, method, isCha, inStackCount); #else vm->GetThreadManager()->EnumerateThreads([method, isCha](ManagedThread *thread) { return InvalidateCompiledMethod(thread, method, isCha); #endif }); if (isCha) { EVENT_CHA_DEOPTIMIZE(std::string(method->GetFullName()), inStackCount); } // NOTE (Trubenkov) clean up compiled code(See issue 1706) method->SetInterpreterEntryPoint(); Thread::GetCurrent()->GetVM()->GetCompiler()->RemoveOsrCode(method); // If deoptimization ocure during OSR compilation, we reset status after finish the compilation if (method->GetCompilationStatus() != Method::COMPILATION) { method->SetCompilationStatus(Method::NOT_COMPILED); } } } void PrevFrameDeopt(FrameKind prevFrameKind, ManagedThread *thread, StackWalker *stack, const uint8_t *pc, Frame *lastIframe, Frame *iframe, CFrame &cframe) { switch (prevFrameKind) { case FrameKind::COMPILER: LOG(DEBUG, INTEROP) << "Deoptimize after cframe"; EVENT_DEOPTIMIZATION(std::string(cframe.GetMethod()->GetFullName()), pc - stack->GetMethod()->GetInstructions(), events::DeoptimizationAfter::CFRAME); // We need to set current frame kind to `compiled` as it's possible that we came here from other Deoptimize // call and in this case frame kind will be `non-compiled`: // // compiled code // | // Deoptimize // | // DeoptimizeAfterCFrame // | // (change frame kind to `non-compiled`) interpreter::Execute -- we don't return after this call // | // FindCatchBlockInCallStack // | // Deoptimize thread->SetCurrentFrameIsCompiled(true); DeoptimizeAfterCFrame(thread, pc, iframe, cframe.GetFrameOrigin(), lastIframe, stack->GetCalleeRegsForDeoptimize().end()); case FrameKind::NONE: case FrameKind::INTERPRETER: EVENT_DEOPTIMIZATION(std::string(cframe.GetMethod()->GetFullName()), pc - stack->GetMethod()->GetInstructions(), prevFrameKind == FrameKind::NONE ? events::DeoptimizationAfter::TOP : events::DeoptimizationAfter::IFRAME); LOG(DEBUG, INTEROP) << "Deoptimize after iframe"; DeoptimizeAfterIFrame(thread, pc, iframe, cframe.GetFrameOrigin(), lastIframe, stack->GetCalleeRegsForDeoptimize().end()); } } NO_ADDRESS_SANITIZE void DestroyMethodWithInvalidatingEP(Method *destroyMethod) { LOG(DEBUG, INTEROP) << "Destroy compiled method: " << destroyMethod->GetFullName(); destroyMethod->SetDestroyed(); PandaSet destroyMethods; destroyMethods.insert(destroyMethod); InvalidateCompiledEntryPoint(destroyMethods, false); } [[noreturn]] NO_ADDRESS_SANITIZE void Deoptimize(StackWalker *stack, const uint8_t *pc, bool hasException, Method *destroyMethod) { ASSERT(stack != nullptr); auto *thread = ManagedThread::GetCurrent(); ASSERT(thread != nullptr); ASSERT(stack->IsCFrame()); auto &cframe = stack->GetCFrame(); UnpoisonAsanStack(cframe.GetFrameOrigin()); auto method = stack->GetMethod(); if (pc == nullptr) { ASSERT(method != nullptr); // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) pc = method->GetInstructions() + stack->GetBytecodePc(); } LOG(INFO, INTEROP) << "Deoptimize frame: " << method->GetFullName() << ", pc=" << std::hex << pc - method->GetInstructions() << std::dec; thread->GetVM()->ClearInteropHandleScopes(thread->GetCurrentFrame()); auto context = thread->GetVM()->GetLanguageContext(); // We must run InvalidateCompiledEntryPoint before we convert the frame, because GC is already may be in the // collecting phase and it can move some object in the deoptimized frame. if (destroyMethod != nullptr) { DestroyMethodWithInvalidatingEP(destroyMethod); } FrameKind prevFrameKind; // We need to execute(find catch block) in all inlined methods. For this we calculate the number of inlined method // Else we can execute previus interpreter frames and we will FreeFrames in incorrect order uint32_t numInlinedMethods = 0; Frame *iframe = stack->ConvertToIFrame(&prevFrameKind, &numInlinedMethods); ASSERT(iframe != nullptr); Frame *lastIframe = iframe; while (numInlinedMethods-- != 0) { EVENT_METHOD_EXIT(last_iframe->GetMethod()->GetFullName() + "(deopt)", events::MethodExitKind::INLINED, thread->RecordMethodExit()); lastIframe = lastIframe->GetPrevFrame(); ASSERT(!StackWalker::IsBoundaryFrame(lastIframe)); } EVENT_METHOD_EXIT(last_iframe->GetMethod()->GetFullName() + "(deopt)", events::MethodExitKind::COMPILED, thread->RecordMethodExit()); if (thread->HasPendingException()) { LOG(DEBUG, INTEROP) << "Deoptimization has pending exception: " << thread->GetException()->ClassAddr()->GetName(); context.SetExceptionToVReg(iframe->GetAcc(), thread->GetException()); } if (!hasException) { thread->ClearException(); } else { ASSERT(thread->HasPendingException()); } PrevFrameDeopt(prevFrameKind, thread, stack, pc, lastIframe, iframe, cframe); UNREACHABLE(); } [[noreturn]] void DropCompiledFrame(StackWalker *stack) { LOG(DEBUG, INTEROP) << "Drop compiled frame: " << stack->GetMethod()->GetFullName(); auto cframeFp = stack->GetCFrame().GetFrameOrigin(); EVENT_METHOD_EXIT(stack->GetMethod()->GetFullName() + "(drop)", events::MethodExitKind::COMPILED, ManagedThread::GetCurrent()->RecordMethodExit()); UnpoisonAsanStack(cframeFp); DropCompiledFrameAndReturn(cframeFp, stack->GetCalleeRegsForDeoptimize().end()); UNREACHABLE(); } } // namespace ark