/* * Copyright (c) 2022 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef ECMASCRIPT_MEM_WORK_MANAGER_INL_H #define ECMASCRIPT_MEM_WORK_MANAGER_INL_H #include "ecmascript/mem/work_manager.h" #include "ecmascript/mem/incremental_marker.h" #include "ecmascript/mem/tlab_allocator-inl.h" namespace panda::ecmascript { void WorkNodeHolder::Setup(Heap *heap, WorkManager *workManager, GlobalWorkStack *workStack) { heap_ = heap; workManager_ = workManager; workStack_ = workStack; continuousQueue_ = new ProcessQueue(); continuousJSWeakMapQueue_ = new JSWeakMapProcessQueue(); } void WorkNodeHolder::Destroy() { continuousQueue_->Destroy(); delete continuousQueue_; continuousQueue_ = nullptr; continuousJSWeakMapQueue_->Destroy(); delete continuousJSWeakMapQueue_; continuousJSWeakMapQueue_ = nullptr; } void WorkNodeHolder::Initialize(TriggerGCType gcType, ParallelGCTaskPhase taskPhase) { inNode_ = workManager_->AllocateWorkNode(); cachedInNode_ = workManager_->AllocateWorkNode(); outNode_ = workManager_->AllocateWorkNode(); weakQueue_ = new ProcessQueue(); weakQueue_->BeginMarking(continuousQueue_); jsWeakMapQueue_ = new JSWeakMapProcessQueue(); jsWeakMapQueue_->BeginMarking(continuousJSWeakMapQueue_); aliveSize_ = 0; promotedSize_ = 0; parallelGCTaskPhase_ = taskPhase; if (gcType != TriggerGCType::OLD_GC) { allocator_ = new TlabAllocator(heap_); } } void WorkNodeHolder::Finish() { if (weakQueue_ != nullptr) { weakQueue_->FinishMarking(continuousQueue_); delete weakQueue_; weakQueue_ = nullptr; } if (jsWeakMapQueue_ != nullptr) { jsWeakMapQueue_->FinishMarking(continuousJSWeakMapQueue_); delete jsWeakMapQueue_; jsWeakMapQueue_ = nullptr; } if (allocator_ != nullptr) { allocator_->Finalize(); delete allocator_; allocator_ = nullptr; } parallelGCTaskPhase_ = ParallelGCTaskPhase::UNDEFINED_TASK; } bool WorkNodeHolder::Push(TaggedObject *object) { if (!inNode_->PushObject(ToUintPtr(object))) { PushWorkNodeToGlobal(); return inNode_->PushObject(ToUintPtr(object)); } return true; } void WorkNodeHolder::PushWorkNodeToGlobal(bool postTask) { if (!inNode_->IsEmpty()) { workStack_->Push(inNode_); inNode_ = cachedInNode_; ASSERT(inNode_ != nullptr); cachedInNode_ = cachedInNode_->Next(); if (cachedInNode_ == nullptr) { cachedInNode_ = workManager_->AllocateWorkNode(); } if (postTask && heap_->IsParallelGCEnabled() && heap_->CheckCanDistributeTask() && !(heap_->IsMarking() && heap_->GetIncrementalMarker()->IsTriggeredIncrementalMark())) { heap_->PostParallelGCTask(parallelGCTaskPhase_); } } } bool WorkNodeHolder::Pop(TaggedObject **object) { if (!outNode_->PopObject(reinterpret_cast(object))) { if (!inNode_->IsEmpty()) { WorkNode *tmp = outNode_; outNode_ = inNode_; inNode_ = tmp; } else { cachedInNode_->SetNext(outNode_); outNode_->SetNext(nullptr); if (!PopWorkNodeFromGlobal()) { return false; } } return outNode_->PopObject(reinterpret_cast(object)); } return true; } bool WorkNodeHolder::PopWorkNodeFromGlobal() { return workStack_->Pop(&outNode_); } void WorkNodeHolder::PushWeakReference(JSTaggedType *weak) { weakQueue_->PushBack(weak); } void WorkNodeHolder::PushJSWeakMap(TaggedObject *jsWeakMap) { jsWeakMapQueue_->PushBack(jsWeakMap); } void WorkNodeHolder::IncreaseAliveSize(size_t size) { aliveSize_ += size; } void WorkNodeHolder::IncreasePromotedSize(size_t size) { promotedSize_ += size; } ProcessQueue *WorkNodeHolder::GetWeakReferenceQueue() const { return weakQueue_; } JSWeakMapProcessQueue *WorkNodeHolder::GetJSWeakMapQueue() const { return jsWeakMapQueue_; } TlabAllocator *WorkNodeHolder::GetTlabAllocator() const { return allocator_; } JSThread *WorkNodeHolder::GetJSThread() const { return heap_->GetJSThread(); } WorkManagerBase::WorkManagerBase(NativeAreaAllocator *allocator) : spaceChunk_(allocator), workSpace_(0), spaceStart_(0), spaceEnd_(0) { auto allocatedSpace = GetSpaceChunk()->Allocate(WORKNODE_SPACE_SIZE); ASSERT(allocatedSpace != nullptr); workSpace_ = ToUintPtr(allocatedSpace); } WorkNode *WorkManagerBase::AllocateWorkNode() { LockHolder lock(mtx_); size_t allocatedSize = sizeof(WorkNode) + sizeof(Stack) + STACK_AREA_SIZE; ASSERT(allocatedSize < WORKNODE_SPACE_SIZE); uintptr_t begin = spaceStart_; if (begin + allocatedSize >= spaceEnd_) { agedSpaces_.emplace_back(workSpace_); workSpace_ = ToUintPtr(GetSpaceChunk()->Allocate(WORKNODE_SPACE_SIZE)); spaceStart_ = workSpace_; spaceEnd_ = workSpace_ + WORKNODE_SPACE_SIZE; begin = spaceStart_; } spaceStart_ = begin + allocatedSize; Stack *stack = reinterpret_cast(begin + sizeof(WorkNode)); stack->ResetBegin(begin + sizeof(WorkNode) + sizeof(Stack), begin + allocatedSize); WorkNode *work = reinterpret_cast(begin); return new (work) WorkNode(stack); } WorkManagerBase::~WorkManagerBase() { if (workSpace_ != 0) { GetSpaceChunk()->Free(reinterpret_cast(workSpace_)); } } WorkManager::WorkManager(Heap *heap, uint32_t threadNum) : WorkManagerBase(heap->GetNativeAreaAllocator()), heap_(heap), threadNum_(threadNum), parallelGCTaskPhase_(UNDEFINED_TASK) { for (uint32_t i = 0; i < threadNum_; i++) { works_.at(i).Setup(heap_, this, &workStack_); } } WorkManager::~WorkManager() { Finish(); for (uint32_t i = 0; i < threadNum_; i++) { works_.at(i).Destroy(); } } size_t WorkManager::Finish() { size_t aliveSize = 0; for (uint32_t i = 0; i < threadNum_; i++) { WorkNodeHolder &holder = works_.at(i); holder.Finish(); aliveSize += holder.aliveSize_; } workStack_.Clear(); FinishBase(); initialized_.store(false, std::memory_order_release); return aliveSize; } void WorkManager::Finish(size_t &aliveSize, size_t &promotedSize) { aliveSize = Finish(); for (uint32_t i = 0; i < threadNum_; i++) { WorkNodeHolder &holder = works_.at(i); promotedSize += holder.promotedSize_; if (holder.allocator_ != nullptr) { holder.allocator_->Finalize(); delete holder.allocator_; holder.allocator_ = nullptr; } } initialized_.store(false, std::memory_order_release); } void WorkManager::Initialize(TriggerGCType gcType, ParallelGCTaskPhase taskPhase) { parallelGCTaskPhase_ = taskPhase; InitializeBase(); for (uint32_t i = 0; i < threadNum_; i++) { WorkNodeHolder &holder = works_.at(i); holder.Initialize(gcType, taskPhase); } if (initialized_.load(std::memory_order_acquire)) { // LOCV_EXCL_BR_LINE LOG_ECMA(FATAL) << "this branch is unreachable"; UNREACHABLE(); } initialized_.store(true, std::memory_order_release); } SharedGCWorkManager::SharedGCWorkManager(SharedHeap *heap, uint32_t threadNum) : WorkManagerBase(heap->GetNativeAreaAllocator()), sHeap_(heap), threadNum_(threadNum), continuousQueue_ { nullptr }, sharedTaskPhase_(SHARED_UNDEFINED_TASK) { for (uint32_t i = 0; i < threadNum_; i++) { continuousQueue_.at(i) = new ProcessQueue(); } } SharedGCWorkManager::~SharedGCWorkManager() { Finish(); for (uint32_t i = 0; i < threadNum_; i++) { continuousQueue_.at(i)->Destroy(); delete continuousQueue_.at(i); continuousQueue_.at(i) = nullptr; } } void SharedGCWorkManager::Initialize(TriggerGCType gcType, SharedParallelMarkPhase taskPhase) { sharedTaskPhase_ = taskPhase; InitializeBase(); for (uint32_t i = 0; i < threadNum_; i++) { SharedGCWorkNodeHolder &holder = works_.at(i); holder.inNode_ = AllocateWorkNode(); holder.cachedInNode_ = AllocateWorkNode(); holder.outNode_ = AllocateWorkNode(); holder.weakQueue_ = new ProcessQueue(); holder.weakQueue_->BeginMarking(continuousQueue_.at(i)); if (gcType == TriggerGCType::SHARED_FULL_GC) { holder.allocator_ = new SharedTlabAllocator(sHeap_); } } if (initialized_.load(std::memory_order_acquire)) { // LOCV_EXCL_BR_LINE LOG_ECMA(FATAL) << "this branch is unreachable"; UNREACHABLE(); } initialized_.store(true, std::memory_order_release); } size_t SharedGCWorkManager::Finish() { size_t aliveSize = 0; for (uint32_t i = 0; i < threadNum_; i++) { SharedGCWorkNodeHolder &holder = works_.at(i); if (holder.weakQueue_ != nullptr) { holder.weakQueue_->FinishMarking(continuousQueue_.at(i)); delete holder.weakQueue_; holder.weakQueue_ = nullptr; } aliveSize += holder.aliveSize_; if (holder.allocator_ != nullptr) { holder.allocator_->Finalize(); delete holder.allocator_; holder.allocator_ = nullptr; } } workStack_.Clear(); FinishBase(); initialized_.store(false, std::memory_order_release); return aliveSize; } bool SharedGCWorkManager::Push(uint32_t threadId, TaggedObject *object) { WorkNode *&inNode = works_.at(threadId).inNode_; if (!inNode->PushObject(ToUintPtr(object))) { PushWorkNodeToGlobal(threadId); return inNode->PushObject(ToUintPtr(object)); } return true; } bool SharedGCWorkManager::PushToLocalMarkingBuffer(WorkNode *&markingBuffer, TaggedObject *object) { if (UNLIKELY(markingBuffer == nullptr)) { markingBuffer = AllocateWorkNode(); } ASSERT(markingBuffer != nullptr); if (UNLIKELY(!markingBuffer->PushObject(ToUintPtr(object)))) { PushLocalBufferToGlobal(markingBuffer); ASSERT(markingBuffer == nullptr); markingBuffer = AllocateWorkNode(); return markingBuffer->PushObject(ToUintPtr(object)); } return true; } void SharedGCWorkManager::PushWorkNodeToGlobal(uint32_t threadId, bool postTask) { WorkNode *&inNode = works_.at(threadId).inNode_; if (!inNode->IsEmpty()) { workStack_.Push(inNode); inNode = works_.at(threadId).cachedInNode_; ASSERT(inNode != nullptr); works_.at(threadId).cachedInNode_ = AllocateWorkNode(); if (postTask && sHeap_->IsParallelGCEnabled() && sHeap_->CheckCanDistributeTask()) { sHeap_->PostGCMarkingTask(sharedTaskPhase_); } } } void SharedGCWorkManager::PushLocalBufferToGlobal(WorkNode *&node, bool postTask) { ASSERT(node != nullptr); ASSERT(!node->IsEmpty()); workStack_.Push(node); if (postTask && sHeap_->IsParallelGCEnabled() && sHeap_->CheckCanDistributeTask()) { sHeap_->PostGCMarkingTask(sharedTaskPhase_); } node = nullptr; } bool SharedGCWorkManager::Pop(uint32_t threadId, TaggedObject **object) { WorkNode *&outNode = works_.at(threadId).outNode_; WorkNode *&inNode = works_.at(threadId).inNode_; if (!outNode->PopObject(reinterpret_cast(object))) { if (!inNode->IsEmpty()) { WorkNode *tmp = outNode; outNode = inNode; inNode = tmp; } else if (!PopWorkNodeFromGlobal(threadId)) { return false; } return outNode->PopObject(reinterpret_cast(object)); } return true; } bool SharedGCWorkManager::PopWorkNodeFromGlobal(uint32_t threadId) { return workStack_.Pop(&works_.at(threadId).outNode_); } } // namespace panda::ecmascript #endif // ECMASCRIPT_MEM_WORK_MANAGER_INL_H