1 /*
2 * Copyright (c) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "runtime/mem/heap_manager.h"
17
18 #include <string>
19
20 #include "heap_manager.h"
21 #include "include/runtime.h"
22 #include "include/locks.h"
23 #include "include/thread.h"
24 #include "libpandabase/mem/mmap_mem_pool-inl.h"
25 #include "libpandabase/mem/pool_manager.h"
26 #include "libpandabase/utils/logger.h"
27 #include "mem/pool_manager.h"
28 #include "mem/mmap_mem_pool-inl.h"
29 #include "mem/internal_allocator-inl.h"
30 #include "mem/gc/hybrid-gc/hybrid_object_allocator.h"
31 #include "runtime/include/locks.h"
32 #include "runtime/include/runtime.h"
33 #include "runtime/include/runtime_notification.h"
34 #include "runtime/include/thread.h"
35 #include "runtime/include/thread_scopes.h"
36 #include "runtime/mem/internal_allocator-inl.h"
37 #include "runtime/handle_base-inl.h"
38 #include "runtime/include/panda_vm.h"
39 #include "runtime/mem/gc/g1/g1-gc.h"
40
41 namespace panda::mem {
42
Initialize(GCType gc_type,bool single_threaded,bool use_tlab,MemStatsType * mem_stats,InternalAllocatorPtr internal_allocator,bool create_pygote_space)43 bool HeapManager::Initialize(GCType gc_type, bool single_threaded, bool use_tlab, MemStatsType *mem_stats,
44 InternalAllocatorPtr internal_allocator, bool create_pygote_space)
45 {
46 trace::ScopedTrace scoped_trace("HeapManager::Initialize");
47 bool ret = false;
48 mem_stats_ = mem_stats;
49 internalAllocator_ = internal_allocator;
50 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
51 #define FWD_GC_INIT(type, mem_stats) \
52 case type: \
53 if (single_threaded) { \
54 ret = Initialize<type, MT_MODE_SINGLE>(mem_stats, create_pygote_space); \
55 } else { \
56 ret = Initialize<type, MT_MODE_MULTI>(mem_stats, create_pygote_space); \
57 } \
58 break
59
60 switch (gc_type) {
61 FWD_GC_INIT(GCType::EPSILON_GC, mem_stats);
62 FWD_GC_INIT(GCType::STW_GC, mem_stats);
63 FWD_GC_INIT(GCType::GEN_GC, mem_stats);
64 FWD_GC_INIT(GCType::HYBRID_GC, mem_stats);
65 FWD_GC_INIT(GCType::G1_GC, mem_stats);
66 default:
67 LOG(FATAL, GC) << "Invalid init for gc_type = " << static_cast<int>(gc_type);
68 break;
69 }
70 #undef FWD_GC_INIT
71 if (!objectAllocator_.AsObjectAllocator()->IsTLABSupported() || single_threaded) {
72 use_tlab = false;
73 }
74 use_tlab_for_allocations_ = use_tlab;
75 // Now, USE_TLAB_FOR_ALLOCATIONS option is supported only for Generational GCs
76 ASSERT(IsGenerationalGCType(gc_type) || (!use_tlab_for_allocations_));
77 return ret;
78 }
79
SetPandaVM(PandaVM * vm)80 void HeapManager::SetPandaVM(PandaVM *vm)
81 {
82 vm_ = vm;
83 gc_ = vm_->GetGC();
84 notification_manager_ = Runtime::GetCurrent()->GetNotificationManager();
85 }
86
Finalize()87 bool HeapManager::Finalize()
88 {
89 delete codeAllocator_;
90 objectAllocator_->VisitAndRemoveAllPools(
91 [](void *mem, [[maybe_unused]] size_t size) { PoolManager::GetMmapMemPool()->FreePool(mem, size); });
92 delete static_cast<Allocator *>(objectAllocator_);
93 objectAllocator_ = nullptr;
94
95 return true;
96 }
97
AllocateObject(BaseClass * cls,size_t size,Alignment align,MTManagedThread * thread)98 ObjectHeader *HeapManager::AllocateObject(BaseClass *cls, size_t size, Alignment align, MTManagedThread *thread)
99 {
100 ASSERT(vm_->GetLanguageContext().GetLanguage() == panda_file::SourceLang::ECMASCRIPT || !GetGC()->IsGCRunning() ||
101 Locks::mutator_lock->HasLock());
102 TriggerGCIfNeeded();
103 if (thread == nullptr) {
104 thread = MTManagedThread::GetCurrent();
105 ASSERT(thread != nullptr);
106 }
107 void *mem = AllocateMemoryForObject(size, align, thread);
108 if (UNLIKELY(mem == nullptr)) {
109 mem = TryGCAndAlloc(size, align, thread);
110 if (UNLIKELY(mem == nullptr)) {
111 ThrowOutOfMemoryError("AllocateObject failed");
112 return nullptr;
113 }
114 }
115 LOG(DEBUG, ALLOC_OBJECT) << "Alloc object at " << std::hex << mem << " size: " << size;
116 ObjectHeader *object = InitObjectHeaderAtMem(cls, mem);
117 bool is_object_finalizable = IsObjectFinalized(cls);
118 if (UNLIKELY(is_object_finalizable || GetNotificationManager()->HasAllocationListeners())) {
119 // Use object handle here as RegisterFinalizedObject can trigger GC
120 [[maybe_unused]] HandleScope<ObjectHeader *> scope(thread);
121 VMHandle<ObjectHeader> handle(thread, object);
122 RegisterFinalizedObject(handle.GetPtr(), cls, is_object_finalizable);
123 GetNotificationManager()->ObjectAllocEvent(cls, handle.GetPtr(), thread, size);
124 object = handle.GetPtr();
125 }
126 return object;
127 }
128
TryGCAndAlloc(size_t size,Alignment align,panda::MTManagedThread * thread)129 void *HeapManager::TryGCAndAlloc(size_t size, Alignment align, panda::MTManagedThread *thread)
130 {
131 // do not try many times in case of OOM scenarios.
132 constexpr size_t ALLOC_RETRY = 4;
133 size_t alloc_try_cnt = 0;
134 void *mem = nullptr;
135 bool is_generational = GetGC()->IsGenerational();
136 ASSERT(!thread->HasPendingException());
137
138 while (mem == nullptr && alloc_try_cnt++ < ALLOC_RETRY) {
139 GCTaskCause cause;
140 // add comment why -1
141 if (alloc_try_cnt == ALLOC_RETRY - 1 || !is_generational) {
142 cause = GCTaskCause::OOM_CAUSE;
143 } else {
144 cause = GCTaskCause::YOUNG_GC_CAUSE;
145 }
146 GetGC()->WaitForGCInManaged(GCTask(cause, thread));
147 mem = AllocateMemoryForObject(size, align, thread);
148 if (mem != nullptr) {
149 // we could set OOM in gc, but we need to clear it if next gc was successful and we allocated memory
150 thread->ClearException();
151 } else {
152 auto reclaimed_bytes = GetGC()->GetLastGCReclaimedBytes();
153 // if last GC reclaimed some bytes - it means that we have a progress in JVM, just this thread was unlucky
154 // to get some memory. We reset alloc_try_cnt to try again.
155 if (reclaimed_bytes != 0) {
156 alloc_try_cnt = 0;
157 }
158 }
159 }
160 return mem;
161 }
162
AllocateMemoryForObject(size_t size,Alignment align,ManagedThread * thread)163 void *HeapManager::AllocateMemoryForObject(size_t size, Alignment align, ManagedThread *thread)
164 {
165 void *mem = nullptr;
166 if (UseTLABForAllocations() && size <= GetTLABMaxAllocSize()) {
167 ASSERT(thread != nullptr);
168 ASSERT(GetGC()->IsTLABsSupported());
169 // Try to allocate an object via TLAB
170 TLAB *current_tlab = thread->GetTLAB();
171 ASSERT(current_tlab != nullptr); // A thread's TLAB must be initialized at least via some ZERO tlab values.
172 mem = current_tlab->Alloc(size);
173 if (mem == nullptr) {
174 // We couldn't allocate an object via current TLAB,
175 // Therefore, create a new one and allocate in it.
176 if (CreateNewTLAB(thread)) {
177 current_tlab = thread->GetTLAB();
178 mem = current_tlab->Alloc(size);
179 }
180 }
181 if (PANDA_TRACK_TLAB_ALLOCATIONS && (mem != nullptr)) {
182 mem_stats_->RecordAllocateObject(GetAlignedObjectSize(size), SpaceType::SPACE_TYPE_OBJECT);
183 }
184 }
185 if (mem == nullptr) { // if mem == nullptr, try to use common allocate scenario
186 mem = objectAllocator_->Allocate(size, align, thread);
187 }
188 return mem;
189 }
190
191 template <bool IsFirstClassClass>
AllocateNonMovableObject(BaseClass * cls,size_t size,Alignment align,ManagedThread * thread)192 ObjectHeader *HeapManager::AllocateNonMovableObject(BaseClass *cls, size_t size, Alignment align, ManagedThread *thread)
193 {
194 ASSERT(vm_->GetLanguageContext().GetLanguage() == panda_file::SourceLang::ECMASCRIPT || !GetGC()->IsGCRunning() ||
195 Locks::mutator_lock->HasLock());
196 TriggerGCIfNeeded();
197 void *mem = objectAllocator_->AllocateNonMovable(size, align, thread);
198 if (UNLIKELY(mem == nullptr)) {
199 GCTaskCause cause = GCTaskCause::OOM_CAUSE;
200 GetGC()->WaitForGCInManaged(GCTask(cause, thread));
201 mem = objectAllocator_->AllocateNonMovable(size, align, thread);
202 }
203 if (UNLIKELY(mem == nullptr)) {
204 ThrowOutOfMemoryError("AllocateNonMovableObject failed");
205 return nullptr;
206 }
207 LOG(DEBUG, ALLOC_OBJECT) << "Alloc non-movable object at " << std::hex << mem;
208 auto *object = InitObjectHeaderAtMem(cls, mem);
209 // cls can be null for first class creation, when we create ClassRoot::Class
210 // NOLINTNEXTLINE(readability-braces-around-statements, readability-misleading-indentation)
211 if constexpr (IsFirstClassClass) {
212 ASSERT(cls == nullptr);
213 // NOLINTNEXTLINE(readability-braces-around-statements, readability-misleading-indentation)
214 } else {
215 ASSERT(cls != nullptr);
216 bool is_object_finalizable = IsObjectFinalized(cls);
217 RegisterFinalizedObject(object, cls, is_object_finalizable);
218 GetNotificationManager()->ObjectAllocEvent(cls, object, thread, size);
219 }
220 return object;
221 }
222
InitObjectHeaderAtMem(BaseClass * cls,void * mem)223 ObjectHeader *HeapManager::InitObjectHeaderAtMem(BaseClass *cls, void *mem)
224 {
225 ASSERT(mem != nullptr);
226 ASSERT(vm_->GetLanguageContext().GetLanguage() == panda_file::SourceLang::ECMASCRIPT || !GetGC()->IsGCRunning() ||
227 Locks::mutator_lock->HasLock());
228
229 auto object = static_cast<ObjectHeader *>(mem);
230 // we need zeroed memory here according to ISA
231 ASSERT(object->AtomicGetMark().GetValue() == 0);
232 ASSERT(object->AtomicClassAddr<BaseClass *>() == nullptr);
233 // The order is crucial here - we need to have 0 class word to avoid data race with concurrent sweep.
234 // Otherwise we can remove not initialized object.
235 GetGC()->InitGCBits(object);
236 object->SetClass(cls);
237 return object;
238 }
239
TriggerGCIfNeeded()240 void HeapManager::TriggerGCIfNeeded()
241 {
242 if (vm_->GetGCTrigger()->IsGcTriggered()) {
243 GetGC()->Trigger();
244 }
245 }
246
AllocateFrame(size_t size)247 Frame *HeapManager::AllocateFrame(size_t size)
248 {
249 ASSERT(vm_->GetLanguageContext().GetLanguage() == panda_file::SourceLang::ECMASCRIPT || !GetGC()->IsGCRunning() ||
250 Locks::mutator_lock->HasLock());
251 StackFrameAllocator *frame_allocator = GetCurrentStackFrameAllocator();
252 return static_cast<Frame *>(frame_allocator->Alloc(size));
253 }
254
CreateNewTLAB(ManagedThread * thread)255 bool HeapManager::CreateNewTLAB(ManagedThread *thread)
256 {
257 ASSERT(vm_->GetLanguageContext().GetLanguage() == panda_file::SourceLang::ECMASCRIPT || !GetGC()->IsGCRunning() ||
258 Locks::mutator_lock->HasLock());
259 ASSERT(thread != nullptr);
260 TLAB *new_tlab = objectAllocator_.AsObjectAllocator()->CreateNewTLAB(thread);
261 if (new_tlab != nullptr) {
262 RegisterTLAB(thread->GetTLAB());
263 thread->UpdateTLAB(new_tlab);
264 return true;
265 }
266 return false;
267 }
268
RegisterTLAB(TLAB * tlab)269 void HeapManager::RegisterTLAB(TLAB *tlab)
270 {
271 ASSERT(tlab != nullptr);
272 if (!PANDA_TRACK_TLAB_ALLOCATIONS && (tlab->GetOccupiedSize() != 0)) {
273 mem_stats_->RecordAllocateObject(tlab->GetOccupiedSize(), SpaceType::SPACE_TYPE_OBJECT);
274 }
275 }
276
FreeFrame(Frame * frame_ptr)277 void HeapManager::FreeFrame(Frame *frame_ptr)
278 {
279 ASSERT(vm_->GetLanguageContext().GetLanguage() == panda_file::SourceLang::ECMASCRIPT || !GetGC()->IsGCRunning() ||
280 Locks::mutator_lock->HasLock());
281 StackFrameAllocator *frame_allocator = GetCurrentStackFrameAllocator();
282 frame_allocator->Free(frame_ptr);
283 }
284
GetCodeAllocator() const285 CodeAllocator *HeapManager::GetCodeAllocator() const
286 {
287 return codeAllocator_;
288 }
289
GetInternalAllocator()290 InternalAllocatorPtr HeapManager::GetInternalAllocator()
291 {
292 return internalAllocator_;
293 }
294
GetObjectAllocator()295 ObjectAllocatorPtr HeapManager::GetObjectAllocator()
296 {
297 return objectAllocator_;
298 }
299
GetCurrentStackFrameAllocator()300 StackFrameAllocator *HeapManager::GetCurrentStackFrameAllocator()
301 {
302 return ManagedThread::GetCurrent()->GetStackFrameAllocator();
303 }
304
PreZygoteFork()305 void HeapManager::PreZygoteFork()
306 {
307 GetGC()->WaitForGCOnPygoteFork(GCTask(GCTaskCause::PYGOTE_FORK_CAUSE));
308 }
309
GetTargetHeapUtilization() const310 float HeapManager::GetTargetHeapUtilization() const
311 {
312 return target_utilization_;
313 }
314
SetTargetHeapUtilization(float target)315 void HeapManager::SetTargetHeapUtilization(float target)
316 {
317 ASSERT_PRINT(target > 0.0F && target < 1.0F, "Target heap utilization should be in the range (0,1)");
318 target_utilization_ = target;
319 }
320
GetTotalMemory() const321 size_t HeapManager::GetTotalMemory() const
322 {
323 return vm_->GetGCTrigger()->GetTargetFootprint();
324 }
325
GetFreeMemory() const326 size_t HeapManager::GetFreeMemory() const
327 {
328 return helpers::UnsignedDifference(GetTotalMemory(), vm_->GetMemStats()->GetFootprintHeap());
329 }
330
DumpHeap(PandaOStringStream * o_string_stream)331 void HeapManager::DumpHeap(PandaOStringStream *o_string_stream)
332 {
333 size_t obj_cnt = 0;
334 *o_string_stream << "Dumping heap" << std::endl;
335 objectAllocator_->IterateOverObjects([&obj_cnt, &o_string_stream](ObjectHeader *mem) {
336 DumpObject(static_cast<ObjectHeader *>(mem), o_string_stream);
337 obj_cnt++;
338 });
339 *o_string_stream << "Total dumped " << obj_cnt << std::endl;
340 }
341
342 /**
343 * \brief Check whether the given object is an instance of the given class.
344 * @param obj - ObjectHeader pointer
345 * @param h_class - Class pointer
346 * @param assignable - whether the subclass of h_class counts
347 * @return true if obj is instanceOf h_class, otherwise false
348 */
MatchesClass(ObjectHeader * obj,Class * h_class,bool assignable)349 static bool MatchesClass(ObjectHeader *obj, Class *h_class, bool assignable)
350 {
351 if (assignable) {
352 return obj->IsInstanceOf(h_class);
353 }
354 return obj->ClassAddr<Class>() == h_class;
355 }
356
CountInstances(const PandaVector<Class * > & classes,bool assignable,uint64_t * counts)357 void HeapManager::CountInstances(const PandaVector<Class *> &classes, bool assignable, uint64_t *counts)
358 {
359 auto objects_checker = [&](ObjectHeader *obj) {
360 for (size_t i = 0; i < classes.size(); ++i) {
361 if (classes[i] == nullptr) {
362 continue;
363 }
364 if (MatchesClass(obj, classes[i], assignable)) {
365 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
366 ++counts[i];
367 }
368 }
369 };
370 {
371 MTManagedThread *thread = MTManagedThread::GetCurrent();
372 ASSERT(thread != nullptr);
373 ScopedChangeThreadStatus sts(thread, ThreadStatus::RUNNING);
374 ScopedSuspendAllThreadsRunning ssatr(Runtime::GetCurrent()->GetPandaVM()->GetRendezvous());
375 GetObjectAllocator().AsObjectAllocator()->IterateOverObjects(objects_checker);
376 }
377 }
378
SetIsFinalizableFunc(IsObjectFinalizebleFunc func)379 void HeapManager::SetIsFinalizableFunc(IsObjectFinalizebleFunc func)
380 {
381 IsObjectFinalizebleFunc_ = func;
382 }
383
SetRegisterFinalizeReferenceFunc(RegisterFinalizeReferenceFunc func)384 void HeapManager::SetRegisterFinalizeReferenceFunc(RegisterFinalizeReferenceFunc func)
385 {
386 RegisterFinalizeReferenceFunc_ = func;
387 }
388
IsObjectFinalized(BaseClass * cls)389 bool HeapManager::IsObjectFinalized(BaseClass *cls)
390 {
391 return IsObjectFinalizebleFunc_ != nullptr && IsObjectFinalizebleFunc_(cls);
392 }
393
RegisterFinalizedObject(ObjectHeader * object,BaseClass * cls,bool is_object_finalizable)394 void HeapManager::RegisterFinalizedObject(ObjectHeader *object, BaseClass *cls, bool is_object_finalizable)
395 {
396 if (is_object_finalizable) {
397 ASSERT(RegisterFinalizeReferenceFunc_ != nullptr);
398 RegisterFinalizeReferenceFunc_(object, cls);
399 }
400 }
401
402 template ObjectHeader *HeapManager::AllocateNonMovableObject<true>(BaseClass *cls, size_t size, Alignment align,
403 ManagedThread *thread);
404
405 template ObjectHeader *HeapManager::AllocateNonMovableObject<false>(BaseClass *cls, size_t size, Alignment align,
406 ManagedThread *thread);
407 } // namespace panda::mem
408