1 /**
2 * Copyright (c) 2021-2025 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include <memory>
17
18 #include "libpandabase/os/cpu_affinity.h"
19 #include "libpandabase/os/mem.h"
20 #include "libpandabase/os/thread.h"
21 #include "libpandabase/utils/time.h"
22 #include "libpandabase/taskmanager/task_manager.h"
23 #include "runtime/assert_gc_scope.h"
24 #include "runtime/include/class.h"
25 #include "runtime/include/coretypes/dyn_objects.h"
26 #include "runtime/include/locks.h"
27 #include "runtime/include/runtime.h"
28 #include "runtime/include/runtime_notification.h"
29 #include "runtime/include/stack_walker-inl.h"
30 #include "runtime/mem/gc/epsilon/epsilon.h"
31 #include "runtime/mem/gc/epsilon-g1/epsilon-g1.h"
32 #include "runtime/mem/gc/gc.h"
33 #include "runtime/mem/gc/gc_root-inl.h"
34 #include "runtime/mem/gc/g1/g1-gc.h"
35 #include "runtime/mem/gc/gen-gc/gen-gc.h"
36 #include "runtime/mem/gc/stw-gc/stw-gc.h"
37 #include "runtime/mem/gc/cmc-gc-adapter/cmc-gc-adapter.h"
38 #include "runtime/mem/gc/workers/gc_workers_task_queue.h"
39 #include "runtime/mem/gc/workers/gc_workers_thread_pool.h"
40 #include "runtime/mem/pygote_space_allocator-inl.h"
41 #include "runtime/mem/heap_manager.h"
42 #include "runtime/mem/gc/reference-processor/reference_processor.h"
43 #include "runtime/mem/gc/gc-hung/gc_hung.h"
44 #include "runtime/include/panda_vm.h"
45 #include "runtime/include/object_accessor-inl.h"
46 #include "runtime/include/coretypes/class.h"
47 #include "runtime/thread_manager.h"
48 #include "runtime/mem/gc/gc_adaptive_stack_inl.h"
49
50 namespace ark::mem {
51 using TaggedValue = coretypes::TaggedValue;
52 using TaggedType = coretypes::TaggedType;
53 using DynClass = coretypes::DynClass;
54
GC(ObjectAllocatorBase * objectAllocator,const GCSettings & settings)55 GC::GC(ObjectAllocatorBase *objectAllocator, const GCSettings &settings)
56 : gcSettings_(settings),
57 objectAllocator_(objectAllocator),
58 internalAllocator_(InternalAllocator<>::GetInternalAllocatorFromRuntime())
59 {
60 if (gcSettings_.UseTaskManagerForGC()) {
61 // Create gc task queue for task manager
62 gcWorkersTaskQueue_ = taskmanager::TaskManager::CreateTaskQueue<decltype(internalAllocator_->Adapter())>(
63 taskmanager::MAX_QUEUE_PRIORITY);
64 ASSERT(gcWorkersTaskQueue_ != nullptr);
65 }
66 }
67
~GC()68 GC::~GC()
69 {
70 InternalAllocatorPtr allocator = GetInternalAllocator();
71 if (gcWorker_ != nullptr) {
72 allocator->Delete(gcWorker_);
73 }
74 if (gcListenerManager_ != nullptr) {
75 allocator->Delete(gcListenerManager_);
76 }
77 if (gcBarrierSet_ != nullptr) {
78 allocator->Delete(gcBarrierSet_);
79 }
80 if (clearedReferences_ != nullptr) {
81 allocator->Delete(clearedReferences_);
82 }
83 if (clearedReferencesLock_ != nullptr) {
84 allocator->Delete(clearedReferencesLock_);
85 }
86 if (workersTaskPool_ != nullptr) {
87 allocator->Delete(workersTaskPool_);
88 }
89 if (gcWorkersTaskQueue_ != nullptr) {
90 taskmanager::TaskManager::DestroyTaskQueue<decltype(allocator->Adapter())>(gcWorkersTaskQueue_);
91 }
92 }
93
GetLogPrefix() const94 Logger::Buffer GC::GetLogPrefix() const
95 {
96 const char *phase = GCScopedPhase::GetPhaseAbbr(GetGCPhase());
97 // Atomic with acquire order reason: data race with gc_counter_
98 size_t counter = gcCounter_.load(std::memory_order_acquire);
99
100 Logger::Buffer buffer;
101 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
102 buffer.Printf("[%zu, %s]: ", counter, phase);
103
104 return buffer;
105 }
106
GetType()107 GCType GC::GetType()
108 {
109 return gcType_;
110 }
111
SetPandaVM(PandaVM * vm)112 void GC::SetPandaVM(PandaVM *vm)
113 {
114 vm_ = vm;
115 referenceProcessor_ = vm->GetReferenceProcessor();
116 }
117
GetNativeGcTriggerType()118 NativeGcTriggerType GC::GetNativeGcTriggerType()
119 {
120 return gcSettings_.GetNativeGcTriggerType();
121 }
122
SimpleNativeAllocationGcWatermark()123 size_t GC::SimpleNativeAllocationGcWatermark()
124 {
125 return GetPandaVm()->GetOptions().GetMaxFree();
126 }
127
WaitForIdleGC()128 NO_THREAD_SAFETY_ANALYSIS void GC::WaitForIdleGC()
129 {
130 while (!CASGCPhase(GCPhase::GC_PHASE_IDLE, GCPhase::GC_PHASE_RUNNING)) {
131 GetPandaVm()->GetRendezvous()->SafepointEnd();
132 // Interrupt the running GC if possible
133 OnWaitForIdleFail();
134 // NOTE(dtrubenkov): resolve it more properly
135 constexpr uint64_t WAIT_FINISHED = 100;
136 // Use NativeSleep for all threads, as this thread shouldn't hold Mutator lock here
137 os::thread::NativeSleepUS(std::chrono::microseconds(WAIT_FINISHED));
138 GetPandaVm()->GetRendezvous()->SafepointBegin();
139 }
140 }
141
TriggerGCForNative()142 inline void GC::TriggerGCForNative()
143 {
144 auto nativeGcTriggerType = GetNativeGcTriggerType();
145 ASSERT_PRINT((nativeGcTriggerType == NativeGcTriggerType::NO_NATIVE_GC_TRIGGER) ||
146 (nativeGcTriggerType == NativeGcTriggerType::SIMPLE_STRATEGY),
147 "Unknown Native GC Trigger type");
148 switch (nativeGcTriggerType) {
149 case NativeGcTriggerType::NO_NATIVE_GC_TRIGGER:
150 break;
151 case NativeGcTriggerType::SIMPLE_STRATEGY:
152 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or
153 // ordering constraints imposed on other reads or writes
154 if (nativeBytesRegistered_.load(std::memory_order_relaxed) > SimpleNativeAllocationGcWatermark()) {
155 auto task = MakePandaUnique<GCTask>(GCTaskCause::NATIVE_ALLOC_CAUSE, time::GetCurrentTimeInNanos());
156 AddGCTask(false, std::move(task));
157 ManagedThread::GetCurrent()->SafepointPoll();
158 }
159 break;
160 default:
161 LOG(FATAL, GC) << "Unknown Native GC Trigger type";
162 break;
163 }
164 }
165
Initialize(PandaVM * vm)166 void GC::Initialize(PandaVM *vm)
167 {
168 trace::ScopedTrace scopedTrace(__PRETTY_FUNCTION__);
169 // GC saved the PandaVM instance, so we get allocator from the PandaVM.
170 auto allocator = GetInternalAllocator();
171 gcListenerManager_ = allocator->template New<GCListenerManager>();
172 clearedReferencesLock_ = allocator->New<os::memory::Mutex>();
173 ASSERT(clearedReferencesLock_ != nullptr);
174 os::memory::LockHolder holder(*clearedReferencesLock_);
175 clearedReferences_ = allocator->New<PandaVector<ark::mem::Reference *>>(allocator->Adapter());
176 this->SetPandaVM(vm);
177 InitializeImpl();
178 gcWorker_ = allocator->New<GCWorker>(this);
179 }
180
CreateWorkersTaskPool()181 void GC::CreateWorkersTaskPool()
182 {
183 ASSERT(workersTaskPool_ == nullptr);
184 if (this->IsWorkerThreadsExist()) {
185 auto allocator = GetInternalAllocator();
186 GCWorkersTaskPool *gcTaskPool = nullptr;
187 if (this->GetSettings()->UseThreadPoolForGC()) {
188 // Use internal gc thread pool
189 gcTaskPool = allocator->New<GCWorkersThreadPool>(this, this->GetSettings()->GCWorkersCount());
190 } else {
191 // Use common TaskManager
192 ASSERT(this->GetSettings()->UseTaskManagerForGC());
193 gcTaskPool = allocator->New<GCWorkersTaskQueue>(this);
194 }
195 ASSERT(gcTaskPool != nullptr);
196 workersTaskPool_ = gcTaskPool;
197 }
198 }
199
DestroyWorkersTaskPool()200 void GC::DestroyWorkersTaskPool()
201 {
202 if (workersTaskPool_ == nullptr) {
203 return;
204 }
205 workersTaskPool_->WaitUntilTasksEnd();
206 auto allocator = this->GetInternalAllocator();
207 allocator->Delete(workersTaskPool_);
208 workersTaskPool_ = nullptr;
209 }
210
StartGC()211 void GC::StartGC()
212 {
213 CreateWorker();
214 }
215
StopGC()216 void GC::StopGC()
217 {
218 DestroyWorker();
219 DestroyWorkersTaskPool();
220 }
221
SetupCpuAffinity()222 void GC::SetupCpuAffinity()
223 {
224 if (!gcSettings_.ManageGcThreadsAffinity()) {
225 return;
226 }
227 // Try to get CPU affinity fo GC Thread
228 if (UNLIKELY(!os::CpuAffinityManager::GetCurrentThreadAffinity(affinityBeforeGc_))) {
229 affinityBeforeGc_.Clear();
230 return;
231 }
232 // Try to use best + middle for preventing issues when best core is used in another thread,
233 // and GC waits for it to finish.
234 if (!os::CpuAffinityManager::SetAffinityForCurrentThread(os::CpuPower::BEST | os::CpuPower::MIDDLE)) {
235 affinityBeforeGc_.Clear();
236 }
237 // Some GCs don't use GC Workers
238 if (workersTaskPool_ != nullptr && this->GetSettings()->UseThreadPoolForGC()) {
239 static_cast<GCWorkersThreadPool *>(workersTaskPool_)->SetAffinityForGCWorkers();
240 }
241 }
242
SetupCpuAffinityAfterConcurrent()243 void GC::SetupCpuAffinityAfterConcurrent()
244 {
245 if (!gcSettings_.ManageGcThreadsAffinity()) {
246 return;
247 }
248 os::CpuAffinityManager::SetAffinityForCurrentThread(os::CpuPower::BEST | os::CpuPower::MIDDLE);
249 // Some GCs don't use GC Workers
250 if (workersTaskPool_ != nullptr && this->GetSettings()->UseThreadPoolForGC()) {
251 static_cast<GCWorkersThreadPool *>(workersTaskPool_)->SetAffinityForGCWorkers();
252 }
253 }
254
ResetCpuAffinity(bool beforeConcurrent)255 void GC::ResetCpuAffinity(bool beforeConcurrent)
256 {
257 if (!gcSettings_.ManageGcThreadsAffinity()) {
258 return;
259 }
260 if (!affinityBeforeGc_.IsEmpty()) {
261 // Set GC Threads on weak CPUs before concurrent if needed
262 if (beforeConcurrent && gcSettings_.UseWeakCpuForGcConcurrent()) {
263 os::CpuAffinityManager::SetAffinityForCurrentThread(os::CpuPower::WEAK);
264 } else { // else set on saved affinity
265 os::CpuAffinityManager::SetAffinityForCurrentThread(affinityBeforeGc_);
266 }
267 }
268 // Some GCs don't use GC Workers
269 if (workersTaskPool_ != nullptr && this->GetSettings()->UseThreadPoolForGC()) {
270 static_cast<GCWorkersThreadPool *>(workersTaskPool_)->UnsetAffinityForGCWorkers();
271 }
272 }
273
SetupCpuAffinityBeforeConcurrent()274 void GC::SetupCpuAffinityBeforeConcurrent()
275 {
276 ResetCpuAffinity(true);
277 }
278
RestoreCpuAffinity()279 void GC::RestoreCpuAffinity()
280 {
281 ResetCpuAffinity(false);
282 }
283
GetFastGCFlag() const284 bool GC::GetFastGCFlag() const
285 {
286 // Atomic with relaxed order reason: data race with no synchronization or ordering constraints imposed
287 // on other reads or writes
288 return fastGC_.load(std::memory_order_relaxed);
289 }
290
SetFastGCFlag(bool fastGC)291 void GC::SetFastGCFlag(bool fastGC)
292 {
293 // Atomic with relaxed order reason: data race with no synchronization or ordering constraints imposed
294 // on other reads or writes
295 fastGC_.store(fastGC, std::memory_order_relaxed);
296 }
297
NeedRunGCAfterWaiting(size_t counterBeforeWaiting,const GCTask & task) const298 bool GC::NeedRunGCAfterWaiting(size_t counterBeforeWaiting, const GCTask &task) const
299 {
300 // Atomic with acquire order reason: data race with gc_counter_ with dependecies on reads after the load which
301 // should become visible
302 auto newCounter = gcCounter_.load(std::memory_order_acquire);
303 ASSERT(newCounter >= counterBeforeWaiting);
304 bool shouldRunGCAccordingToFlag = !(GetFastGCFlag() && task.reason == GCTaskCause::HEAP_USAGE_THRESHOLD_CAUSE);
305 // Atomic with acquire order reason: data race with last_cause_ with dependecies on reads after the load which
306 // should become visible
307 return (newCounter == counterBeforeWaiting || lastCause_.load(std::memory_order_acquire) < task.reason) &&
308 shouldRunGCAccordingToFlag;
309 }
310
GCPhasesPreparation(const GCTask & task)311 bool GC::GCPhasesPreparation(const GCTask &task)
312 {
313 // Atomic with acquire order reason: data race with gc_counter_ with dependecies on reads after the load which
314 // should become visible
315 auto oldCounter = gcCounter_.load(std::memory_order_acquire);
316 WaitForIdleGC();
317 if (!this->NeedRunGCAfterWaiting(oldCounter, task)) {
318 SetGCPhase(GCPhase::GC_PHASE_IDLE);
319 return false;
320 }
321 this->SetupCpuAffinity();
322 this->GetTiming()->Reset(); // Clear records.
323 // Atomic with release order reason: data race with last_cause_ with dependecies on writes before the store which
324 // should become visible acquire
325 lastCause_.store(task.reason, std::memory_order_release);
326 if (gcSettings_.PreGCHeapVerification()) {
327 trace::ScopedTrace preHeapVerifierTrace("PreGCHeapVeriFier");
328 size_t failCount = VerifyHeap();
329 if (gcSettings_.FailOnHeapVerification() && failCount > 0) {
330 LOG(FATAL, GC) << "Heap corrupted before GC, HeapVerifier found " << failCount << " corruptions";
331 }
332 }
333 // Atomic with acq_rel order reason: data race with gc_counter_ with dependecies on reads after the load and on
334 // writes before the store
335 gcCounter_.fetch_add(1, std::memory_order_acq_rel);
336 if (gcSettings_.IsDumpHeap()) {
337 PandaOStringStream os;
338 os << "Heap dump before GC" << std::endl;
339 GetPandaVm()->DumpHeap(&os);
340 std::cerr << os.str() << std::endl;
341 }
342 return true;
343 }
344
GCPhasesFinish(const GCTask & task)345 void GC::GCPhasesFinish(const GCTask &task)
346 {
347 ASSERT(task.collectionType != GCCollectionType::NONE);
348 LOG(INFO, GC) << "[" << gcCounter_ << "] [" << task.collectionType << " (" << task.reason << ")] "
349 << GetPandaVm()->GetGCStats()->GetStatistics();
350
351 if (gcSettings_.IsDumpHeap()) {
352 PandaOStringStream os;
353 os << "Heap dump after GC" << std::endl;
354 GetPandaVm()->DumpHeap(&os);
355 std::cerr << os.str() << std::endl;
356 }
357
358 if (gcSettings_.PostGCHeapVerification()) {
359 trace::ScopedTrace postHeapVerifierTrace("PostGCHeapVeriFier");
360 size_t failCount = VerifyHeap();
361 if (gcSettings_.FailOnHeapVerification() && failCount > 0) {
362 LOG(FATAL, GC) << "Heap corrupted after GC, HeapVerifier found " << failCount << " corruptions";
363 }
364 }
365 this->RestoreCpuAffinity();
366
367 SetGCPhase(GCPhase::GC_PHASE_IDLE);
368 }
369
370 // NOLINTNEXTLINE(performance-unnecessary-value-param)
RunPhases(GCTask & task)371 void GC::RunPhases(GCTask &task)
372 {
373 DCHECK_ALLOW_GARBAGE_COLLECTION;
374 trace::ScopedTrace scopedTrace(__FUNCTION__);
375 bool needRunGCAfterWaiting = GCPhasesPreparation(task);
376 if (!needRunGCAfterWaiting) {
377 return;
378 }
379 size_t bytesInHeapBeforeGc = GetPandaVm()->GetMemStats()->GetFootprintHeap();
380 LOG_DEBUG_GC << "Bytes in heap before GC " << std::dec << bytesInHeapBeforeGc;
381 {
382 GCScopedStats scopedStats(GetPandaVm()->GetGCStats(), gcType_ == GCType::STW_GC ? GetStats() : nullptr);
383 ScopedGcHung scopedHung(&task);
384 GetPandaVm()->GetGCStats()->ResetLastPause();
385
386 FireGCStarted(task, bytesInHeapBeforeGc);
387 PreRunPhasesImpl();
388 clearSoftReferencesEnabled_ = task.reason == GCTaskCause::OOM_CAUSE || IsExplicitFull(task);
389 // NOLINTNEXTLINE(performance-unnecessary-value-param)
390 RunPhasesImpl(task);
391 // Clear Internal allocator unused pools (must do it on pause to avoid race conditions):
392 // - Clear global part:
393 InternalAllocator<>::GetInternalAllocatorFromRuntime()->VisitAndRemoveFreePools(
394 [](void *mem, [[maybe_unused]] size_t size) { PoolManager::GetMmapMemPool()->FreePool(mem, size); });
395 // - Clear local part:
396 ClearLocalInternalAllocatorPools();
397
398 size_t bytesInHeapAfterGc = GetPandaVm()->GetMemStats()->GetFootprintHeap();
399 // There is case than bytes_in_heap_after_gc > 0 and bytes_in_heap_before_gc == 0.
400 // Because TLABs are registered during GC
401 if (bytesInHeapAfterGc > 0 && bytesInHeapBeforeGc > 0) {
402 GetStats()->AddReclaimRatioValue(1 - static_cast<double>(bytesInHeapAfterGc) / bytesInHeapBeforeGc);
403 }
404 LOG_DEBUG_GC << "Bytes in heap after GC " << std::dec << bytesInHeapAfterGc;
405 FireGCFinished(task, bytesInHeapBeforeGc, bytesInHeapAfterGc);
406 }
407 GCPhasesFinish(task);
408 }
409
410 template <class LanguageConfig>
CreateGC(GCType gcType,ObjectAllocatorBase * objectAllocator,const GCSettings & settings)411 GC *CreateGC(GCType gcType, ObjectAllocatorBase *objectAllocator, const GCSettings &settings)
412 {
413 GC *ret = nullptr;
414 InternalAllocatorPtr allocator {InternalAllocator<>::GetInternalAllocatorFromRuntime()};
415
416 switch (gcType) {
417 case GCType::EPSILON_GC:
418 ret = allocator->New<EpsilonGC<LanguageConfig>>(objectAllocator, settings);
419 break;
420 case GCType::EPSILON_G1_GC:
421 ret = allocator->New<EpsilonG1GC<LanguageConfig>>(objectAllocator, settings);
422 break;
423 case GCType::STW_GC:
424 ret = allocator->New<StwGC<LanguageConfig>>(objectAllocator, settings);
425 break;
426 case GCType::GEN_GC:
427 ret = allocator->New<GenGC<LanguageConfig>>(objectAllocator, settings);
428 break;
429 case GCType::G1_GC:
430 ret = allocator->New<G1GC<LanguageConfig>>(objectAllocator, settings);
431 break;
432 case GCType::CMC_GC:
433 ret = allocator->New<CMCGCAdapter<LanguageConfig>>(objectAllocator, settings);
434 break;
435 default:
436 LOG(FATAL, GC) << "Unknown GC type";
437 break;
438 }
439 return ret;
440 }
441
CheckGCCause(GCTaskCause cause) const442 bool GC::CheckGCCause(GCTaskCause cause) const
443 {
444 return cause != GCTaskCause::INVALID_CAUSE;
445 }
446
IsMarkedEx(const ObjectHeader * object) const447 bool GC::IsMarkedEx(const ObjectHeader *object) const
448 {
449 return IsMarked(object);
450 }
451
MarkObjectIfNotMarked(ObjectHeader * objectHeader)452 bool GC::MarkObjectIfNotMarked(ObjectHeader *objectHeader)
453 {
454 ASSERT(objectHeader != nullptr);
455 if (IsMarked(objectHeader)) {
456 return false;
457 }
458 MarkObject(objectHeader);
459 return true;
460 }
461
ProcessReference(GCMarkingStackType * objectsStack,const BaseClass * cls,const ObjectHeader * ref,const ReferenceProcessPredicateT & pred)462 void GC::ProcessReference(GCMarkingStackType *objectsStack, const BaseClass *cls, const ObjectHeader *ref,
463 const ReferenceProcessPredicateT &pred)
464 {
465 ASSERT(referenceProcessor_ != nullptr);
466 referenceProcessor_->HandleReference(this, objectsStack, cls, ref, pred);
467 }
468
ProcessReferenceForSinglePassCompaction(const BaseClass * cls,const ObjectHeader * ref,const ReferenceProcessorT & processor)469 void GC::ProcessReferenceForSinglePassCompaction(const BaseClass *cls, const ObjectHeader *ref,
470 const ReferenceProcessorT &processor)
471 {
472 ASSERT(referenceProcessor_ != nullptr);
473 referenceProcessor_->HandleReference(this, cls, ref, processor);
474 }
475
AddReference(ObjectHeader * fromObj,ObjectHeader * object)476 void GC::AddReference(ObjectHeader *fromObj, ObjectHeader *object)
477 {
478 ASSERT(IsMarked(object));
479 GCMarkingStackType references(this);
480 // NOTE(alovkov): support stack with workers here & put all refs in stack and only then process altogether for once
481 ASSERT(!references.IsWorkersTaskSupported());
482 references.PushToStack(fromObj, object);
483 MarkReferences(&references, phase_);
484 if (gcType_ != GCType::EPSILON_GC) {
485 ASSERT(references.Empty());
486 }
487 }
488
489 // NOLINTNEXTLINE(performance-unnecessary-value-param)
ProcessReferences(GCPhase gcPhase,const GCTask & task,const ReferenceClearPredicateT & pred)490 void GC::ProcessReferences(GCPhase gcPhase, const GCTask &task, const ReferenceClearPredicateT &pred)
491 {
492 trace::ScopedTrace scopedTrace(__FUNCTION__);
493 LOG(DEBUG, REF_PROC) << "Start processing cleared references";
494 ASSERT(referenceProcessor_ != nullptr);
495 bool clearSoftReferences = task.reason == GCTaskCause::OOM_CAUSE || IsExplicitFull(task);
496 referenceProcessor_->ProcessReferences(false, clearSoftReferences, gcPhase, pred);
497 Reference *processedRef = referenceProcessor_->CollectClearedReferences();
498 if (processedRef != nullptr) {
499 os::memory::LockHolder holder(*clearedReferencesLock_);
500 // NOTE(alovkov): ged rid of cleared_references_ and just enqueue refs here?
501 clearedReferences_->push_back(processedRef);
502 }
503 }
504
ProcessReferences(const mem::GC::ReferenceClearPredicateT & pred)505 void GC::ProcessReferences(const mem::GC::ReferenceClearPredicateT &pred)
506 {
507 ASSERT(!this->IsFullGC());
508 trace::ScopedTrace scopedTrace(__FUNCTION__);
509 LOG(DEBUG, REF_PROC) << "Start processing cleared references";
510 ASSERT(referenceProcessor_ != nullptr);
511 referenceProcessor_->ProcessReferencesAfterCompaction(pred);
512 Reference *processedRef = referenceProcessor_->CollectClearedReferences();
513 if (processedRef != nullptr) {
514 os::memory::LockHolder holder(*clearedReferencesLock_);
515 clearedReferences_->push_back(processedRef);
516 }
517 }
518
EvacuateStartingWith(void * ref)519 void GC::EvacuateStartingWith([[maybe_unused]] void *ref)
520 {
521 ASSERT_PRINT(false, "Should be implemented by subclasses");
522 }
523
IsClearSoftReferencesEnabled() const524 bool GC::IsClearSoftReferencesEnabled() const
525 {
526 return clearSoftReferencesEnabled_;
527 }
528
SetGCPhase(GCPhase gcPhase)529 void GC::SetGCPhase(GCPhase gcPhase)
530 {
531 phase_ = gcPhase;
532 }
533
GetCounter() const534 size_t GC::GetCounter() const
535 {
536 return gcCounter_;
537 }
538
PostponeGCStart()539 void GC::PostponeGCStart()
540 {
541 ASSERT(IsPostponeGCSupported());
542 isPostponeEnabled_ = true;
543 }
544
PostponeGCEnd()545 void GC::PostponeGCEnd()
546 {
547 ASSERT(IsPostponeGCSupported());
548 // Don't check IsPostponeEnabled because runtime can be created
549 // during app launch. In this case PostponeGCStart is not called.
550 isPostponeEnabled_ = false;
551 gcWorker_->OnPostponeGCEnd();
552 }
553
IsPostponeEnabled() const554 bool GC::IsPostponeEnabled() const
555 {
556 return isPostponeEnabled_;
557 }
558
DestroyWorker()559 void GC::DestroyWorker()
560 {
561 // Atomic with seq_cst order reason: data race with gc_running_ with requirement for sequentially consistent order
562 // where threads observe all modifications in the same order
563 gcRunning_.store(false, std::memory_order_seq_cst);
564 gcWorker_->FinalizeAndDestroyWorker();
565 }
566
CreateWorker()567 void GC::CreateWorker()
568 {
569 // Atomic with seq_cst order reason: data race with gc_running_ with requirement for sequentially consistent order
570 // where threads observe all modifications in the same order
571 gcRunning_.store(true, std::memory_order_seq_cst);
572 ASSERT(gcWorker_ != nullptr);
573 gcWorker_->CreateAndStartWorker();
574 }
575
DisableWorkerThreads()576 void GC::DisableWorkerThreads()
577 {
578 gcSettings_.SetGCWorkersCount(0);
579 gcSettings_.SetParallelMarkingEnabled(false);
580 gcSettings_.SetParallelCompactingEnabled(false);
581 gcSettings_.SetParallelRefUpdatingEnabled(false);
582 }
583
EnableWorkerThreads()584 void GC::EnableWorkerThreads()
585 {
586 const RuntimeOptions &options = Runtime::GetOptions();
587 gcSettings_.SetGCWorkersCount(options.GetGcWorkersCount());
588 gcSettings_.SetParallelMarkingEnabled(options.IsGcParallelMarkingEnabled() && (options.GetGcWorkersCount() != 0));
589 gcSettings_.SetParallelCompactingEnabled(options.IsGcParallelCompactingEnabled() &&
590 (options.GetGcWorkersCount() != 0));
591 gcSettings_.SetParallelRefUpdatingEnabled(options.IsGcParallelRefUpdatingEnabled() &&
592 (options.GetGcWorkersCount() != 0));
593 }
594
PreZygoteFork()595 void GC::PreZygoteFork()
596 {
597 DestroyWorker();
598 if (gcSettings_.UseTaskManagerForGC()) {
599 ASSERT(gcWorkersTaskQueue_ != nullptr);
600 ASSERT(gcWorkersTaskQueue_->IsEmpty());
601 }
602 }
603
PostZygoteFork()604 void GC::PostZygoteFork()
605 {
606 CreateWorker();
607 }
608
609 class GC::PostForkGCTask : public GCTask {
610 public:
PostForkGCTask(GCTaskCause gcReason,uint64_t gcTargetTime,size_t restoreLimit)611 PostForkGCTask(GCTaskCause gcReason, uint64_t gcTargetTime, size_t restoreLimit)
612 : GCTask(gcReason, gcTargetTime), restoreLimit_(restoreLimit)
613 {
614 }
615
Run(mem::GC & gc)616 void Run(mem::GC &gc) override
617 {
618 LOG(DEBUG, GC) << "Runing PostForkGCTask";
619 gc.GetPandaVm()->GetGCTrigger()->RestoreMinTargetFootprint();
620 gc.PostForkCallback(restoreLimit_);
621 GCTask::Run(gc);
622 }
623
624 ~PostForkGCTask() override = default;
625
626 NO_COPY_SEMANTIC(PostForkGCTask);
627 NO_MOVE_SEMANTIC(PostForkGCTask);
628
629 private:
630 // For g1gc, it saves young space original size.
631 size_t restoreLimit_ {0};
632 };
633
PreStartup()634 void GC::PreStartup()
635 {
636 // Add a delay GCTask.
637 if ((!Runtime::GetCurrent()->IsZygote()) && (!gcSettings_.RunGCInPlace())) {
638 // divide 2 to temporarily set target footprint to a high value to disable GC during App startup.
639 size_t startupLimit = Runtime::GetOptions().GetHeapSizeLimit() / 2;
640 GetPandaVm()->GetGCTrigger()->SetMinTargetFootprint(startupLimit);
641 PreStartupImp();
642 size_t originSize = AdujustStartupLimit(startupLimit);
643 constexpr uint64_t DISABLE_GC_DURATION_NS = 3000ULL * 1000 * 1000;
644 auto task = MakePandaUnique<PostForkGCTask>(GCTaskCause::STARTUP_COMPLETE_CAUSE,
645 time::GetCurrentTimeInNanos() + DISABLE_GC_DURATION_NS, originSize);
646 AddGCTask(true, std::move(task));
647 LOG(DEBUG, GC) << "Add PostForkGCTask";
648 }
649 }
650
651 // NOLINTNEXTLINE(performance-unnecessary-value-param)
AddGCTask(bool isManaged,PandaUniquePtr<GCTask> task)652 bool GC::AddGCTask(bool isManaged, PandaUniquePtr<GCTask> task)
653 {
654 bool triggeredByThreshold = (task->reason == GCTaskCause::HEAP_USAGE_THRESHOLD_CAUSE);
655 if (gcSettings_.RunGCInPlace()) {
656 auto *gcTask = task.get();
657 if (IsGCRunning()) {
658 if (isManaged) {
659 return WaitForGCInManaged(*gcTask);
660 }
661 return WaitForGC(*gcTask);
662 }
663 } else {
664 if (triggeredByThreshold) {
665 bool expect = true;
666 if (canAddGcTask_.compare_exchange_strong(expect, false, std::memory_order_seq_cst)) {
667 return gcWorker_->AddTask(std::move(task));
668 }
669 } else {
670 return gcWorker_->AddTask(std::move(task));
671 }
672 }
673 return false;
674 }
675
IsReference(const BaseClass * cls,const ObjectHeader * ref,const ReferenceCheckPredicateT & pred)676 bool GC::IsReference(const BaseClass *cls, const ObjectHeader *ref, const ReferenceCheckPredicateT &pred)
677 {
678 ASSERT(referenceProcessor_ != nullptr);
679 return referenceProcessor_->IsReference(cls, ref, pred);
680 }
681
EnqueueReferences()682 void GC::EnqueueReferences()
683 {
684 while (true) {
685 ark::mem::Reference *ref = nullptr;
686 {
687 os::memory::LockHolder holder(*clearedReferencesLock_);
688 if (clearedReferences_->empty()) {
689 break;
690 }
691 ref = clearedReferences_->back();
692 clearedReferences_->pop_back();
693 }
694 ASSERT(ref != nullptr);
695 ASSERT(referenceProcessor_ != nullptr);
696 referenceProcessor_->ScheduleForEnqueue(ref);
697 }
698 }
699
IsFullGC() const700 bool GC::IsFullGC() const
701 {
702 // Atomic with relaxed order reason: data race with is_full_gc_ with no synchronization or ordering
703 // constraints imposed on other reads or writes
704 return isFullGc_.load(std::memory_order_relaxed);
705 }
706
SetFullGC(bool value)707 void GC::SetFullGC(bool value)
708 {
709 // Atomic with relaxed order reason: data race with is_full_gc_ with no synchronization or ordering
710 // constraints imposed on other reads or writes
711 isFullGc_.store(value, std::memory_order_relaxed);
712 }
713
NotifyNativeAllocations()714 void GC::NotifyNativeAllocations()
715 {
716 // Atomic with relaxed order reason: data race with native_objects_notified_ with no synchronization or ordering
717 // constraints imposed on other reads or writes
718 nativeObjectsNotified_.fetch_add(NOTIFY_NATIVE_INTERVAL, std::memory_order_relaxed);
719 TriggerGCForNative();
720 }
721
RegisterNativeAllocation(size_t bytes)722 void GC::RegisterNativeAllocation(size_t bytes)
723 {
724 ASSERT_NATIVE_CODE();
725 size_t allocated;
726 do {
727 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
728 // constraints imposed on other reads or writes
729 allocated = nativeBytesRegistered_.load(std::memory_order_relaxed);
730 } while (!nativeBytesRegistered_.compare_exchange_weak(allocated, allocated + bytes));
731 if (allocated > std::numeric_limits<size_t>::max() - bytes) {
732 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
733 // constraints imposed on other reads or writes
734 nativeBytesRegistered_.store(std::numeric_limits<size_t>::max(), std::memory_order_relaxed);
735 }
736 TriggerGCForNative();
737 }
738
RegisterNativeFree(size_t bytes)739 void GC::RegisterNativeFree(size_t bytes)
740 {
741 size_t allocated;
742 size_t newFreedBytes;
743 do {
744 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
745 // constraints imposed on other reads or writes
746 allocated = nativeBytesRegistered_.load(std::memory_order_relaxed);
747 newFreedBytes = std::min(allocated, bytes);
748 } while (!nativeBytesRegistered_.compare_exchange_weak(allocated, allocated - newFreedBytes));
749 }
750
GetNativeBytesFromMallinfoAndRegister() const751 size_t GC::GetNativeBytesFromMallinfoAndRegister() const
752 {
753 size_t mallinfoBytes = ark::os::mem::GetNativeBytesFromMallinfo();
754 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
755 // constraints imposed on other reads or writes
756 size_t allBytes = mallinfoBytes + nativeBytesRegistered_.load(std::memory_order_relaxed);
757 return allBytes;
758 }
759
WaitForGC(GCTask task)760 bool GC::WaitForGC(GCTask task)
761 {
762 // NOTE(maksenov): Notify only about pauses (#4681)
763 Runtime::GetCurrent()->GetNotificationManager()->GarbageCollectorStartEvent();
764 // Atomic with acquire order reason: data race with gc_counter_ with dependecies on reads after the load which
765 // should become visible
766 auto oldCounter = this->gcCounter_.load(std::memory_order_acquire);
767 Timing suspendThreadsTiming;
768 {
769 ScopedTiming t("SuspendThreads", suspendThreadsTiming);
770 this->GetPandaVm()->GetRendezvous()->SafepointBegin();
771 }
772 if (!this->NeedRunGCAfterWaiting(oldCounter, task)) {
773 this->GetPandaVm()->GetRendezvous()->SafepointEnd();
774 return false;
775 }
776
777 // Create a copy of the constant GCTask to be able to change its value
778 this->RunPhases(task);
779
780 if (UNLIKELY(this->IsLogDetailedGcInfoEnabled())) {
781 PrintDetailedLog();
782 }
783
784 this->GetPandaVm()->GetRendezvous()->SafepointEnd();
785 Runtime::GetCurrent()->GetNotificationManager()->GarbageCollectorFinishEvent();
786 this->GetPandaVm()->HandleGCFinished();
787 this->GetPandaVm()->HandleEnqueueReferences();
788 this->GetPandaVm()->ProcessReferenceFinalizers();
789 return true;
790 }
791
WaitForGCInManaged(const GCTask & task)792 bool GC::WaitForGCInManaged(const GCTask &task)
793 {
794 Thread *baseThread = Thread::GetCurrent();
795 if (ManagedThread::ThreadIsManagedThread(baseThread)) {
796 ManagedThread *thread = ManagedThread::CastFromThread(baseThread);
797 ASSERT(thread->GetMutatorLock()->HasLock());
798 [[maybe_unused]] bool isDaemon = MTManagedThread::ThreadIsMTManagedThread(baseThread) &&
799 MTManagedThread::CastFromThread(baseThread)->IsDaemon();
800 ASSERT(!isDaemon || thread->GetStatus() == ThreadStatus::RUNNING);
801 vm_->GetMutatorLock()->Unlock();
802 thread->PrintSuspensionStackIfNeeded();
803 WaitForGC(task);
804 vm_->GetMutatorLock()->ReadLock();
805 ASSERT(vm_->GetMutatorLock()->HasLock());
806 this->GetPandaVm()->HandleGCRoutineInMutator();
807 return true;
808 }
809 return false;
810 }
811
StartConcurrentScopeRoutine() const812 void GC::StartConcurrentScopeRoutine() const {}
813
EndConcurrentScopeRoutine() const814 void GC::EndConcurrentScopeRoutine() const {}
815
PrintDetailedLog()816 void GC::PrintDetailedLog()
817 {
818 for (auto &footprint : this->footprintList_) {
819 LOG(INFO, GC) << footprint.first << " : " << footprint.second;
820 }
821 LOG(INFO, GC) << this->GetTiming()->Dump();
822 }
823
ConcurrentScope(GC * gc,bool autoStart)824 ConcurrentScope::ConcurrentScope(GC *gc, bool autoStart)
825 {
826 LOG(DEBUG, GC) << "Start ConcurrentScope";
827 gc_ = gc;
828 if (autoStart) {
829 Start();
830 }
831 }
832
~ConcurrentScope()833 ConcurrentScope::~ConcurrentScope()
834 {
835 LOG(DEBUG, GC) << "Stop ConcurrentScope";
836 if (started_ && gc_->IsConcurrencyAllowed()) {
837 gc_->GetPandaVm()->GetRendezvous()->SafepointBegin();
838 gc_->SetupCpuAffinityAfterConcurrent();
839 gc_->EndConcurrentScopeRoutine();
840 }
841 }
842
Start()843 NO_THREAD_SAFETY_ANALYSIS void ConcurrentScope::Start()
844 {
845 if (!started_ && gc_->IsConcurrencyAllowed()) {
846 gc_->StartConcurrentScopeRoutine();
847 gc_->SetupCpuAffinityBeforeConcurrent();
848 gc_->GetPandaVm()->GetRendezvous()->SafepointEnd();
849 started_ = true;
850 }
851 }
852
WaitForGCOnPygoteFork(const GCTask & task)853 void GC::WaitForGCOnPygoteFork(const GCTask &task)
854 {
855 // do nothing if no pygote space
856 auto pygoteSpaceAllocator = objectAllocator_->GetPygoteSpaceAllocator();
857 if (pygoteSpaceAllocator == nullptr) {
858 return;
859 }
860
861 // do nothing if not at first pygote fork
862 if (pygoteSpaceAllocator->GetState() != PygoteSpaceState::STATE_PYGOTE_INIT) {
863 return;
864 }
865
866 LOG(DEBUG, GC) << "== GC WaitForGCOnPygoteFork Start ==";
867
868 // do we need a lock?
869 // looks all other threads have been stopped before pygote fork
870
871 // 0. indicate that we're rebuilding pygote space
872 pygoteSpaceAllocator->SetState(PygoteSpaceState::STATE_PYGOTE_FORKING);
873
874 // 1. trigger gc
875 WaitForGC(task);
876
877 // 2. move other space to pygote space
878 MoveObjectsToPygoteSpace();
879
880 // 3. indicate that we have done
881 pygoteSpaceAllocator->SetState(PygoteSpaceState::STATE_PYGOTE_FORKED);
882
883 // 4. disable pygote for allocation
884 objectAllocator_->DisablePygoteAlloc();
885
886 LOG(DEBUG, GC) << "== GC WaitForGCOnPygoteFork End ==";
887 }
888
IsOnPygoteFork() const889 bool GC::IsOnPygoteFork() const
890 {
891 auto pygoteSpaceAllocator = objectAllocator_->GetPygoteSpaceAllocator();
892 return pygoteSpaceAllocator != nullptr &&
893 pygoteSpaceAllocator->GetState() == PygoteSpaceState::STATE_PYGOTE_FORKING;
894 }
895
MoveObjectsToPygoteSpace()896 void GC::MoveObjectsToPygoteSpace()
897 {
898 trace::ScopedTrace scopedTrace(__FUNCTION__);
899 LOG(DEBUG, GC) << "MoveObjectsToPygoteSpace: start";
900
901 size_t allSizeMove = 0;
902 size_t movedObjectsNum = 0;
903 size_t bytesInHeapBeforeMove = GetPandaVm()->GetMemStats()->GetFootprintHeap();
904 auto pygoteSpaceAllocator = objectAllocator_->GetPygoteSpaceAllocator();
905 ObjectVisitor moveVisitor([this, &pygoteSpaceAllocator, &movedObjectsNum, &allSizeMove](ObjectHeader *src) -> void {
906 size_t size = GetObjectSize(src);
907 auto dst = reinterpret_cast<ObjectHeader *>(pygoteSpaceAllocator->Alloc(size));
908 ASSERT(dst != nullptr);
909 memcpy_s(dst, size, src, size);
910 allSizeMove += size;
911 movedObjectsNum++;
912 SetForwardAddress(src, dst);
913 LOG_DEBUG_GC << "object MOVED from " << std::hex << src << " to " << dst << ", size = " << std::dec << size;
914 });
915
916 // move all small movable objects to pygote space
917 objectAllocator_->IterateRegularSizeObjects(moveVisitor);
918
919 LOG(DEBUG, GC) << "MoveObjectsToPygoteSpace: move_num = " << movedObjectsNum << ", move_size = " << allSizeMove;
920
921 if (allSizeMove > 0) {
922 GetStats()->AddMemoryValue(allSizeMove, MemoryTypeStats::MOVED_BYTES);
923 GetStats()->AddObjectsValue(movedObjectsNum, ObjectTypeStats::MOVED_OBJECTS);
924 }
925 if (bytesInHeapBeforeMove > 0) {
926 GetStats()->AddCopiedRatioValue(static_cast<double>(allSizeMove) / bytesInHeapBeforeMove);
927 }
928
929 // Update because we moved objects from object_allocator -> pygote space
930 UpdateRefsToMovedObjectsInPygoteSpace();
931 CommonUpdateRefsToMovedObjects();
932
933 // Clear the moved objects in old space
934 objectAllocator_->FreeObjectsMovedToPygoteSpace();
935
936 LOG(DEBUG, GC) << "MoveObjectsToPygoteSpace: finish";
937 }
938
SetForwardAddress(ObjectHeader * src,ObjectHeader * dst)939 void GC::SetForwardAddress(ObjectHeader *src, ObjectHeader *dst)
940 {
941 auto baseCls = src->ClassAddr<BaseClass>();
942 if (baseCls->IsDynamicClass()) {
943 auto cls = static_cast<HClass *>(baseCls);
944 // Note: During moving phase, 'src => dst'. Consider the src is a DynClass,
945 // since 'dst' is not in GC-status the 'manage-object' inside 'dst' won't be updated to
946 // 'dst'. To fix it, we update 'manage-object' here rather than upating phase.
947 if (cls->IsHClass()) {
948 size_t offset = ObjectHeader::ObjectHeaderSize() + HClass::GetManagedObjectOffset();
949 dst->SetFieldObject<false, false, true>(GetPandaVm()->GetAssociatedThread(), offset, dst);
950 }
951 }
952
953 // Set fwd address in src
954 bool updateRes = false;
955 do {
956 MarkWord markWord = src->AtomicGetMark();
957 MarkWord fwdMarkWord =
958 markWord.DecodeFromForwardingAddress(static_cast<MarkWord::MarkWordSize>(ToUintPtr(dst)));
959 updateRes = src->AtomicSetMark<false>(markWord, fwdMarkWord);
960 } while (!updateRes);
961 }
962
PopObjectFromStack(GCMarkingStackType * objectsStack)963 const ObjectHeader *GC::PopObjectFromStack(GCMarkingStackType *objectsStack)
964 {
965 auto *object = objectsStack->PopFromStack();
966 ASSERT(object != nullptr);
967 return object;
968 }
969
IsGenerational() const970 bool GC::IsGenerational() const
971 {
972 return IsGenerationalGCType(gcType_);
973 }
974
AddListener(GCListener * listener)975 void GC::GCListenerManager::AddListener(GCListener *listener)
976 {
977 os::memory::LockHolder lh(listenerLock_);
978 newListeners_.push_back(listener);
979 }
980
RemoveListener(GCListener * listener)981 void GC::GCListenerManager::RemoveListener(GCListener *listener)
982 {
983 os::memory::LockHolder lh(listenerLock_);
984 listenersForRemove_.push_back(listener);
985 }
986
NormalizeListenersOnStartGC()987 void GC::GCListenerManager::NormalizeListenersOnStartGC()
988 {
989 os::memory::LockHolder lh(listenerLock_);
990 for (auto *listenerForRemove : listenersForRemove_) {
991 auto it = std::find(newListeners_.begin(), newListeners_.end(), listenerForRemove);
992 if (it != newListeners_.end()) {
993 newListeners_.erase(it);
994 }
995 it = std::find(currentListeners_.begin(), currentListeners_.end(), listenerForRemove);
996 if (it != currentListeners_.end()) {
997 LOG(DEBUG, GC) << "Remove listener for GC: " << listenerForRemove;
998 currentListeners_.erase(it);
999 }
1000 }
1001 listenersForRemove_.clear();
1002 for (auto *newListener : newListeners_) {
1003 LOG(DEBUG, GC) << "Add new listener for GC: " << newListener;
1004 currentListeners_.push_back(newListener);
1005 }
1006 newListeners_.clear();
1007 }
1008
FireGCStarted(const GCTask & task,size_t bytesInHeapBeforeGc)1009 void GC::FireGCStarted(const GCTask &task, size_t bytesInHeapBeforeGc)
1010 {
1011 gcListenerManager_->NormalizeListenersOnStartGC();
1012 gcListenerManager_->IterateOverListeners(
1013 [&](GCListener *listener) { listener->GCStarted(task, bytesInHeapBeforeGc); });
1014 }
1015
FireGCFinished(const GCTask & task,size_t bytesInHeapBeforeGc,size_t bytesInHeapAfterGc)1016 void GC::FireGCFinished(const GCTask &task, size_t bytesInHeapBeforeGc, size_t bytesInHeapAfterGc)
1017 {
1018 gcListenerManager_->IterateOverListeners(
1019 [&](GCListener *listener) { listener->GCFinished(task, bytesInHeapBeforeGc, bytesInHeapAfterGc); });
1020 }
1021
FireGCPhaseStarted(GCPhase phase)1022 void GC::FireGCPhaseStarted(GCPhase phase)
1023 {
1024 gcListenerManager_->IterateOverListeners([phase](GCListener *listener) { listener->GCPhaseStarted(phase); });
1025 }
1026
FireGCPhaseFinished(GCPhase phase)1027 void GC::FireGCPhaseFinished(GCPhase phase)
1028 {
1029 gcListenerManager_->IterateOverListeners([phase](GCListener *listener) { listener->GCPhaseFinished(phase); });
1030 }
1031
OnWaitForIdleFail()1032 void GC::OnWaitForIdleFail() {}
1033
1034 TEMPLATE_GC_CREATE_GC();
1035
1036 } // namespace ark::mem
1037