1 /**
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include <memory>
17
18 #include "libpandabase/os/cpu_affinity.h"
19 #include "libpandabase/os/mem.h"
20 #include "libpandabase/os/thread.h"
21 #include "libpandabase/utils/time.h"
22 #include "runtime/assert_gc_scope.h"
23 #include "runtime/include/class.h"
24 #include "runtime/include/coretypes/dyn_objects.h"
25 #include "runtime/include/locks.h"
26 #include "runtime/include/runtime.h"
27 #include "runtime/include/runtime_notification.h"
28 #include "runtime/include/stack_walker-inl.h"
29 #include "runtime/mem/gc/epsilon/epsilon.h"
30 #include "runtime/mem/gc/epsilon-g1/epsilon-g1.h"
31 #include "runtime/mem/gc/gc.h"
32 #include "runtime/mem/gc/gc_root-inl.h"
33 #include "runtime/mem/gc/g1/g1-gc.h"
34 #include "runtime/mem/gc/gen-gc/gen-gc.h"
35 #include "runtime/mem/gc/stw-gc/stw-gc.h"
36 #include "runtime/mem/gc/workers/gc_workers_task_queue.h"
37 #include "runtime/mem/gc/workers/gc_workers_thread_pool.h"
38 #include "runtime/mem/pygote_space_allocator-inl.h"
39 #include "runtime/mem/heap_manager.h"
40 #include "runtime/mem/gc/reference-processor/reference_processor.h"
41 #include "runtime/mem/gc/gc-hung/gc_hung.h"
42 #include "runtime/include/panda_vm.h"
43 #include "runtime/include/object_accessor-inl.h"
44 #include "runtime/include/coretypes/class.h"
45 #include "runtime/thread_manager.h"
46
47 namespace ark::mem {
48 using TaggedValue = coretypes::TaggedValue;
49 using TaggedType = coretypes::TaggedType;
50 using DynClass = coretypes::DynClass;
51
GC(ObjectAllocatorBase * objectAllocator,const GCSettings & settings)52 GC::GC(ObjectAllocatorBase *objectAllocator, const GCSettings &settings)
53 : gcSettings_(settings),
54 objectAllocator_(objectAllocator),
55 internalAllocator_(InternalAllocator<>::GetInternalAllocatorFromRuntime())
56 {
57 if (gcSettings_.UseTaskManagerForGC()) {
58 // Create gc task queue for task manager
59 auto *tm = taskmanager::TaskScheduler::GetTaskScheduler();
60 gcWorkersTaskQueue_ = tm->CreateAndRegisterTaskQueue<decltype(internalAllocator_->Adapter())>(
61 taskmanager::TaskType::GC, taskmanager::VMType::STATIC_VM, GC_TASK_QUEUE_PRIORITY);
62 ASSERT(gcWorkersTaskQueue_ != nullptr);
63 }
64 }
65
~GC()66 GC::~GC()
67 {
68 InternalAllocatorPtr allocator = GetInternalAllocator();
69 if (gcWorker_ != nullptr) {
70 allocator->Delete(gcWorker_);
71 }
72 if (gcListenerManager_ != nullptr) {
73 allocator->Delete(gcListenerManager_);
74 }
75 if (gcBarrierSet_ != nullptr) {
76 allocator->Delete(gcBarrierSet_);
77 }
78 if (clearedReferences_ != nullptr) {
79 allocator->Delete(clearedReferences_);
80 }
81 if (clearedReferencesLock_ != nullptr) {
82 allocator->Delete(clearedReferencesLock_);
83 }
84 if (workersTaskPool_ != nullptr) {
85 allocator->Delete(workersTaskPool_);
86 }
87 if (gcWorkersTaskQueue_ != nullptr) {
88 taskmanager::TaskScheduler::GetTaskScheduler()->UnregisterAndDestroyTaskQueue<decltype(allocator->Adapter())>(
89 gcWorkersTaskQueue_);
90 }
91 }
92
GetLogPrefix() const93 Logger::Buffer GC::GetLogPrefix() const
94 {
95 const char *phase = GCScopedPhase::GetPhaseAbbr(GetGCPhase());
96 // Atomic with acquire order reason: data race with gc_counter_
97 size_t counter = gcCounter_.load(std::memory_order_acquire);
98
99 Logger::Buffer buffer;
100 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
101 buffer.Printf("[%zu, %s]: ", counter, phase);
102
103 return buffer;
104 }
105
GetType()106 GCType GC::GetType()
107 {
108 return gcType_;
109 }
110
SetPandaVM(PandaVM * vm)111 void GC::SetPandaVM(PandaVM *vm)
112 {
113 vm_ = vm;
114 referenceProcessor_ = vm->GetReferenceProcessor();
115 }
116
GetNativeGcTriggerType()117 NativeGcTriggerType GC::GetNativeGcTriggerType()
118 {
119 return gcSettings_.GetNativeGcTriggerType();
120 }
121
SimpleNativeAllocationGcWatermark()122 size_t GC::SimpleNativeAllocationGcWatermark()
123 {
124 return GetPandaVm()->GetOptions().GetMaxFree();
125 }
126
WaitForIdleGC()127 NO_THREAD_SAFETY_ANALYSIS void GC::WaitForIdleGC()
128 {
129 while (!CASGCPhase(GCPhase::GC_PHASE_IDLE, GCPhase::GC_PHASE_RUNNING)) {
130 GetPandaVm()->GetRendezvous()->SafepointEnd();
131 // Interrupt the running GC if possible
132 OnWaitForIdleFail();
133 // NOTE(dtrubenkov): resolve it more properly
134 constexpr uint64_t WAIT_FINISHED = 100;
135 // Use NativeSleep for all threads, as this thread shouldn't hold Mutator lock here
136 os::thread::NativeSleepUS(std::chrono::microseconds(WAIT_FINISHED));
137 GetPandaVm()->GetRendezvous()->SafepointBegin();
138 }
139 }
140
TriggerGCForNative()141 inline void GC::TriggerGCForNative()
142 {
143 auto nativeGcTriggerType = GetNativeGcTriggerType();
144 ASSERT_PRINT((nativeGcTriggerType == NativeGcTriggerType::NO_NATIVE_GC_TRIGGER) ||
145 (nativeGcTriggerType == NativeGcTriggerType::SIMPLE_STRATEGY),
146 "Unknown Native GC Trigger type");
147 switch (nativeGcTriggerType) {
148 case NativeGcTriggerType::NO_NATIVE_GC_TRIGGER:
149 break;
150 case NativeGcTriggerType::SIMPLE_STRATEGY:
151 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or
152 // ordering constraints imposed on other reads or writes
153 if (nativeBytesRegistered_.load(std::memory_order_relaxed) > SimpleNativeAllocationGcWatermark()) {
154 auto task = MakePandaUnique<GCTask>(GCTaskCause::NATIVE_ALLOC_CAUSE, time::GetCurrentTimeInNanos());
155 AddGCTask(false, std::move(task));
156 ManagedThread::GetCurrent()->SafepointPoll();
157 }
158 break;
159 default:
160 LOG(FATAL, GC) << "Unknown Native GC Trigger type";
161 break;
162 }
163 }
164
Initialize(PandaVM * vm)165 void GC::Initialize(PandaVM *vm)
166 {
167 trace::ScopedTrace scopedTrace(__PRETTY_FUNCTION__);
168 // GC saved the PandaVM instance, so we get allocator from the PandaVM.
169 auto allocator = GetInternalAllocator();
170 gcListenerManager_ = allocator->template New<GCListenerManager>();
171 clearedReferencesLock_ = allocator->New<os::memory::Mutex>();
172 os::memory::LockHolder holder(*clearedReferencesLock_);
173 clearedReferences_ = allocator->New<PandaVector<ark::mem::Reference *>>(allocator->Adapter());
174 this->SetPandaVM(vm);
175 InitializeImpl();
176 gcWorker_ = allocator->New<GCWorker>(this);
177 }
178
CreateWorkersTaskPool()179 void GC::CreateWorkersTaskPool()
180 {
181 ASSERT(workersTaskPool_ == nullptr);
182 if (this->IsWorkerThreadsExist()) {
183 auto allocator = GetInternalAllocator();
184 GCWorkersTaskPool *gcTaskPool = nullptr;
185 if (this->GetSettings()->UseThreadPoolForGC()) {
186 // Use internal gc thread pool
187 gcTaskPool = allocator->New<GCWorkersThreadPool>(this, this->GetSettings()->GCWorkersCount());
188 } else {
189 // Use common TaskManager
190 ASSERT(this->GetSettings()->UseTaskManagerForGC());
191 gcTaskPool = allocator->New<GCWorkersTaskQueue>(this);
192 }
193 ASSERT(gcTaskPool != nullptr);
194 workersTaskPool_ = gcTaskPool;
195 }
196 }
197
DestroyWorkersTaskPool()198 void GC::DestroyWorkersTaskPool()
199 {
200 if (workersTaskPool_ == nullptr) {
201 return;
202 }
203 workersTaskPool_->WaitUntilTasksEnd();
204 auto allocator = this->GetInternalAllocator();
205 allocator->Delete(workersTaskPool_);
206 workersTaskPool_ = nullptr;
207 }
208
StartGC()209 void GC::StartGC()
210 {
211 CreateWorker();
212 }
213
StopGC()214 void GC::StopGC()
215 {
216 DestroyWorker();
217 DestroyWorkersTaskPool();
218 }
219
SetupCpuAffinity()220 void GC::SetupCpuAffinity()
221 {
222 if (!gcSettings_.ManageGcThreadsAffinity()) {
223 return;
224 }
225 // Try to get CPU affinity fo GC Thread
226 if (UNLIKELY(!os::CpuAffinityManager::GetCurrentThreadAffinity(affinityBeforeGc_))) {
227 affinityBeforeGc_.Clear();
228 return;
229 }
230 // Try to use best + middle for preventing issues when best core is used in another thread,
231 // and GC waits for it to finish.
232 if (!os::CpuAffinityManager::SetAffinityForCurrentThread(os::CpuPower::BEST | os::CpuPower::MIDDLE)) {
233 affinityBeforeGc_.Clear();
234 }
235 // Some GCs don't use GC Workers
236 if (workersTaskPool_ != nullptr && this->GetSettings()->UseThreadPoolForGC()) {
237 static_cast<GCWorkersThreadPool *>(workersTaskPool_)->SetAffinityForGCWorkers();
238 }
239 }
240
SetupCpuAffinityAfterConcurrent()241 void GC::SetupCpuAffinityAfterConcurrent()
242 {
243 if (!gcSettings_.ManageGcThreadsAffinity()) {
244 return;
245 }
246 os::CpuAffinityManager::SetAffinityForCurrentThread(os::CpuPower::BEST | os::CpuPower::MIDDLE);
247 // Some GCs don't use GC Workers
248 if (workersTaskPool_ != nullptr && this->GetSettings()->UseThreadPoolForGC()) {
249 static_cast<GCWorkersThreadPool *>(workersTaskPool_)->SetAffinityForGCWorkers();
250 }
251 }
252
ResetCpuAffinity(bool beforeConcurrent)253 void GC::ResetCpuAffinity(bool beforeConcurrent)
254 {
255 if (!gcSettings_.ManageGcThreadsAffinity()) {
256 return;
257 }
258 if (!affinityBeforeGc_.IsEmpty()) {
259 // Set GC Threads on weak CPUs before concurrent if needed
260 if (beforeConcurrent && gcSettings_.UseWeakCpuForGcConcurrent()) {
261 os::CpuAffinityManager::SetAffinityForCurrentThread(os::CpuPower::WEAK);
262 } else { // else set on saved affinity
263 os::CpuAffinityManager::SetAffinityForCurrentThread(affinityBeforeGc_);
264 }
265 }
266 // Some GCs don't use GC Workers
267 if (workersTaskPool_ != nullptr && this->GetSettings()->UseThreadPoolForGC()) {
268 static_cast<GCWorkersThreadPool *>(workersTaskPool_)->UnsetAffinityForGCWorkers();
269 }
270 }
271
SetupCpuAffinityBeforeConcurrent()272 void GC::SetupCpuAffinityBeforeConcurrent()
273 {
274 ResetCpuAffinity(true);
275 }
276
RestoreCpuAffinity()277 void GC::RestoreCpuAffinity()
278 {
279 ResetCpuAffinity(false);
280 }
281
NeedRunGCAfterWaiting(size_t counterBeforeWaiting,const GCTask & task) const282 bool GC::NeedRunGCAfterWaiting(size_t counterBeforeWaiting, const GCTask &task) const
283 {
284 // Atomic with acquire order reason: data race with gc_counter_ with dependecies on reads after the load which
285 // should become visible
286 auto newCounter = gcCounter_.load(std::memory_order_acquire);
287 ASSERT(newCounter >= counterBeforeWaiting);
288 // Atomic with acquire order reason: data race with last_cause_ with dependecies on reads after the load which
289 // should become visible
290 return (newCounter == counterBeforeWaiting || lastCause_.load(std::memory_order_acquire) < task.reason);
291 }
292
GCPhasesPreparation(const GCTask & task)293 bool GC::GCPhasesPreparation(const GCTask &task)
294 {
295 // Atomic with acquire order reason: data race with gc_counter_ with dependecies on reads after the load which
296 // should become visible
297 auto oldCounter = gcCounter_.load(std::memory_order_acquire);
298 WaitForIdleGC();
299 if (!this->NeedRunGCAfterWaiting(oldCounter, task)) {
300 SetGCPhase(GCPhase::GC_PHASE_IDLE);
301 return false;
302 }
303 this->SetupCpuAffinity();
304 this->GetTiming()->Reset(); // Clear records.
305 // Atomic with release order reason: data race with last_cause_ with dependecies on writes before the store which
306 // should become visible acquire
307 lastCause_.store(task.reason, std::memory_order_release);
308 if (gcSettings_.PreGCHeapVerification()) {
309 trace::ScopedTrace preHeapVerifierTrace("PreGCHeapVeriFier");
310 size_t failCount = VerifyHeap();
311 if (gcSettings_.FailOnHeapVerification() && failCount > 0) {
312 LOG(FATAL, GC) << "Heap corrupted before GC, HeapVerifier found " << failCount << " corruptions";
313 }
314 }
315 // Atomic with acq_rel order reason: data race with gc_counter_ with dependecies on reads after the load and on
316 // writes before the store
317 gcCounter_.fetch_add(1, std::memory_order_acq_rel);
318 if (gcSettings_.IsDumpHeap()) {
319 PandaOStringStream os;
320 os << "Heap dump before GC" << std::endl;
321 GetPandaVm()->DumpHeap(&os);
322 std::cerr << os.str() << std::endl;
323 }
324 return true;
325 }
326
GCPhasesFinish(const GCTask & task)327 void GC::GCPhasesFinish(const GCTask &task)
328 {
329 ASSERT(task.collectionType != GCCollectionType::NONE);
330 LOG(INFO, GC) << "[" << gcCounter_ << "] [" << task.collectionType << " (" << task.reason << ")] "
331 << GetPandaVm()->GetGCStats()->GetStatistics();
332
333 if (gcSettings_.IsDumpHeap()) {
334 PandaOStringStream os;
335 os << "Heap dump after GC" << std::endl;
336 GetPandaVm()->DumpHeap(&os);
337 std::cerr << os.str() << std::endl;
338 }
339
340 if (gcSettings_.PostGCHeapVerification()) {
341 trace::ScopedTrace postHeapVerifierTrace("PostGCHeapVeriFier");
342 size_t failCount = VerifyHeap();
343 if (gcSettings_.FailOnHeapVerification() && failCount > 0) {
344 LOG(FATAL, GC) << "Heap corrupted after GC, HeapVerifier found " << failCount << " corruptions";
345 }
346 }
347 this->RestoreCpuAffinity();
348
349 SetGCPhase(GCPhase::GC_PHASE_IDLE);
350 }
351
352 // NOLINTNEXTLINE(performance-unnecessary-value-param)
RunPhases(GCTask & task)353 void GC::RunPhases(GCTask &task)
354 {
355 DCHECK_ALLOW_GARBAGE_COLLECTION;
356 trace::ScopedTrace scopedTrace(__FUNCTION__);
357 bool needRunGCAfterWaiting = GCPhasesPreparation(task);
358 if (!needRunGCAfterWaiting) {
359 return;
360 }
361 size_t bytesInHeapBeforeGc = GetPandaVm()->GetMemStats()->GetFootprintHeap();
362 LOG_DEBUG_GC << "Bytes in heap before GC " << std::dec << bytesInHeapBeforeGc;
363 {
364 GCScopedStats scopedStats(GetPandaVm()->GetGCStats(), gcType_ == GCType::STW_GC ? GetStats() : nullptr);
365 ScopedGcHung scopedHung(&task);
366 GetPandaVm()->GetGCStats()->ResetLastPause();
367
368 FireGCStarted(task, bytesInHeapBeforeGc);
369 PreRunPhasesImpl();
370 // NOLINTNEXTLINE(performance-unnecessary-value-param)
371 RunPhasesImpl(task);
372 // Clear Internal allocator unused pools (must do it on pause to avoid race conditions):
373 // - Clear global part:
374 InternalAllocator<>::GetInternalAllocatorFromRuntime()->VisitAndRemoveFreePools(
375 [](void *mem, [[maybe_unused]] size_t size) { PoolManager::GetMmapMemPool()->FreePool(mem, size); });
376 // - Clear local part:
377 ClearLocalInternalAllocatorPools();
378
379 size_t bytesInHeapAfterGc = GetPandaVm()->GetMemStats()->GetFootprintHeap();
380 // There is case than bytes_in_heap_after_gc > 0 and bytes_in_heap_before_gc == 0.
381 // Because TLABs are registered during GC
382 if (bytesInHeapAfterGc > 0 && bytesInHeapBeforeGc > 0) {
383 GetStats()->AddReclaimRatioValue(1 - static_cast<double>(bytesInHeapAfterGc) / bytesInHeapBeforeGc);
384 }
385 LOG_DEBUG_GC << "Bytes in heap after GC " << std::dec << bytesInHeapAfterGc;
386 FireGCFinished(task, bytesInHeapBeforeGc, bytesInHeapAfterGc);
387 }
388 GCPhasesFinish(task);
389 }
390
391 template <class LanguageConfig>
CreateGC(GCType gcType,ObjectAllocatorBase * objectAllocator,const GCSettings & settings)392 GC *CreateGC(GCType gcType, ObjectAllocatorBase *objectAllocator, const GCSettings &settings)
393 {
394 GC *ret = nullptr;
395 InternalAllocatorPtr allocator {InternalAllocator<>::GetInternalAllocatorFromRuntime()};
396
397 switch (gcType) {
398 case GCType::EPSILON_GC:
399 ret = allocator->New<EpsilonGC<LanguageConfig>>(objectAllocator, settings);
400 break;
401 case GCType::EPSILON_G1_GC:
402 ret = allocator->New<EpsilonG1GC<LanguageConfig>>(objectAllocator, settings);
403 break;
404 case GCType::STW_GC:
405 ret = allocator->New<StwGC<LanguageConfig>>(objectAllocator, settings);
406 break;
407 case GCType::GEN_GC:
408 ret = allocator->New<GenGC<LanguageConfig>>(objectAllocator, settings);
409 break;
410 case GCType::G1_GC:
411 ret = allocator->New<G1GC<LanguageConfig>>(objectAllocator, settings);
412 break;
413 default:
414 LOG(FATAL, GC) << "Unknown GC type";
415 break;
416 }
417 return ret;
418 }
419
CheckGCCause(GCTaskCause cause) const420 bool GC::CheckGCCause(GCTaskCause cause) const
421 {
422 return cause != GCTaskCause::INVALID_CAUSE;
423 }
424
MarkObjectIfNotMarked(ObjectHeader * objectHeader)425 bool GC::MarkObjectIfNotMarked(ObjectHeader *objectHeader)
426 {
427 ASSERT(objectHeader != nullptr);
428 if (IsMarked(objectHeader)) {
429 return false;
430 }
431 MarkObject(objectHeader);
432 return true;
433 }
434
ProcessReference(GCMarkingStackType * objectsStack,const BaseClass * cls,const ObjectHeader * ref,const ReferenceProcessPredicateT & pred)435 void GC::ProcessReference(GCMarkingStackType *objectsStack, const BaseClass *cls, const ObjectHeader *ref,
436 const ReferenceProcessPredicateT &pred)
437 {
438 ASSERT(referenceProcessor_ != nullptr);
439 referenceProcessor_->HandleReference(this, objectsStack, cls, ref, pred);
440 }
441
ProcessReferenceForSinglePassCompaction(const BaseClass * cls,const ObjectHeader * ref,const ReferenceProcessorT & processor)442 void GC::ProcessReferenceForSinglePassCompaction(const BaseClass *cls, const ObjectHeader *ref,
443 const ReferenceProcessorT &processor)
444 {
445 ASSERT(referenceProcessor_ != nullptr);
446 referenceProcessor_->HandleReference(this, cls, ref, processor);
447 }
448
AddReference(ObjectHeader * fromObj,ObjectHeader * object)449 void GC::AddReference(ObjectHeader *fromObj, ObjectHeader *object)
450 {
451 ASSERT(IsMarked(object));
452 GCMarkingStackType references(this);
453 // NOTE(alovkov): support stack with workers here & put all refs in stack and only then process altogether for once
454 ASSERT(!references.IsWorkersTaskSupported());
455 references.PushToStack(fromObj, object);
456 MarkReferences(&references, phase_);
457 if (gcType_ != GCType::EPSILON_GC) {
458 ASSERT(references.Empty());
459 }
460 }
461
462 // NOLINTNEXTLINE(performance-unnecessary-value-param)
ProcessReferences(GCPhase gcPhase,const GCTask & task,const ReferenceClearPredicateT & pred)463 void GC::ProcessReferences(GCPhase gcPhase, const GCTask &task, const ReferenceClearPredicateT &pred)
464 {
465 trace::ScopedTrace scopedTrace(__FUNCTION__);
466 LOG(DEBUG, REF_PROC) << "Start processing cleared references";
467 ASSERT(referenceProcessor_ != nullptr);
468 bool clearSoftReferences = task.reason == GCTaskCause::OOM_CAUSE || IsExplicitFull(task);
469 referenceProcessor_->ProcessReferences(false, clearSoftReferences, gcPhase, pred);
470 Reference *processedRef = referenceProcessor_->CollectClearedReferences();
471 if (processedRef != nullptr) {
472 os::memory::LockHolder holder(*clearedReferencesLock_);
473 // NOTE(alovkov): ged rid of cleared_references_ and just enqueue refs here?
474 clearedReferences_->push_back(processedRef);
475 }
476 }
477
ProcessReferences(const GCTask & task,const ReferenceClearPredicateT & pred)478 void GC::ProcessReferences(const GCTask &task, const ReferenceClearPredicateT &pred)
479 {
480 trace::ScopedTrace scopedTrace(__FUNCTION__);
481 LOG(DEBUG, REF_PROC) << "Start processing cleared references";
482 ASSERT(referenceProcessor_ != nullptr);
483 bool clearSoftReferences = task.reason == GCTaskCause::OOM_CAUSE || IsExplicitFull(task);
484 referenceProcessor_->ProcessReferencesAfterCompaction(clearSoftReferences, pred);
485 Reference *processedRef = referenceProcessor_->CollectClearedReferences();
486 if (processedRef != nullptr) {
487 os::memory::LockHolder holder(*clearedReferencesLock_);
488 clearedReferences_->push_back(processedRef);
489 }
490 }
491
DestroyWorker()492 void GC::DestroyWorker()
493 {
494 // Atomic with seq_cst order reason: data race with gc_running_ with requirement for sequentially consistent order
495 // where threads observe all modifications in the same order
496 gcRunning_.store(false, std::memory_order_seq_cst);
497 gcWorker_->FinalizeAndDestroyWorker();
498 }
499
CreateWorker()500 void GC::CreateWorker()
501 {
502 // Atomic with seq_cst order reason: data race with gc_running_ with requirement for sequentially consistent order
503 // where threads observe all modifications in the same order
504 gcRunning_.store(true, std::memory_order_seq_cst);
505 ASSERT(gcWorker_ != nullptr);
506 gcWorker_->CreateAndStartWorker();
507 }
508
DisableWorkerThreads()509 void GC::DisableWorkerThreads()
510 {
511 gcSettings_.SetGCWorkersCount(0);
512 gcSettings_.SetParallelMarkingEnabled(false);
513 gcSettings_.SetParallelCompactingEnabled(false);
514 gcSettings_.SetParallelRefUpdatingEnabled(false);
515 }
516
EnableWorkerThreads()517 void GC::EnableWorkerThreads()
518 {
519 const RuntimeOptions &options = Runtime::GetOptions();
520 gcSettings_.SetGCWorkersCount(options.GetGcWorkersCount());
521 gcSettings_.SetParallelMarkingEnabled(options.IsGcParallelMarkingEnabled() && (options.GetGcWorkersCount() != 0));
522 gcSettings_.SetParallelCompactingEnabled(options.IsGcParallelCompactingEnabled() &&
523 (options.GetGcWorkersCount() != 0));
524 gcSettings_.SetParallelRefUpdatingEnabled(options.IsGcParallelRefUpdatingEnabled() &&
525 (options.GetGcWorkersCount() != 0));
526 }
527
PreZygoteFork()528 void GC::PreZygoteFork()
529 {
530 DestroyWorker();
531 if (gcSettings_.UseTaskManagerForGC()) {
532 ASSERT(gcWorkersTaskQueue_ != nullptr);
533 ASSERT(gcWorkersTaskQueue_->IsEmpty());
534 }
535 }
536
PostZygoteFork()537 void GC::PostZygoteFork()
538 {
539 CreateWorker();
540 }
541
542 class GC::PostForkGCTask : public GCTask {
543 public:
PostForkGCTask(GCTaskCause gcReason,uint64_t gcTargetTime)544 PostForkGCTask(GCTaskCause gcReason, uint64_t gcTargetTime) : GCTask(gcReason, gcTargetTime) {}
545
Run(mem::GC & gc)546 void Run(mem::GC &gc) override
547 {
548 LOG(DEBUG, GC) << "Runing PostForkGCTask";
549 gc.GetPandaVm()->GetGCTrigger()->RestoreMinTargetFootprint();
550 gc.PostForkCallback();
551 GCTask::Run(gc);
552 }
553
554 ~PostForkGCTask() override = default;
555
556 NO_COPY_SEMANTIC(PostForkGCTask);
557 NO_MOVE_SEMANTIC(PostForkGCTask);
558 };
559
PreStartup()560 void GC::PreStartup()
561 {
562 // Add a delay GCTask.
563 if ((!Runtime::GetCurrent()->IsZygote()) && (!gcSettings_.RunGCInPlace())) {
564 // divide 2 to temporarily set target footprint to a high value to disable GC during App startup.
565 GetPandaVm()->GetGCTrigger()->SetMinTargetFootprint(Runtime::GetOptions().GetHeapSizeLimit() / 2);
566 PreStartupImp();
567 constexpr uint64_t DISABLE_GC_DURATION_NS = 2000 * 1000 * 1000;
568 auto task = MakePandaUnique<PostForkGCTask>(GCTaskCause::STARTUP_COMPLETE_CAUSE,
569 time::GetCurrentTimeInNanos() + DISABLE_GC_DURATION_NS);
570 AddGCTask(true, std::move(task));
571 LOG(DEBUG, GC) << "Add PostForkGCTask";
572 }
573 }
574
575 // NOLINTNEXTLINE(performance-unnecessary-value-param)
AddGCTask(bool isManaged,PandaUniquePtr<GCTask> task)576 bool GC::AddGCTask(bool isManaged, PandaUniquePtr<GCTask> task)
577 {
578 bool triggeredByThreshold = (task->reason == GCTaskCause::HEAP_USAGE_THRESHOLD_CAUSE);
579 if (gcSettings_.RunGCInPlace()) {
580 auto *gcTask = task.get();
581 if (IsGCRunning()) {
582 if (isManaged) {
583 return WaitForGCInManaged(*gcTask);
584 }
585 return WaitForGC(*gcTask);
586 }
587 } else {
588 if (triggeredByThreshold) {
589 bool expect = true;
590 if (canAddGcTask_.compare_exchange_strong(expect, false, std::memory_order_seq_cst)) {
591 return gcWorker_->AddTask(std::move(task));
592 }
593 } else {
594 return gcWorker_->AddTask(std::move(task));
595 }
596 }
597 return false;
598 }
599
IsReference(const BaseClass * cls,const ObjectHeader * ref,const ReferenceCheckPredicateT & pred)600 bool GC::IsReference(const BaseClass *cls, const ObjectHeader *ref, const ReferenceCheckPredicateT &pred)
601 {
602 ASSERT(referenceProcessor_ != nullptr);
603 return referenceProcessor_->IsReference(cls, ref, pred);
604 }
605
IsReference(const BaseClass * cls,const ObjectHeader * ref)606 bool GC::IsReference(const BaseClass *cls, const ObjectHeader *ref)
607 {
608 ASSERT(referenceProcessor_ != nullptr);
609 return referenceProcessor_->IsReference(cls, ref);
610 }
611
EnqueueReferences()612 void GC::EnqueueReferences()
613 {
614 while (true) {
615 ark::mem::Reference *ref = nullptr;
616 {
617 os::memory::LockHolder holder(*clearedReferencesLock_);
618 if (clearedReferences_->empty()) {
619 break;
620 }
621 ref = clearedReferences_->back();
622 clearedReferences_->pop_back();
623 }
624 ASSERT(ref != nullptr);
625 ASSERT(referenceProcessor_ != nullptr);
626 referenceProcessor_->ScheduleForEnqueue(ref);
627 }
628 }
629
IsFullGC() const630 bool GC::IsFullGC() const
631 {
632 // Atomic with relaxed order reason: data race with is_full_gc_ with no synchronization or ordering
633 // constraints imposed on other reads or writes
634 return isFullGc_.load(std::memory_order_relaxed);
635 }
636
SetFullGC(bool value)637 void GC::SetFullGC(bool value)
638 {
639 // Atomic with relaxed order reason: data race with is_full_gc_ with no synchronization or ordering
640 // constraints imposed on other reads or writes
641 isFullGc_.store(value, std::memory_order_relaxed);
642 }
643
NotifyNativeAllocations()644 void GC::NotifyNativeAllocations()
645 {
646 // Atomic with relaxed order reason: data race with native_objects_notified_ with no synchronization or ordering
647 // constraints imposed on other reads or writes
648 nativeObjectsNotified_.fetch_add(NOTIFY_NATIVE_INTERVAL, std::memory_order_relaxed);
649 TriggerGCForNative();
650 }
651
RegisterNativeAllocation(size_t bytes)652 void GC::RegisterNativeAllocation(size_t bytes)
653 {
654 ASSERT_NATIVE_CODE();
655 size_t allocated;
656 do {
657 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
658 // constraints imposed on other reads or writes
659 allocated = nativeBytesRegistered_.load(std::memory_order_relaxed);
660 } while (!nativeBytesRegistered_.compare_exchange_weak(allocated, allocated + bytes));
661 if (allocated > std::numeric_limits<size_t>::max() - bytes) {
662 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
663 // constraints imposed on other reads or writes
664 nativeBytesRegistered_.store(std::numeric_limits<size_t>::max(), std::memory_order_relaxed);
665 }
666 TriggerGCForNative();
667 }
668
RegisterNativeFree(size_t bytes)669 void GC::RegisterNativeFree(size_t bytes)
670 {
671 size_t allocated;
672 size_t newFreedBytes;
673 do {
674 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
675 // constraints imposed on other reads or writes
676 allocated = nativeBytesRegistered_.load(std::memory_order_relaxed);
677 newFreedBytes = std::min(allocated, bytes);
678 } while (!nativeBytesRegistered_.compare_exchange_weak(allocated, allocated - newFreedBytes));
679 }
680
GetNativeBytesFromMallinfoAndRegister() const681 size_t GC::GetNativeBytesFromMallinfoAndRegister() const
682 {
683 size_t mallinfoBytes = ark::os::mem::GetNativeBytesFromMallinfo();
684 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
685 // constraints imposed on other reads or writes
686 size_t allBytes = mallinfoBytes + nativeBytesRegistered_.load(std::memory_order_relaxed);
687 return allBytes;
688 }
689
WaitForGC(GCTask task)690 bool GC::WaitForGC(GCTask task)
691 {
692 // NOTE(maksenov): Notify only about pauses (#4681)
693 Runtime::GetCurrent()->GetNotificationManager()->GarbageCollectorStartEvent();
694 // Atomic with acquire order reason: data race with gc_counter_ with dependecies on reads after the load which
695 // should become visible
696 auto oldCounter = this->gcCounter_.load(std::memory_order_acquire);
697 Timing suspendThreadsTiming;
698 {
699 ScopedTiming t("SuspendThreads", suspendThreadsTiming);
700 this->GetPandaVm()->GetRendezvous()->SafepointBegin();
701 }
702 if (!this->NeedRunGCAfterWaiting(oldCounter, task)) {
703 this->GetPandaVm()->GetRendezvous()->SafepointEnd();
704 return false;
705 }
706
707 // Create a copy of the constant GCTask to be able to change its value
708 this->RunPhases(task);
709
710 if (UNLIKELY(this->IsLogDetailedGcInfoEnabled())) {
711 PrintDetailedLog();
712 }
713
714 this->GetPandaVm()->GetRendezvous()->SafepointEnd();
715 Runtime::GetCurrent()->GetNotificationManager()->GarbageCollectorFinishEvent();
716 this->GetPandaVm()->HandleGCFinished();
717 this->GetPandaVm()->HandleEnqueueReferences();
718 this->GetPandaVm()->ProcessReferenceFinalizers();
719 return true;
720 }
721
WaitForGCInManaged(const GCTask & task)722 bool GC::WaitForGCInManaged(const GCTask &task)
723 {
724 Thread *baseThread = Thread::GetCurrent();
725 if (ManagedThread::ThreadIsManagedThread(baseThread)) {
726 ManagedThread *thread = ManagedThread::CastFromThread(baseThread);
727 ASSERT(thread->GetMutatorLock()->HasLock());
728 [[maybe_unused]] bool isDaemon = MTManagedThread::ThreadIsMTManagedThread(baseThread) &&
729 MTManagedThread::CastFromThread(baseThread)->IsDaemon();
730 ASSERT(!isDaemon || thread->GetStatus() == ThreadStatus::RUNNING);
731 vm_->GetMutatorLock()->Unlock();
732 thread->PrintSuspensionStackIfNeeded();
733 WaitForGC(task);
734 vm_->GetMutatorLock()->ReadLock();
735 ASSERT(vm_->GetMutatorLock()->HasLock());
736 this->GetPandaVm()->HandleGCRoutineInMutator();
737 return true;
738 }
739 return false;
740 }
741
StartConcurrentScopeRoutine() const742 void GC::StartConcurrentScopeRoutine() const {}
743
EndConcurrentScopeRoutine() const744 void GC::EndConcurrentScopeRoutine() const {}
745
PrintDetailedLog()746 void GC::PrintDetailedLog()
747 {
748 for (auto &footprint : this->footprintList_) {
749 LOG(INFO, GC) << footprint.first << " : " << footprint.second;
750 }
751 LOG(INFO, GC) << this->GetTiming()->Dump();
752 }
753
ConcurrentScope(GC * gc,bool autoStart)754 ConcurrentScope::ConcurrentScope(GC *gc, bool autoStart)
755 {
756 LOG(DEBUG, GC) << "Start ConcurrentScope";
757 gc_ = gc;
758 if (autoStart) {
759 Start();
760 }
761 }
762
~ConcurrentScope()763 ConcurrentScope::~ConcurrentScope()
764 {
765 LOG(DEBUG, GC) << "Stop ConcurrentScope";
766 if (started_ && gc_->IsConcurrencyAllowed()) {
767 gc_->GetPandaVm()->GetRendezvous()->SafepointBegin();
768 gc_->SetupCpuAffinityAfterConcurrent();
769 gc_->EndConcurrentScopeRoutine();
770 }
771 }
772
Start()773 NO_THREAD_SAFETY_ANALYSIS void ConcurrentScope::Start()
774 {
775 if (!started_ && gc_->IsConcurrencyAllowed()) {
776 gc_->StartConcurrentScopeRoutine();
777 gc_->SetupCpuAffinityBeforeConcurrent();
778 gc_->GetPandaVm()->GetRendezvous()->SafepointEnd();
779 started_ = true;
780 }
781 }
782
WaitForGCOnPygoteFork(const GCTask & task)783 void GC::WaitForGCOnPygoteFork(const GCTask &task)
784 {
785 // do nothing if no pygote space
786 auto pygoteSpaceAllocator = objectAllocator_->GetPygoteSpaceAllocator();
787 if (pygoteSpaceAllocator == nullptr) {
788 return;
789 }
790
791 // do nothing if not at first pygote fork
792 if (pygoteSpaceAllocator->GetState() != PygoteSpaceState::STATE_PYGOTE_INIT) {
793 return;
794 }
795
796 LOG(DEBUG, GC) << "== GC WaitForGCOnPygoteFork Start ==";
797
798 // do we need a lock?
799 // looks all other threads have been stopped before pygote fork
800
801 // 0. indicate that we're rebuilding pygote space
802 pygoteSpaceAllocator->SetState(PygoteSpaceState::STATE_PYGOTE_FORKING);
803
804 // 1. trigger gc
805 WaitForGC(task);
806
807 // 2. move other space to pygote space
808 MoveObjectsToPygoteSpace();
809
810 // 3. indicate that we have done
811 pygoteSpaceAllocator->SetState(PygoteSpaceState::STATE_PYGOTE_FORKED);
812
813 // 4. disable pygote for allocation
814 objectAllocator_->DisablePygoteAlloc();
815
816 LOG(DEBUG, GC) << "== GC WaitForGCOnPygoteFork End ==";
817 }
818
IsOnPygoteFork() const819 bool GC::IsOnPygoteFork() const
820 {
821 auto pygoteSpaceAllocator = objectAllocator_->GetPygoteSpaceAllocator();
822 return pygoteSpaceAllocator != nullptr &&
823 pygoteSpaceAllocator->GetState() == PygoteSpaceState::STATE_PYGOTE_FORKING;
824 }
825
MoveObjectsToPygoteSpace()826 void GC::MoveObjectsToPygoteSpace()
827 {
828 trace::ScopedTrace scopedTrace(__FUNCTION__);
829 LOG(DEBUG, GC) << "MoveObjectsToPygoteSpace: start";
830
831 size_t allSizeMove = 0;
832 size_t movedObjectsNum = 0;
833 size_t bytesInHeapBeforeMove = GetPandaVm()->GetMemStats()->GetFootprintHeap();
834 auto pygoteSpaceAllocator = objectAllocator_->GetPygoteSpaceAllocator();
835 ObjectVisitor moveVisitor([this, &pygoteSpaceAllocator, &movedObjectsNum, &allSizeMove](ObjectHeader *src) -> void {
836 size_t size = GetObjectSize(src);
837 auto dst = reinterpret_cast<ObjectHeader *>(pygoteSpaceAllocator->Alloc(size));
838 ASSERT(dst != nullptr);
839 memcpy_s(dst, size, src, size);
840 allSizeMove += size;
841 movedObjectsNum++;
842 SetForwardAddress(src, dst);
843 LOG_DEBUG_GC << "object MOVED from " << std::hex << src << " to " << dst << ", size = " << std::dec << size;
844 });
845
846 // move all small movable objects to pygote space
847 objectAllocator_->IterateRegularSizeObjects(moveVisitor);
848
849 LOG(DEBUG, GC) << "MoveObjectsToPygoteSpace: move_num = " << movedObjectsNum << ", move_size = " << allSizeMove;
850
851 if (allSizeMove > 0) {
852 GetStats()->AddMemoryValue(allSizeMove, MemoryTypeStats::MOVED_BYTES);
853 GetStats()->AddObjectsValue(movedObjectsNum, ObjectTypeStats::MOVED_OBJECTS);
854 }
855 if (bytesInHeapBeforeMove > 0) {
856 GetStats()->AddCopiedRatioValue(static_cast<double>(allSizeMove) / bytesInHeapBeforeMove);
857 }
858
859 // Update because we moved objects from object_allocator -> pygote space
860 UpdateRefsToMovedObjectsInPygoteSpace();
861 CommonUpdateRefsToMovedObjects();
862
863 // Clear the moved objects in old space
864 objectAllocator_->FreeObjectsMovedToPygoteSpace();
865
866 LOG(DEBUG, GC) << "MoveObjectsToPygoteSpace: finish";
867 }
868
SetForwardAddress(ObjectHeader * src,ObjectHeader * dst)869 void GC::SetForwardAddress(ObjectHeader *src, ObjectHeader *dst)
870 {
871 auto baseCls = src->ClassAddr<BaseClass>();
872 if (baseCls->IsDynamicClass()) {
873 auto cls = static_cast<HClass *>(baseCls);
874 // Note: During moving phase, 'src => dst'. Consider the src is a DynClass,
875 // since 'dst' is not in GC-status the 'manage-object' inside 'dst' won't be updated to
876 // 'dst'. To fix it, we update 'manage-object' here rather than upating phase.
877 if (cls->IsHClass()) {
878 size_t offset = ObjectHeader::ObjectHeaderSize() + HClass::GetManagedObjectOffset();
879 dst->SetFieldObject<false, false, true>(GetPandaVm()->GetAssociatedThread(), offset, dst);
880 }
881 }
882
883 // Set fwd address in src
884 bool updateRes = false;
885 do {
886 MarkWord markWord = src->AtomicGetMark();
887 MarkWord fwdMarkWord =
888 markWord.DecodeFromForwardingAddress(static_cast<MarkWord::MarkWordSize>(ToUintPtr(dst)));
889 updateRes = src->AtomicSetMark<false>(markWord, fwdMarkWord);
890 } while (!updateRes);
891 }
892
UpdateRefsInVRegs(ManagedThread * thread)893 void GC::UpdateRefsInVRegs(ManagedThread *thread)
894 {
895 LOG_DEBUG_GC << "Update frames for thread: " << thread->GetId();
896 for (auto pframe = StackWalker::Create(thread); pframe.HasFrame(); pframe.NextFrame()) {
897 LOG_DEBUG_GC << "Frame for method " << pframe.GetMethod()->GetFullName();
898 auto iterator = [&pframe, this](auto ®Info, auto &vreg) {
899 ObjectHeader *objectHeader = vreg.GetReference();
900 if (objectHeader == nullptr) {
901 return true;
902 }
903 MarkWord markWord = objectHeader->AtomicGetMark();
904 if (markWord.GetState() != MarkWord::ObjectState::STATE_GC) {
905 return true;
906 }
907 MarkWord::MarkWordSize addr = markWord.GetForwardingAddress();
908 LOG_DEBUG_GC << "Update vreg, vreg old val = " << std::hex << objectHeader << ", new val = 0x" << addr;
909 LOG_IF(regInfo.IsAccumulator(), DEBUG, GC) << "^ acc reg";
910 if (!pframe.IsCFrame() && regInfo.IsAccumulator()) {
911 LOG_DEBUG_GC << "^ acc updated";
912 vreg.SetReference(reinterpret_cast<ObjectHeader *>(addr));
913 } else {
914 pframe.template SetVRegValue<std::is_same_v<decltype(vreg), interpreter::DynamicVRegisterRef &>>(
915 regInfo, reinterpret_cast<ObjectHeader *>(addr));
916 }
917 return true;
918 };
919 pframe.IterateObjectsWithInfo(iterator);
920 }
921 }
922
PopObjectFromStack(GCMarkingStackType * objectsStack)923 const ObjectHeader *GC::PopObjectFromStack(GCMarkingStackType *objectsStack)
924 {
925 auto *object = objectsStack->PopFromStack();
926 ASSERT(object != nullptr);
927 return object;
928 }
929
IsGenerational() const930 bool GC::IsGenerational() const
931 {
932 return IsGenerationalGCType(gcType_);
933 }
934
AddListener(GCListener * listener)935 void GC::GCListenerManager::AddListener(GCListener *listener)
936 {
937 os::memory::LockHolder lh(listenerLock_);
938 newListeners_.insert(listener);
939 }
940
RemoveListener(GCListener * listener)941 void GC::GCListenerManager::RemoveListener(GCListener *listener)
942 {
943 os::memory::LockHolder lh(listenerLock_);
944 listenersForRemove_.insert(listener);
945 }
946
NormalizeListenersOnStartGC()947 void GC::GCListenerManager::NormalizeListenersOnStartGC()
948 {
949 os::memory::LockHolder lh(listenerLock_);
950 for (auto *listenerForRemove : listenersForRemove_) {
951 if (newListeners_.find(listenerForRemove) != newListeners_.end()) {
952 newListeners_.erase(listenerForRemove);
953 }
954 auto it = currentListeners_.find(listenerForRemove);
955 if (it != currentListeners_.end()) {
956 LOG(DEBUG, GC) << "Remove listener for GC: " << listenerForRemove;
957 currentListeners_.erase(it);
958 }
959 }
960 listenersForRemove_.clear();
961 for (auto *newListener : newListeners_) {
962 LOG(DEBUG, GC) << "Add new listener for GC: " << newListener;
963 currentListeners_.insert(newListener);
964 }
965 newListeners_.clear();
966 }
967
FireGCStarted(const GCTask & task,size_t bytesInHeapBeforeGc)968 void GC::FireGCStarted(const GCTask &task, size_t bytesInHeapBeforeGc)
969 {
970 gcListenerManager_->NormalizeListenersOnStartGC();
971 gcListenerManager_->IterateOverListeners(
972 [&](GCListener *listener) { listener->GCStarted(task, bytesInHeapBeforeGc); });
973 }
974
FireGCFinished(const GCTask & task,size_t bytesInHeapBeforeGc,size_t bytesInHeapAfterGc)975 void GC::FireGCFinished(const GCTask &task, size_t bytesInHeapBeforeGc, size_t bytesInHeapAfterGc)
976 {
977 gcListenerManager_->IterateOverListeners(
978 [&](GCListener *listener) { listener->GCFinished(task, bytesInHeapBeforeGc, bytesInHeapAfterGc); });
979 }
980
FireGCPhaseStarted(GCPhase phase)981 void GC::FireGCPhaseStarted(GCPhase phase)
982 {
983 gcListenerManager_->IterateOverListeners([phase](GCListener *listener) { listener->GCPhaseStarted(phase); });
984 }
985
FireGCPhaseFinished(GCPhase phase)986 void GC::FireGCPhaseFinished(GCPhase phase)
987 {
988 gcListenerManager_->IterateOverListeners([phase](GCListener *listener) { listener->GCPhaseFinished(phase); });
989 }
990
OnWaitForIdleFail()991 void GC::OnWaitForIdleFail() {}
992
993 TEMPLATE_GC_CREATE_GC();
994
995 } // namespace ark::mem
996