1 /**
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include <memory>
17
18 #include "libpandabase/os/cpu_affinity.h"
19 #include "libpandabase/os/mem.h"
20 #include "libpandabase/os/thread.h"
21 #include "libpandabase/utils/time.h"
22 #include "runtime/assert_gc_scope.h"
23 #include "runtime/include/class.h"
24 #include "runtime/include/coretypes/dyn_objects.h"
25 #include "runtime/include/locks.h"
26 #include "runtime/include/runtime.h"
27 #include "runtime/include/runtime_notification.h"
28 #include "runtime/include/stack_walker-inl.h"
29 #include "runtime/mem/gc/epsilon/epsilon.h"
30 #include "runtime/mem/gc/epsilon-g1/epsilon-g1.h"
31 #include "runtime/mem/gc/gc.h"
32 #include "runtime/mem/gc/gc_root-inl.h"
33 #include "runtime/mem/gc/g1/g1-gc.h"
34 #include "runtime/mem/gc/gen-gc/gen-gc.h"
35 #include "runtime/mem/gc/stw-gc/stw-gc.h"
36 #include "runtime/mem/gc/workers/gc_workers_task_queue.h"
37 #include "runtime/mem/gc/workers/gc_workers_thread_pool.h"
38 #include "runtime/mem/pygote_space_allocator-inl.h"
39 #include "runtime/mem/heap_manager.h"
40 #include "runtime/mem/gc/reference-processor/reference_processor.h"
41 #include "runtime/mem/gc/gc-hung/gc_hung.h"
42 #include "runtime/include/panda_vm.h"
43 #include "runtime/include/object_accessor-inl.h"
44 #include "runtime/include/coretypes/class.h"
45 #include "runtime/thread_manager.h"
46
47 namespace panda::mem {
48 using TaggedValue = coretypes::TaggedValue;
49 using TaggedType = coretypes::TaggedType;
50 using DynClass = coretypes::DynClass;
51
GC(ObjectAllocatorBase * objectAllocator,const GCSettings & settings)52 GC::GC(ObjectAllocatorBase *objectAllocator, const GCSettings &settings)
53 : gcSettings_(settings),
54 objectAllocator_(objectAllocator),
55 internalAllocator_(InternalAllocator<>::GetInternalAllocatorFromRuntime())
56 {
57 if (gcSettings_.UseTaskManagerForGC()) {
58 // Create gc task queue for task manager
59 auto *tm = taskmanager::TaskScheduler::GetTaskScheduler();
60 gcWorkersTaskQueue_ = tm->CreateAndRegisterTaskQueue<decltype(internalAllocator_->Adapter())>(
61 taskmanager::TaskType::GC, taskmanager::VMType::STATIC_VM, GC_TASK_QUEUE_PRIORITY);
62 ASSERT(gcWorkersTaskQueue_ != nullptr);
63 }
64 }
65
~GC()66 GC::~GC()
67 {
68 InternalAllocatorPtr allocator = GetInternalAllocator();
69 if (gcWorker_ != nullptr) {
70 allocator->Delete(gcWorker_);
71 }
72 if (gcListenerManager_ != nullptr) {
73 allocator->Delete(gcListenerManager_);
74 }
75 if (gcBarrierSet_ != nullptr) {
76 allocator->Delete(gcBarrierSet_);
77 }
78 if (clearedReferences_ != nullptr) {
79 allocator->Delete(clearedReferences_);
80 }
81 if (clearedReferencesLock_ != nullptr) {
82 allocator->Delete(clearedReferencesLock_);
83 }
84 if (workersTaskPool_ != nullptr) {
85 allocator->Delete(workersTaskPool_);
86 }
87 if (gcWorkersTaskQueue_ != nullptr) {
88 taskmanager::TaskScheduler::GetTaskScheduler()->UnregisterAndDestroyTaskQueue<decltype(allocator->Adapter())>(
89 gcWorkersTaskQueue_);
90 }
91 }
92
GetLogPrefix() const93 Logger::Buffer GC::GetLogPrefix() const
94 {
95 const char *phase = GCScopedPhase::GetPhaseAbbr(GetGCPhase());
96 // Atomic with acquire order reason: data race with gc_counter_
97 size_t counter = gcCounter_.load(std::memory_order_acquire);
98
99 Logger::Buffer buffer;
100 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
101 buffer.Printf("[%zu, %s]: ", counter, phase);
102
103 return buffer;
104 }
105
GetType()106 GCType GC::GetType()
107 {
108 return gcType_;
109 }
110
SetPandaVM(PandaVM * vm)111 void GC::SetPandaVM(PandaVM *vm)
112 {
113 vm_ = vm;
114 referenceProcessor_ = vm->GetReferenceProcessor();
115 }
116
GetNativeGcTriggerType()117 NativeGcTriggerType GC::GetNativeGcTriggerType()
118 {
119 return gcSettings_.GetNativeGcTriggerType();
120 }
121
SimpleNativeAllocationGcWatermark()122 size_t GC::SimpleNativeAllocationGcWatermark()
123 {
124 return GetPandaVm()->GetOptions().GetMaxFree();
125 }
126
WaitForIdleGC()127 NO_THREAD_SAFETY_ANALYSIS void GC::WaitForIdleGC()
128 {
129 while (!CASGCPhase(GCPhase::GC_PHASE_IDLE, GCPhase::GC_PHASE_RUNNING)) {
130 GetPandaVm()->GetRendezvous()->SafepointEnd();
131 // Interrupt the running GC if possible
132 OnWaitForIdleFail();
133 // NOTE(dtrubenkov): resolve it more properly
134 constexpr uint64_t WAIT_FINISHED = 100;
135 // Use NativeSleep for all threads, as this thread shouldn't hold Mutator lock here
136 os::thread::NativeSleepUS(std::chrono::microseconds(WAIT_FINISHED));
137 GetPandaVm()->GetRendezvous()->SafepointBegin();
138 }
139 }
140
TriggerGCForNative()141 inline void GC::TriggerGCForNative()
142 {
143 auto nativeGcTriggerType = GetNativeGcTriggerType();
144 ASSERT_PRINT((nativeGcTriggerType == NativeGcTriggerType::NO_NATIVE_GC_TRIGGER) ||
145 (nativeGcTriggerType == NativeGcTriggerType::SIMPLE_STRATEGY),
146 "Unknown Native GC Trigger type");
147 switch (nativeGcTriggerType) {
148 case NativeGcTriggerType::NO_NATIVE_GC_TRIGGER:
149 break;
150 case NativeGcTriggerType::SIMPLE_STRATEGY:
151 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or
152 // ordering constraints imposed on other reads or writes
153 if (nativeBytesRegistered_.load(std::memory_order_relaxed) > SimpleNativeAllocationGcWatermark()) {
154 auto task = MakePandaUnique<GCTask>(GCTaskCause::NATIVE_ALLOC_CAUSE, time::GetCurrentTimeInNanos());
155 AddGCTask(false, std::move(task));
156 ManagedThread::GetCurrent()->SafepointPoll();
157 }
158 break;
159 default:
160 LOG(FATAL, GC) << "Unknown Native GC Trigger type";
161 break;
162 }
163 }
164
Initialize(PandaVM * vm)165 void GC::Initialize(PandaVM *vm)
166 {
167 trace::ScopedTrace scopedTrace(__PRETTY_FUNCTION__);
168 // GC saved the PandaVM instance, so we get allocator from the PandaVM.
169 auto allocator = GetInternalAllocator();
170 gcListenerManager_ = allocator->template New<GCListenerManager>();
171 clearedReferencesLock_ = allocator->New<os::memory::Mutex>();
172 os::memory::LockHolder holder(*clearedReferencesLock_);
173 clearedReferences_ = allocator->New<PandaVector<panda::mem::Reference *>>(allocator->Adapter());
174 this->SetPandaVM(vm);
175 InitializeImpl();
176 gcWorker_ = allocator->New<GCWorker>(this);
177 }
178
CreateWorkersTaskPool()179 void GC::CreateWorkersTaskPool()
180 {
181 ASSERT(workersTaskPool_ == nullptr);
182 if (this->IsWorkerThreadsExist()) {
183 auto allocator = GetInternalAllocator();
184 GCWorkersTaskPool *gcTaskPool = nullptr;
185 if (this->GetSettings()->UseThreadPoolForGC()) {
186 // Use internal gc thread pool
187 gcTaskPool = allocator->New<GCWorkersThreadPool>(this, this->GetSettings()->GCWorkersCount());
188 } else {
189 // Use common TaskManager
190 ASSERT(this->GetSettings()->UseTaskManagerForGC());
191 gcTaskPool = allocator->New<GCWorkersTaskQueue>(this);
192 }
193 ASSERT(gcTaskPool != nullptr);
194 workersTaskPool_ = gcTaskPool;
195 }
196 }
197
DestroyWorkersTaskPool()198 void GC::DestroyWorkersTaskPool()
199 {
200 if (workersTaskPool_ == nullptr) {
201 return;
202 }
203 workersTaskPool_->WaitUntilTasksEnd();
204 auto allocator = this->GetInternalAllocator();
205 allocator->Delete(workersTaskPool_);
206 workersTaskPool_ = nullptr;
207 }
208
StartGC()209 void GC::StartGC()
210 {
211 CreateWorker();
212 }
213
StopGC()214 void GC::StopGC()
215 {
216 DestroyWorker();
217 DestroyWorkersTaskPool();
218 }
219
SetupCpuAffinity()220 void GC::SetupCpuAffinity()
221 {
222 if (!gcSettings_.ManageGcThreadsAffinity()) {
223 return;
224 }
225 // Try to get CPU affinity fo GC Thread
226 if (UNLIKELY(!os::CpuAffinityManager::GetCurrentThreadAffinity(affinityBeforeGc_))) {
227 affinityBeforeGc_.Clear();
228 return;
229 }
230 // Try to use best + middle for preventing issues when best core is used in another thread,
231 // and GC waits for it to finish.
232 if (!os::CpuAffinityManager::SetAffinityForCurrentThread(os::CpuPower::BEST | os::CpuPower::MIDDLE)) {
233 affinityBeforeGc_.Clear();
234 }
235 // Some GCs don't use GC Workers
236 if (workersTaskPool_ != nullptr && this->GetSettings()->UseThreadPoolForGC()) {
237 static_cast<GCWorkersThreadPool *>(workersTaskPool_)->SetAffinityForGCWorkers();
238 }
239 }
240
SetupCpuAffinityAfterConcurrent()241 void GC::SetupCpuAffinityAfterConcurrent()
242 {
243 if (!gcSettings_.ManageGcThreadsAffinity()) {
244 return;
245 }
246 // Try to set GC Thread on best CPUs after concurrent
247 if (!os::CpuAffinityManager::SetAffinityForCurrentThread(os::CpuPower::BEST)) {
248 // If it failed for only best CPUs then try to use best + middle CPUs mask after concurrent
249 os::CpuAffinityManager::SetAffinityForCurrentThread(os::CpuPower::BEST | os::CpuPower::MIDDLE);
250 }
251 // Some GCs don't use GC Workers
252 if (workersTaskPool_ != nullptr && this->GetSettings()->UseThreadPoolForGC()) {
253 static_cast<GCWorkersThreadPool *>(workersTaskPool_)->SetAffinityForGCWorkers();
254 }
255 }
256
ResetCpuAffinity(bool beforeConcurrent)257 void GC::ResetCpuAffinity(bool beforeConcurrent)
258 {
259 if (!gcSettings_.ManageGcThreadsAffinity()) {
260 return;
261 }
262 if (!affinityBeforeGc_.IsEmpty()) {
263 // Set GC Threads on weak CPUs before concurrent if needed
264 if (beforeConcurrent && gcSettings_.UseWeakCpuForGcConcurrent()) {
265 os::CpuAffinityManager::SetAffinityForCurrentThread(os::CpuPower::WEAK);
266 } else { // else set on saved affinity
267 os::CpuAffinityManager::SetAffinityForCurrentThread(affinityBeforeGc_);
268 }
269 }
270 // Some GCs don't use GC Workers
271 if (workersTaskPool_ != nullptr && this->GetSettings()->UseThreadPoolForGC()) {
272 static_cast<GCWorkersThreadPool *>(workersTaskPool_)->UnsetAffinityForGCWorkers();
273 }
274 }
275
SetupCpuAffinityBeforeConcurrent()276 void GC::SetupCpuAffinityBeforeConcurrent()
277 {
278 ResetCpuAffinity(true);
279 }
280
RestoreCpuAffinity()281 void GC::RestoreCpuAffinity()
282 {
283 ResetCpuAffinity(false);
284 }
285
NeedRunGCAfterWaiting(size_t counterBeforeWaiting,const GCTask & task) const286 bool GC::NeedRunGCAfterWaiting(size_t counterBeforeWaiting, const GCTask &task) const
287 {
288 // Atomic with acquire order reason: data race with gc_counter_ with dependecies on reads after the load which
289 // should become visible
290 auto newCounter = gcCounter_.load(std::memory_order_acquire);
291 ASSERT(newCounter >= counterBeforeWaiting);
292 // Atomic with acquire order reason: data race with last_cause_ with dependecies on reads after the load which
293 // should become visible
294 return (newCounter == counterBeforeWaiting || lastCause_.load(std::memory_order_acquire) < task.reason);
295 }
296
297 // NOLINTNEXTLINE(performance-unnecessary-value-param)
RunPhases(GCTask & task)298 void GC::RunPhases(GCTask &task)
299 {
300 DCHECK_ALLOW_GARBAGE_COLLECTION;
301 trace::ScopedTrace scopedTrace(__FUNCTION__);
302 // Atomic with acquire order reason: data race with gc_counter_ with dependecies on reads after the load which
303 // should become visible
304 auto oldCounter = gcCounter_.load(std::memory_order_acquire);
305 WaitForIdleGC();
306 if (!this->NeedRunGCAfterWaiting(oldCounter, task)) {
307 SetGCPhase(GCPhase::GC_PHASE_IDLE);
308 return;
309 }
310 this->SetupCpuAffinity();
311 this->GetTiming()->Reset(); // Clear records.
312 // Atomic with release order reason: data race with last_cause_ with dependecies on writes before the store which
313 // should become visible acquire
314 lastCause_.store(task.reason, std::memory_order_release);
315 if (gcSettings_.PreGCHeapVerification()) {
316 trace::ScopedTrace preHeapVerifierTrace("PreGCHeapVeriFier");
317 size_t failCount = VerifyHeap();
318 if (gcSettings_.FailOnHeapVerification() && failCount > 0) {
319 LOG(FATAL, GC) << "Heap corrupted before GC, HeapVerifier found " << failCount << " corruptions";
320 }
321 }
322 // Atomic with acq_rel order reason: data race with gc_counter_ with dependecies on reads after the load and on
323 // writes before the store
324 gcCounter_.fetch_add(1, std::memory_order_acq_rel);
325 if (gcSettings_.IsDumpHeap()) {
326 PandaOStringStream os;
327 os << "Heap dump before GC" << std::endl;
328 GetPandaVm()->DumpHeap(&os);
329 std::cerr << os.str() << std::endl;
330 }
331 size_t bytesInHeapBeforeGc = GetPandaVm()->GetMemStats()->GetFootprintHeap();
332 LOG_DEBUG_GC << "Bytes in heap before GC " << std::dec << bytesInHeapBeforeGc;
333 {
334 GCScopedStats scopedStats(GetPandaVm()->GetGCStats(), gcType_ == GCType::STW_GC ? GetStats() : nullptr);
335 ScopedGcHung scopedHung(&task);
336 GetPandaVm()->GetGCStats()->ResetLastPause();
337
338 FireGCStarted(task, bytesInHeapBeforeGc);
339 PreRunPhasesImpl();
340 // NOLINTNEXTLINE(performance-unnecessary-value-param)
341 RunPhasesImpl(task);
342 // Clear Internal allocator unused pools (must do it on pause to avoid race conditions):
343 // - Clear global part:
344 InternalAllocator<>::GetInternalAllocatorFromRuntime()->VisitAndRemoveFreePools(
345 [](void *mem, [[maybe_unused]] size_t size) { PoolManager::GetMmapMemPool()->FreePool(mem, size); });
346 // - Clear local part:
347 ClearLocalInternalAllocatorPools();
348
349 size_t bytesInHeapAfterGc = GetPandaVm()->GetMemStats()->GetFootprintHeap();
350 // There is case than bytes_in_heap_after_gc > 0 and bytes_in_heap_before_gc == 0.
351 // Because TLABs are registered during GC
352 if (bytesInHeapAfterGc > 0 && bytesInHeapBeforeGc > 0) {
353 GetStats()->AddReclaimRatioValue(1 - static_cast<double>(bytesInHeapAfterGc) / bytesInHeapBeforeGc);
354 }
355 LOG_DEBUG_GC << "Bytes in heap after GC " << std::dec << bytesInHeapAfterGc;
356 FireGCFinished(task, bytesInHeapBeforeGc, bytesInHeapAfterGc);
357 }
358 ASSERT(task.collectionType != GCCollectionType::NONE);
359 LOG(INFO, GC) << "[" << gcCounter_ << "] [" << task.collectionType << " (" << task.reason << ")] "
360 << GetPandaVm()->GetGCStats()->GetStatistics();
361
362 if (gcSettings_.IsDumpHeap()) {
363 PandaOStringStream os;
364 os << "Heap dump after GC" << std::endl;
365 GetPandaVm()->DumpHeap(&os);
366 std::cerr << os.str() << std::endl;
367 }
368
369 if (gcSettings_.PostGCHeapVerification()) {
370 trace::ScopedTrace postHeapVerifierTrace("PostGCHeapVeriFier");
371 size_t failCount = VerifyHeap();
372 if (gcSettings_.FailOnHeapVerification() && failCount > 0) {
373 LOG(FATAL, GC) << "Heap corrupted after GC, HeapVerifier found " << failCount << " corruptions";
374 }
375 }
376 this->RestoreCpuAffinity();
377
378 SetGCPhase(GCPhase::GC_PHASE_IDLE);
379 }
380
381 template <class LanguageConfig>
CreateGC(GCType gcType,ObjectAllocatorBase * objectAllocator,const GCSettings & settings)382 GC *CreateGC(GCType gcType, ObjectAllocatorBase *objectAllocator, const GCSettings &settings)
383 {
384 GC *ret = nullptr;
385 InternalAllocatorPtr allocator {InternalAllocator<>::GetInternalAllocatorFromRuntime()};
386
387 switch (gcType) {
388 case GCType::EPSILON_GC:
389 ret = allocator->New<EpsilonGC<LanguageConfig>>(objectAllocator, settings);
390 break;
391 case GCType::EPSILON_G1_GC:
392 ret = allocator->New<EpsilonG1GC<LanguageConfig>>(objectAllocator, settings);
393 break;
394 case GCType::STW_GC:
395 ret = allocator->New<StwGC<LanguageConfig>>(objectAllocator, settings);
396 break;
397 case GCType::GEN_GC:
398 ret = allocator->New<GenGC<LanguageConfig>>(objectAllocator, settings);
399 break;
400 case GCType::G1_GC:
401 ret = allocator->New<G1GC<LanguageConfig>>(objectAllocator, settings);
402 break;
403 default:
404 LOG(FATAL, GC) << "Unknown GC type";
405 break;
406 }
407 return ret;
408 }
409
CheckGCCause(GCTaskCause cause) const410 bool GC::CheckGCCause(GCTaskCause cause) const
411 {
412 return cause != GCTaskCause::INVALID_CAUSE;
413 }
414
MarkObjectIfNotMarked(ObjectHeader * objectHeader)415 bool GC::MarkObjectIfNotMarked(ObjectHeader *objectHeader)
416 {
417 ASSERT(objectHeader != nullptr);
418 if (IsMarked(objectHeader)) {
419 return false;
420 }
421 MarkObject(objectHeader);
422 return true;
423 }
424
ProcessReference(GCMarkingStackType * objectsStack,const BaseClass * cls,const ObjectHeader * ref,const ReferenceProcessPredicateT & pred)425 void GC::ProcessReference(GCMarkingStackType *objectsStack, const BaseClass *cls, const ObjectHeader *ref,
426 const ReferenceProcessPredicateT &pred)
427 {
428 ASSERT(referenceProcessor_ != nullptr);
429 referenceProcessor_->HandleReference(this, objectsStack, cls, ref, pred);
430 }
431
AddReference(ObjectHeader * fromObj,ObjectHeader * object)432 void GC::AddReference(ObjectHeader *fromObj, ObjectHeader *object)
433 {
434 ASSERT(IsMarked(object));
435 GCMarkingStackType references(this);
436 // NOTE(alovkov): support stack with workers here & put all refs in stack and only then process altogether for once
437 ASSERT(!references.IsWorkersTaskSupported());
438 references.PushToStack(fromObj, object);
439 MarkReferences(&references, phase_);
440 if (gcType_ != GCType::EPSILON_GC) {
441 ASSERT(references.Empty());
442 }
443 }
444
445 // NOLINTNEXTLINE(performance-unnecessary-value-param)
ProcessReferences(GCPhase gcPhase,const GCTask & task,const ReferenceClearPredicateT & pred)446 void GC::ProcessReferences(GCPhase gcPhase, const GCTask &task, const ReferenceClearPredicateT &pred)
447 {
448 trace::ScopedTrace scopedTrace(__FUNCTION__);
449 LOG(DEBUG, REF_PROC) << "Start processing cleared references";
450 ASSERT(referenceProcessor_ != nullptr);
451 bool clearSoftReferences = task.reason == GCTaskCause::OOM_CAUSE || IsExplicitFull(task);
452 referenceProcessor_->ProcessReferences(false, clearSoftReferences, gcPhase, pred);
453 Reference *processedRef = referenceProcessor_->CollectClearedReferences();
454 if (processedRef != nullptr) {
455 os::memory::LockHolder holder(*clearedReferencesLock_);
456 // NOTE(alovkov): ged rid of cleared_references_ and just enqueue refs here?
457 clearedReferences_->push_back(processedRef);
458 }
459 }
460
DestroyWorker()461 void GC::DestroyWorker()
462 {
463 // Atomic with seq_cst order reason: data race with gc_running_ with requirement for sequentially consistent order
464 // where threads observe all modifications in the same order
465 gcRunning_.store(false, std::memory_order_seq_cst);
466 gcWorker_->FinalizeAndDestroyWorker();
467 }
468
CreateWorker()469 void GC::CreateWorker()
470 {
471 // Atomic with seq_cst order reason: data race with gc_running_ with requirement for sequentially consistent order
472 // where threads observe all modifications in the same order
473 gcRunning_.store(true, std::memory_order_seq_cst);
474 ASSERT(gcWorker_ != nullptr);
475 gcWorker_->CreateAndStartWorker();
476 }
477
DisableWorkerThreads()478 void GC::DisableWorkerThreads()
479 {
480 gcSettings_.SetGCWorkersCount(0);
481 gcSettings_.SetParallelMarkingEnabled(false);
482 gcSettings_.SetParallelCompactingEnabled(false);
483 gcSettings_.SetParallelRefUpdatingEnabled(false);
484 }
485
EnableWorkerThreads()486 void GC::EnableWorkerThreads()
487 {
488 const RuntimeOptions &options = Runtime::GetOptions();
489 gcSettings_.SetGCWorkersCount(options.GetGcWorkersCount());
490 gcSettings_.SetParallelMarkingEnabled(options.IsGcParallelMarkingEnabled() && (options.GetGcWorkersCount() != 0));
491 gcSettings_.SetParallelCompactingEnabled(options.IsGcParallelCompactingEnabled() &&
492 (options.GetGcWorkersCount() != 0));
493 gcSettings_.SetParallelRefUpdatingEnabled(options.IsGcParallelRefUpdatingEnabled() &&
494 (options.GetGcWorkersCount() != 0));
495 }
496
PreZygoteFork()497 void GC::PreZygoteFork()
498 {
499 DestroyWorker();
500 if (gcSettings_.UseTaskManagerForGC()) {
501 ASSERT(gcWorkersTaskQueue_ != nullptr);
502 ASSERT(gcWorkersTaskQueue_->IsEmpty());
503 }
504 }
505
PostZygoteFork()506 void GC::PostZygoteFork()
507 {
508 CreateWorker();
509 }
510
511 class GC::PostForkGCTask : public GCTask {
512 public:
PostForkGCTask(GCTaskCause gcReason,uint64_t gcTargetTime)513 PostForkGCTask(GCTaskCause gcReason, uint64_t gcTargetTime) : GCTask(gcReason, gcTargetTime) {}
514
Run(mem::GC & gc)515 void Run(mem::GC &gc) override
516 {
517 LOG(DEBUG, GC) << "Runing PostForkGCTask";
518 gc.GetPandaVm()->GetGCTrigger()->RestoreMinTargetFootprint();
519 gc.PostForkCallback();
520 GCTask::Run(gc);
521 }
522
523 ~PostForkGCTask() override = default;
524
525 NO_COPY_SEMANTIC(PostForkGCTask);
526 NO_MOVE_SEMANTIC(PostForkGCTask);
527 };
528
PreStartup()529 void GC::PreStartup()
530 {
531 // Add a delay GCTask.
532 if ((!Runtime::GetCurrent()->IsZygote()) && (!gcSettings_.RunGCInPlace())) {
533 // divide 2 to temporarily set target footprint to a high value to disable GC during App startup.
534 GetPandaVm()->GetGCTrigger()->SetMinTargetFootprint(Runtime::GetOptions().GetHeapSizeLimit() / 2);
535 PreStartupImp();
536 constexpr uint64_t DISABLE_GC_DURATION_NS = 2000 * 1000 * 1000;
537 auto task = MakePandaUnique<PostForkGCTask>(GCTaskCause::STARTUP_COMPLETE_CAUSE,
538 time::GetCurrentTimeInNanos() + DISABLE_GC_DURATION_NS);
539 AddGCTask(true, std::move(task));
540 LOG(DEBUG, GC) << "Add PostForkGCTask";
541 }
542 }
543
544 // NOLINTNEXTLINE(performance-unnecessary-value-param)
AddGCTask(bool isManaged,PandaUniquePtr<GCTask> task)545 bool GC::AddGCTask(bool isManaged, PandaUniquePtr<GCTask> task)
546 {
547 bool triggeredByThreshold = (task->reason == GCTaskCause::HEAP_USAGE_THRESHOLD_CAUSE);
548 if (gcSettings_.RunGCInPlace()) {
549 auto *gcTask = task.get();
550 if (IsGCRunning()) {
551 if (isManaged) {
552 return WaitForGCInManaged(*gcTask);
553 }
554 return WaitForGC(*gcTask);
555 }
556 } else {
557 if (triggeredByThreshold) {
558 bool expect = true;
559 if (canAddGcTask_.compare_exchange_strong(expect, false, std::memory_order_seq_cst)) {
560 return gcWorker_->AddTask(std::move(task));
561 }
562 } else {
563 return gcWorker_->AddTask(std::move(task));
564 }
565 }
566 return false;
567 }
568
IsReference(const BaseClass * cls,const ObjectHeader * ref,const ReferenceCheckPredicateT & pred)569 bool GC::IsReference(const BaseClass *cls, const ObjectHeader *ref, const ReferenceCheckPredicateT &pred)
570 {
571 ASSERT(referenceProcessor_ != nullptr);
572 return referenceProcessor_->IsReference(cls, ref, pred);
573 }
574
EnqueueReferences()575 void GC::EnqueueReferences()
576 {
577 while (true) {
578 panda::mem::Reference *ref = nullptr;
579 {
580 os::memory::LockHolder holder(*clearedReferencesLock_);
581 if (clearedReferences_->empty()) {
582 break;
583 }
584 ref = clearedReferences_->back();
585 clearedReferences_->pop_back();
586 }
587 ASSERT(ref != nullptr);
588 ASSERT(referenceProcessor_ != nullptr);
589 referenceProcessor_->ScheduleForEnqueue(ref);
590 }
591 }
592
IsFullGC() const593 bool GC::IsFullGC() const
594 {
595 // Atomic with relaxed order reason: data race with is_full_gc_ with no synchronization or ordering
596 // constraints imposed on other reads or writes
597 return isFullGc_.load(std::memory_order_relaxed);
598 }
599
SetFullGC(bool value)600 void GC::SetFullGC(bool value)
601 {
602 // Atomic with relaxed order reason: data race with is_full_gc_ with no synchronization or ordering
603 // constraints imposed on other reads or writes
604 isFullGc_.store(value, std::memory_order_relaxed);
605 }
606
NotifyNativeAllocations()607 void GC::NotifyNativeAllocations()
608 {
609 // Atomic with relaxed order reason: data race with native_objects_notified_ with no synchronization or ordering
610 // constraints imposed on other reads or writes
611 nativeObjectsNotified_.fetch_add(NOTIFY_NATIVE_INTERVAL, std::memory_order_relaxed);
612 TriggerGCForNative();
613 }
614
RegisterNativeAllocation(size_t bytes)615 void GC::RegisterNativeAllocation(size_t bytes)
616 {
617 ASSERT_NATIVE_CODE();
618 size_t allocated;
619 do {
620 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
621 // constraints imposed on other reads or writes
622 allocated = nativeBytesRegistered_.load(std::memory_order_relaxed);
623 } while (!nativeBytesRegistered_.compare_exchange_weak(allocated, allocated + bytes));
624 if (allocated > std::numeric_limits<size_t>::max() - bytes) {
625 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
626 // constraints imposed on other reads or writes
627 nativeBytesRegistered_.store(std::numeric_limits<size_t>::max(), std::memory_order_relaxed);
628 }
629 TriggerGCForNative();
630 }
631
RegisterNativeFree(size_t bytes)632 void GC::RegisterNativeFree(size_t bytes)
633 {
634 size_t allocated;
635 size_t newFreedBytes;
636 do {
637 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
638 // constraints imposed on other reads or writes
639 allocated = nativeBytesRegistered_.load(std::memory_order_relaxed);
640 newFreedBytes = std::min(allocated, bytes);
641 } while (!nativeBytesRegistered_.compare_exchange_weak(allocated, allocated - newFreedBytes));
642 }
643
GetNativeBytesFromMallinfoAndRegister() const644 size_t GC::GetNativeBytesFromMallinfoAndRegister() const
645 {
646 size_t mallinfoBytes = panda::os::mem::GetNativeBytesFromMallinfo();
647 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
648 // constraints imposed on other reads or writes
649 size_t allBytes = mallinfoBytes + nativeBytesRegistered_.load(std::memory_order_relaxed);
650 return allBytes;
651 }
652
WaitForGC(GCTask task)653 bool GC::WaitForGC(GCTask task)
654 {
655 // NOTE(maksenov): Notify only about pauses (#4681)
656 Runtime::GetCurrent()->GetNotificationManager()->GarbageCollectorStartEvent();
657 // Atomic with acquire order reason: data race with gc_counter_ with dependecies on reads after the load which
658 // should become visible
659 auto oldCounter = this->gcCounter_.load(std::memory_order_acquire);
660 Timing suspendThreadsTiming;
661 {
662 ScopedTiming t("SuspendThreads", suspendThreadsTiming);
663 this->GetPandaVm()->GetRendezvous()->SafepointBegin();
664 }
665 if (!this->NeedRunGCAfterWaiting(oldCounter, task)) {
666 this->GetPandaVm()->GetRendezvous()->SafepointEnd();
667 return false;
668 }
669
670 // Create a copy of the constant GCTask to be able to change its value
671 this->RunPhases(task);
672
673 if (UNLIKELY(this->IsLogDetailedGcInfoEnabled())) {
674 PrintDetailedLog();
675 }
676
677 this->GetPandaVm()->GetRendezvous()->SafepointEnd();
678 Runtime::GetCurrent()->GetNotificationManager()->GarbageCollectorFinishEvent();
679 this->GetPandaVm()->HandleGCFinished();
680 this->GetPandaVm()->HandleEnqueueReferences();
681 return true;
682 }
683
WaitForGCInManaged(const GCTask & task)684 bool GC::WaitForGCInManaged(const GCTask &task)
685 {
686 Thread *baseThread = Thread::GetCurrent();
687 if (ManagedThread::ThreadIsManagedThread(baseThread)) {
688 ManagedThread *thread = ManagedThread::CastFromThread(baseThread);
689 ASSERT(thread->GetMutatorLock()->HasLock());
690 [[maybe_unused]] bool isDaemon = MTManagedThread::ThreadIsMTManagedThread(baseThread) &&
691 MTManagedThread::CastFromThread(baseThread)->IsDaemon();
692 ASSERT(!isDaemon || thread->GetStatus() == ThreadStatus::RUNNING);
693 vm_->GetMutatorLock()->Unlock();
694 thread->PrintSuspensionStackIfNeeded();
695 WaitForGC(task);
696 vm_->GetMutatorLock()->ReadLock();
697 ASSERT(vm_->GetMutatorLock()->HasLock());
698 this->GetPandaVm()->HandleGCRoutineInMutator();
699 return true;
700 }
701 return false;
702 }
703
StartConcurrentScopeRoutine() const704 void GC::StartConcurrentScopeRoutine() const {}
705
EndConcurrentScopeRoutine() const706 void GC::EndConcurrentScopeRoutine() const {}
707
PrintDetailedLog()708 void GC::PrintDetailedLog()
709 {
710 for (auto &footprint : this->footprintList_) {
711 LOG(INFO, GC) << footprint.first << " : " << footprint.second;
712 }
713 LOG(INFO, GC) << this->GetTiming()->Dump();
714 }
715
ConcurrentScope(GC * gc,bool autoStart)716 ConcurrentScope::ConcurrentScope(GC *gc, bool autoStart)
717 {
718 LOG(DEBUG, GC) << "Start ConcurrentScope";
719 gc_ = gc;
720 if (autoStart) {
721 Start();
722 }
723 }
724
~ConcurrentScope()725 ConcurrentScope::~ConcurrentScope()
726 {
727 LOG(DEBUG, GC) << "Stop ConcurrentScope";
728 if (started_ && gc_->IsConcurrencyAllowed()) {
729 gc_->EndConcurrentScopeRoutine();
730 gc_->GetPandaVm()->GetRendezvous()->SafepointBegin();
731 gc_->SetupCpuAffinityAfterConcurrent();
732 }
733 }
734
Start()735 NO_THREAD_SAFETY_ANALYSIS void ConcurrentScope::Start()
736 {
737 if (!started_ && gc_->IsConcurrencyAllowed()) {
738 gc_->StartConcurrentScopeRoutine();
739 gc_->SetupCpuAffinityBeforeConcurrent();
740 gc_->GetPandaVm()->GetRendezvous()->SafepointEnd();
741 started_ = true;
742 }
743 }
744
WaitForGCOnPygoteFork(const GCTask & task)745 void GC::WaitForGCOnPygoteFork(const GCTask &task)
746 {
747 // do nothing if no pygote space
748 auto pygoteSpaceAllocator = objectAllocator_->GetPygoteSpaceAllocator();
749 if (pygoteSpaceAllocator == nullptr) {
750 return;
751 }
752
753 // do nothing if not at first pygote fork
754 if (pygoteSpaceAllocator->GetState() != PygoteSpaceState::STATE_PYGOTE_INIT) {
755 return;
756 }
757
758 LOG(DEBUG, GC) << "== GC WaitForGCOnPygoteFork Start ==";
759
760 // do we need a lock?
761 // looks all other threads have been stopped before pygote fork
762
763 // 0. indicate that we're rebuilding pygote space
764 pygoteSpaceAllocator->SetState(PygoteSpaceState::STATE_PYGOTE_FORKING);
765
766 // 1. trigger gc
767 WaitForGC(task);
768
769 // 2. move other space to pygote space
770 MoveObjectsToPygoteSpace();
771
772 // 3. indicate that we have done
773 pygoteSpaceAllocator->SetState(PygoteSpaceState::STATE_PYGOTE_FORKED);
774
775 // 4. disable pygote for allocation
776 objectAllocator_->DisablePygoteAlloc();
777
778 LOG(DEBUG, GC) << "== GC WaitForGCOnPygoteFork End ==";
779 }
780
IsOnPygoteFork() const781 bool GC::IsOnPygoteFork() const
782 {
783 auto pygoteSpaceAllocator = objectAllocator_->GetPygoteSpaceAllocator();
784 return pygoteSpaceAllocator != nullptr &&
785 pygoteSpaceAllocator->GetState() == PygoteSpaceState::STATE_PYGOTE_FORKING;
786 }
787
MoveObjectsToPygoteSpace()788 void GC::MoveObjectsToPygoteSpace()
789 {
790 trace::ScopedTrace scopedTrace(__FUNCTION__);
791 LOG(DEBUG, GC) << "MoveObjectsToPygoteSpace: start";
792
793 size_t allSizeMove = 0;
794 size_t movedObjectsNum = 0;
795 size_t bytesInHeapBeforeMove = GetPandaVm()->GetMemStats()->GetFootprintHeap();
796 auto pygoteSpaceAllocator = objectAllocator_->GetPygoteSpaceAllocator();
797 ObjectVisitor moveVisitor([this, &pygoteSpaceAllocator, &movedObjectsNum, &allSizeMove](ObjectHeader *src) -> void {
798 size_t size = GetObjectSize(src);
799 auto dst = reinterpret_cast<ObjectHeader *>(pygoteSpaceAllocator->Alloc(size));
800 ASSERT(dst != nullptr);
801 memcpy_s(dst, size, src, size);
802 allSizeMove += size;
803 movedObjectsNum++;
804 SetForwardAddress(src, dst);
805 LOG_DEBUG_GC << "object MOVED from " << std::hex << src << " to " << dst << ", size = " << std::dec << size;
806 });
807
808 // move all small movable objects to pygote space
809 objectAllocator_->IterateRegularSizeObjects(moveVisitor);
810
811 LOG(DEBUG, GC) << "MoveObjectsToPygoteSpace: move_num = " << movedObjectsNum << ", move_size = " << allSizeMove;
812
813 if (allSizeMove > 0) {
814 GetStats()->AddMemoryValue(allSizeMove, MemoryTypeStats::MOVED_BYTES);
815 GetStats()->AddObjectsValue(movedObjectsNum, ObjectTypeStats::MOVED_OBJECTS);
816 }
817 if (bytesInHeapBeforeMove > 0) {
818 GetStats()->AddCopiedRatioValue(static_cast<double>(allSizeMove) / bytesInHeapBeforeMove);
819 }
820
821 // Update because we moved objects from object_allocator -> pygote space
822 UpdateRefsToMovedObjectsInPygoteSpace();
823 CommonUpdateRefsToMovedObjects();
824
825 // Clear the moved objects in old space
826 objectAllocator_->FreeObjectsMovedToPygoteSpace();
827
828 LOG(DEBUG, GC) << "MoveObjectsToPygoteSpace: finish";
829 }
830
SetForwardAddress(ObjectHeader * src,ObjectHeader * dst)831 void GC::SetForwardAddress(ObjectHeader *src, ObjectHeader *dst)
832 {
833 auto baseCls = src->ClassAddr<BaseClass>();
834 if (baseCls->IsDynamicClass()) {
835 auto cls = static_cast<HClass *>(baseCls);
836 // Note: During moving phase, 'src => dst'. Consider the src is a DynClass,
837 // since 'dst' is not in GC-status the 'manage-object' inside 'dst' won't be updated to
838 // 'dst'. To fix it, we update 'manage-object' here rather than upating phase.
839 if (cls->IsHClass()) {
840 size_t offset = ObjectHeader::ObjectHeaderSize() + HClass::GetManagedObjectOffset();
841 dst->SetFieldObject<false, false, true>(GetPandaVm()->GetAssociatedThread(), offset, dst);
842 }
843 }
844
845 // Set fwd address in src
846 bool updateRes = false;
847 do {
848 MarkWord markWord = src->AtomicGetMark();
849 MarkWord fwdMarkWord =
850 markWord.DecodeFromForwardingAddress(static_cast<MarkWord::MarkWordSize>(ToUintPtr(dst)));
851 updateRes = src->AtomicSetMark<false>(markWord, fwdMarkWord);
852 } while (!updateRes);
853 }
854
UpdateRefsInVRegs(ManagedThread * thread)855 void GC::UpdateRefsInVRegs(ManagedThread *thread)
856 {
857 LOG_DEBUG_GC << "Update frames for thread: " << thread->GetId();
858 for (auto pframe = StackWalker::Create(thread); pframe.HasFrame(); pframe.NextFrame()) {
859 LOG_DEBUG_GC << "Frame for method " << pframe.GetMethod()->GetFullName();
860 auto iterator = [&pframe, this](auto ®Info, auto &vreg) {
861 ObjectHeader *objectHeader = vreg.GetReference();
862 if (objectHeader == nullptr) {
863 return true;
864 }
865 MarkWord markWord = objectHeader->AtomicGetMark();
866 if (markWord.GetState() == MarkWord::ObjectState::STATE_GC) {
867 MarkWord::MarkWordSize addr = markWord.GetForwardingAddress();
868 LOG_DEBUG_GC << "Update vreg, vreg old val = " << std::hex << objectHeader << ", new val = 0x" << addr;
869 LOG_IF(regInfo.IsAccumulator(), DEBUG, GC) << "^ acc reg";
870 if (!pframe.IsCFrame() && regInfo.IsAccumulator()) {
871 LOG_DEBUG_GC << "^ acc updated";
872 vreg.SetReference(reinterpret_cast<ObjectHeader *>(addr));
873 } else {
874 pframe.template SetVRegValue<std::is_same_v<decltype(vreg), interpreter::DynamicVRegisterRef &>>(
875 regInfo, reinterpret_cast<ObjectHeader *>(addr));
876 }
877 }
878 return true;
879 };
880 pframe.IterateObjectsWithInfo(iterator);
881 }
882 }
883
PopObjectFromStack(GCMarkingStackType * objectsStack)884 const ObjectHeader *GC::PopObjectFromStack(GCMarkingStackType *objectsStack)
885 {
886 auto *object = objectsStack->PopFromStack();
887 ASSERT(object != nullptr);
888 return object;
889 }
890
IsGenerational() const891 bool GC::IsGenerational() const
892 {
893 return IsGenerationalGCType(gcType_);
894 }
895
AddListener(GCListener * listener)896 void GC::GCListenerManager::AddListener(GCListener *listener)
897 {
898 os::memory::LockHolder lh(listenerLock_);
899 newListeners_.insert(listener);
900 }
901
RemoveListener(GCListener * listener)902 void GC::GCListenerManager::RemoveListener(GCListener *listener)
903 {
904 os::memory::LockHolder lh(listenerLock_);
905 listenersForRemove_.insert(listener);
906 }
907
NormalizeListenersOnStartGC()908 void GC::GCListenerManager::NormalizeListenersOnStartGC()
909 {
910 os::memory::LockHolder lh(listenerLock_);
911 for (auto *listenerForRemove : listenersForRemove_) {
912 if (newListeners_.find(listenerForRemove) != newListeners_.end()) {
913 newListeners_.erase(listenerForRemove);
914 }
915 auto it = currentListeners_.find(listenerForRemove);
916 if (it != currentListeners_.end()) {
917 LOG(DEBUG, GC) << "Remove listener for GC: " << listenerForRemove;
918 currentListeners_.erase(it);
919 }
920 }
921 listenersForRemove_.clear();
922 for (auto *newListener : newListeners_) {
923 LOG(DEBUG, GC) << "Add new listener for GC: " << newListener;
924 currentListeners_.insert(newListener);
925 }
926 newListeners_.clear();
927 }
928
FireGCStarted(const GCTask & task,size_t bytesInHeapBeforeGc)929 void GC::FireGCStarted(const GCTask &task, size_t bytesInHeapBeforeGc)
930 {
931 gcListenerManager_->NormalizeListenersOnStartGC();
932 gcListenerManager_->IterateOverListeners(
933 [&](GCListener *listener) { listener->GCStarted(task, bytesInHeapBeforeGc); });
934 }
935
FireGCFinished(const GCTask & task,size_t bytesInHeapBeforeGc,size_t bytesInHeapAfterGc)936 void GC::FireGCFinished(const GCTask &task, size_t bytesInHeapBeforeGc, size_t bytesInHeapAfterGc)
937 {
938 gcListenerManager_->IterateOverListeners(
939 [&](GCListener *listener) { listener->GCFinished(task, bytesInHeapBeforeGc, bytesInHeapAfterGc); });
940 }
941
FireGCPhaseStarted(GCPhase phase)942 void GC::FireGCPhaseStarted(GCPhase phase)
943 {
944 gcListenerManager_->IterateOverListeners([phase](GCListener *listener) { listener->GCPhaseStarted(phase); });
945 }
946
FireGCPhaseFinished(GCPhase phase)947 void GC::FireGCPhaseFinished(GCPhase phase)
948 {
949 gcListenerManager_->IterateOverListeners([phase](GCListener *listener) { listener->GCPhaseFinished(phase); });
950 }
951
OnWaitForIdleFail()952 void GC::OnWaitForIdleFail() {}
953
954 TEMPLATE_GC_CREATE_GC();
955
956 } // namespace panda::mem
957