1 /**
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16
17 #include "runtime/mem/gc/gc.h"
18
19 #include <memory>
20
21 #include "libpandabase/os/mem.h"
22 #include "libpandabase/os/thread.h"
23 #include "libpandabase/utils/time.h"
24 #include "runtime/assert_gc_scope.h"
25 #include "runtime/include/class.h"
26 #include "runtime/include/coretypes/dyn_objects.h"
27 #include "runtime/include/locks.h"
28 #include "runtime/include/runtime.h"
29 #include "runtime/include/runtime_notification.h"
30 #include "runtime/include/stack_walker-inl.h"
31 #include "runtime/mem/gc/epsilon/epsilon.h"
32 #include "runtime/mem/gc/gc_root-inl.h"
33 #include "runtime/mem/gc/gc_queue.h"
34 #include "runtime/mem/gc/g1/g1-gc.h"
35 #include "runtime/mem/gc/gen-gc/gen-gc.h"
36 #include "runtime/mem/gc/stw-gc/stw-gc.h"
37 #include "runtime/mem/gc/gc_workers_thread_pool.h"
38 #include "runtime/mem/pygote_space_allocator-inl.h"
39 #include "runtime/mem/heap_manager.h"
40 #include "runtime/mem/gc/reference-processor/reference_processor.h"
41 #include "runtime/mem/gc/gc-hung/gc_hung.h"
42 #include "runtime/include/panda_vm.h"
43 #include "runtime/include/object_accessor-inl.h"
44 #include "runtime/include/coretypes/class.h"
45 #include "runtime/thread_manager.h"
46
47 namespace panda::mem {
48 using TaggedValue = coretypes::TaggedValue;
49 using TaggedType = coretypes::TaggedType;
50 using DynClass = coretypes::DynClass;
51
GC(ObjectAllocatorBase * object_allocator,const GCSettings & settings)52 GC::GC(ObjectAllocatorBase *object_allocator, const GCSettings &settings)
53 : gc_settings_(settings),
54 object_allocator_(object_allocator),
55 internal_allocator_(InternalAllocator<>::GetInternalAllocatorFromRuntime())
56 {
57 }
58
~GC()59 GC::~GC()
60 {
61 InternalAllocatorPtr allocator = GetInternalAllocator();
62 if (gc_queue_ != nullptr) {
63 allocator->Delete(gc_queue_);
64 }
65 if (gc_listeners_ptr_ != nullptr) {
66 allocator->Delete(gc_listeners_ptr_);
67 }
68 if (gc_barrier_set_ != nullptr) {
69 allocator->Delete(gc_barrier_set_);
70 }
71 if (cleared_references_ != nullptr) {
72 allocator->Delete(cleared_references_);
73 }
74 if (cleared_references_lock_ != nullptr) {
75 allocator->Delete(cleared_references_lock_);
76 }
77 if (workers_pool_ != nullptr) {
78 allocator->Delete(workers_pool_);
79 }
80 }
81
GetLogPrefix() const82 Logger::Buffer GC::GetLogPrefix() const
83 {
84 const char *phase = GCScopedPhase::GetPhaseAbbr(GetGCPhase());
85 // Atomic with acquire order reason: data race with gc_counter_
86 size_t counter = gc_counter_.load(std::memory_order_acquire);
87
88 Logger::Buffer buffer;
89 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
90 buffer.printf("[%zu, %s]: ", counter, phase);
91
92 return buffer;
93 }
94
GetType()95 GCType GC::GetType()
96 {
97 return gc_type_;
98 }
99
SetPandaVM(PandaVM * vm)100 void GC::SetPandaVM(PandaVM *vm)
101 {
102 vm_ = vm;
103 reference_processor_ = vm->GetReferenceProcessor();
104 }
105
GetNativeGcTriggerType()106 NativeGcTriggerType GC::GetNativeGcTriggerType()
107 {
108 return gc_settings_.GetNativeGcTriggerType();
109 }
110
SimpleNativeAllocationGcWatermark()111 size_t GC::SimpleNativeAllocationGcWatermark()
112 {
113 return GetPandaVm()->GetOptions().GetMaxFree();
114 }
115
WaitForIdleGC()116 NO_THREAD_SAFETY_ANALYSIS void GC::WaitForIdleGC()
117 {
118 while (!CASGCPhase(GCPhase::GC_PHASE_IDLE, GCPhase::GC_PHASE_RUNNING)) {
119 GetPandaVm()->GetRendezvous()->SafepointEnd();
120 // Interrupt the running GC if possible
121 OnWaitForIdleFail();
122 // TODO(dtrubenkov): resolve it more properly
123 constexpr uint64_t WAIT_FINISHED = 10;
124 // Use NativeSleep for all threads, as this thread shouldn't hold Mutator lock here
125 os::thread::NativeSleep(WAIT_FINISHED);
126 GetPandaVm()->GetRendezvous()->SafepointBegin();
127 }
128 }
129
TriggerGCForNative()130 inline void GC::TriggerGCForNative()
131 {
132 auto native_gc_trigger_type = GetNativeGcTriggerType();
133 ASSERT_PRINT((native_gc_trigger_type == NativeGcTriggerType::NO_NATIVE_GC_TRIGGER) ||
134 (native_gc_trigger_type == NativeGcTriggerType::SIMPLE_STRATEGY),
135 "Unknown Native GC Trigger type");
136 switch (native_gc_trigger_type) {
137 case NativeGcTriggerType::NO_NATIVE_GC_TRIGGER:
138 break;
139 case NativeGcTriggerType::SIMPLE_STRATEGY:
140 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or
141 // ordering constraints imposed on other reads or writes
142 if (native_bytes_registered_.load(std::memory_order_relaxed) > SimpleNativeAllocationGcWatermark()) {
143 auto task = MakePandaUnique<GCTask>(GCTaskCause::NATIVE_ALLOC_CAUSE, time::GetCurrentTimeInNanos());
144 AddGCTask(false, std::move(task), true);
145 ManagedThread::GetCurrent()->SafepointPoll();
146 }
147 break;
148 default:
149 LOG(FATAL, GC) << "Unknown Native GC Trigger type";
150 break;
151 }
152 }
153
Initialize(PandaVM * vm)154 void GC::Initialize(PandaVM *vm)
155 {
156 trace::ScopedTrace scoped_trace(__PRETTY_FUNCTION__);
157 // GC saved the PandaVM instance, so we get allocator from the PandaVM.
158 auto allocator = GetInternalAllocator();
159 gc_listeners_ptr_ = allocator->template New<PandaVector<GCListener *>>(allocator->Adapter());
160 cleared_references_lock_ = allocator->New<os::memory::Mutex>();
161 os::memory::LockHolder holder(*cleared_references_lock_);
162 cleared_references_ = allocator->New<PandaVector<panda::mem::Reference *>>(allocator->Adapter());
163 gc_queue_ = allocator->New<GCQueueWithTime>(this);
164 this->SetPandaVM(vm);
165 InitializeImpl();
166 }
167
StartGC()168 void GC::StartGC()
169 {
170 CreateWorker();
171 }
172
StopGC()173 void GC::StopGC()
174 {
175 JoinWorker();
176 ASSERT(gc_queue_ != nullptr);
177 gc_queue_->Finalize();
178 if (workers_pool_ != nullptr) {
179 InternalAllocatorPtr allocator = GetInternalAllocator();
180 allocator->Delete(workers_pool_);
181 workers_pool_ = nullptr;
182 }
183 }
184
185 // NOLINTNEXTLINE(performance-unnecessary-value-param)
RunPhases(GCTask & task)186 void GC::RunPhases(GCTask &task)
187 {
188 DCHECK_ALLOW_GARBAGE_COLLECTION;
189 trace::ScopedTrace scoped_trace(__FUNCTION__);
190 // Atomic with acquire order reason: data race with gc_counter_ with dependecies on reads after the load which
191 // should become visible
192 auto old_counter = gc_counter_.load(std::memory_order_acquire);
193 WaitForIdleGC();
194 // Atomic with acquire order reason: data race with gc_counter_ with dependecies on reads after the load which
195 // should become visible
196 auto new_counter = gc_counter_.load(std::memory_order_acquire);
197 if (new_counter > old_counter) {
198 SetGCPhase(GCPhase::GC_PHASE_IDLE);
199 return;
200 }
201 // Atomic with release order reason: data race with last_cause_ with dependecies on writes before the store which
202 // should become visible acquire
203 last_cause_.store(task.reason_, std::memory_order_release);
204 if (gc_settings_.PreGCHeapVerification()) {
205 trace::ScopedTrace pre_heap_verifier_trace("PreGCHeapVeriFier");
206 size_t fail_count = VerifyHeap();
207 if (gc_settings_.FailOnHeapVerification() && fail_count > 0) {
208 LOG(FATAL, GC) << "Heap corrupted before GC, HeapVerifier found " << fail_count << " corruptions";
209 }
210 }
211 // Atomic with acq_rel order reason: data race with gc_counter_ with dependecies on reads after the load and on
212 // writes before the store
213 gc_counter_.fetch_add(1, std::memory_order_acq_rel);
214 if (gc_settings_.IsDumpHeap()) {
215 PandaOStringStream os;
216 os << "Heap dump before GC" << std::endl;
217 GetPandaVm()->GetHeapManager()->DumpHeap(&os);
218 std::cerr << os.str() << std::endl;
219 }
220 size_t bytes_in_heap_before_gc = GetPandaVm()->GetMemStats()->GetFootprintHeap();
221 LOG_DEBUG_GC << "Bytes in heap before GC " << std::dec << bytes_in_heap_before_gc;
222 {
223 GCScopedStats scoped_stats(GetPandaVm()->GetGCStats(), gc_type_ == GCType::STW_GC ? GetStats() : nullptr);
224 ScopedGcHung scoped_hung(&task);
225 for (auto listener : *gc_listeners_ptr_) {
226 if (listener != nullptr) {
227 listener->GCStarted(bytes_in_heap_before_gc);
228 }
229 }
230
231 PreRunPhasesImpl();
232 // NOLINTNEXTLINE(performance-unnecessary-value-param)
233 RunPhasesImpl(task);
234 // Clear Internal allocator unused pools (must do it on pause to avoid race conditions):
235 // - Clear global part:
236 InternalAllocator<>::GetInternalAllocatorFromRuntime()->VisitAndRemoveFreePools(
237 [](void *mem, [[maybe_unused]] size_t size) { PoolManager::GetMmapMemPool()->FreePool(mem, size); });
238 // - Clear local part:
239 ClearLocalInternalAllocatorPools();
240
241 size_t bytes_in_heap_after_gc = GetPandaVm()->GetMemStats()->GetFootprintHeap();
242 // There is case than bytes_in_heap_after_gc > 0 and bytes_in_heap_before_gc == 0.
243 // Because TLABs are registered during GC
244 if (bytes_in_heap_after_gc > 0 && bytes_in_heap_before_gc > 0) {
245 GetStats()->AddReclaimRatioValue(1 - static_cast<double>(bytes_in_heap_after_gc) / bytes_in_heap_before_gc);
246 }
247 LOG_DEBUG_GC << "Bytes in heap after GC " << std::dec << bytes_in_heap_after_gc;
248 for (auto listener : *gc_listeners_ptr_) {
249 if (listener != nullptr) {
250 listener->GCFinished(task, bytes_in_heap_before_gc, bytes_in_heap_after_gc);
251 }
252 }
253 }
254 ASSERT(task.collection_type_ != GCCollectionType::NONE);
255 LOG(INFO, GC) << "[" << gc_counter_ << "] [" << task.collection_type_ << " (" << task.reason_ << ")] "
256 << GetPandaVm()->GetGCStats()->GetStatistics();
257 if (gc_settings_.IsDumpHeap()) {
258 PandaOStringStream os;
259 os << "Heap dump after GC" << std::endl;
260 GetPandaVm()->GetHeapManager()->DumpHeap(&os);
261 std::cerr << os.str() << std::endl;
262 }
263
264 if (gc_settings_.PostGCHeapVerification()) {
265 trace::ScopedTrace post_heap_verifier_trace("PostGCHeapVeriFier");
266 size_t fail_count = VerifyHeap();
267 if (gc_settings_.FailOnHeapVerification() && fail_count > 0) {
268 LOG(FATAL, GC) << "Heap corrupted after GC, HeapVerifier found " << fail_count << " corruptions";
269 }
270 }
271
272 SetGCPhase(GCPhase::GC_PHASE_IDLE);
273 }
274
275 template <class LanguageConfig>
CreateGC(GCType gc_type,ObjectAllocatorBase * object_allocator,const GCSettings & settings)276 GC *CreateGC(GCType gc_type, ObjectAllocatorBase *object_allocator, const GCSettings &settings)
277 {
278 GC *ret = nullptr;
279 ASSERT_PRINT((gc_type == GCType::EPSILON_GC) || (gc_type == GCType::STW_GC) || (gc_type == GCType::GEN_GC) ||
280 (gc_type == GCType::G1_GC),
281 "Unknown GC type");
282 InternalAllocatorPtr allocator {InternalAllocator<>::GetInternalAllocatorFromRuntime()};
283
284 switch (gc_type) {
285 case GCType::EPSILON_GC:
286 ret = allocator->New<EpsilonGC<LanguageConfig>>(object_allocator, settings);
287 break;
288 case GCType::STW_GC:
289 ret = allocator->New<StwGC<LanguageConfig>>(object_allocator, settings);
290 break;
291 case GCType::GEN_GC:
292 ret = allocator->New<GenGC<LanguageConfig>>(object_allocator, settings);
293 break;
294 case GCType::G1_GC:
295 ret = allocator->New<G1GC<LanguageConfig>>(object_allocator, settings);
296 break;
297 default:
298 LOG(FATAL, GC) << "Unknown GC type";
299 break;
300 }
301 return ret;
302 }
303
MarkObjectIfNotMarked(ObjectHeader * object_header)304 bool GC::MarkObjectIfNotMarked(ObjectHeader *object_header)
305 {
306 ASSERT(object_header != nullptr);
307 if (IsMarked(object_header)) {
308 return false;
309 }
310 MarkObject(object_header);
311 return true;
312 }
313
ProcessReference(GCMarkingStackType * objects_stack,const BaseClass * cls,const ObjectHeader * ref,const ReferenceProcessPredicateT & pred)314 void GC::ProcessReference(GCMarkingStackType *objects_stack, const BaseClass *cls, const ObjectHeader *ref,
315 const ReferenceProcessPredicateT &pred)
316 {
317 ASSERT(reference_processor_ != nullptr);
318 reference_processor_->HandleReference(this, objects_stack, cls, ref, pred);
319 }
320
AddReference(ObjectHeader * from_obj,ObjectHeader * object)321 void GC::AddReference(ObjectHeader *from_obj, ObjectHeader *object)
322 {
323 ASSERT(IsMarked(object));
324 GCMarkingStackType references(this);
325 // TODO(alovkov): support stack with workers here & put all refs in stack and only then process altogether for once
326 ASSERT(!references.IsWorkersTaskSupported());
327 references.PushToStack(from_obj, object);
328 MarkReferences(&references, phase_);
329 if (gc_type_ != GCType::EPSILON_GC) {
330 ASSERT(references.Empty());
331 }
332 }
333
334 // NOLINTNEXTLINE(performance-unnecessary-value-param)
ProcessReferences(GCPhase gc_phase,const GCTask & task,const ReferenceClearPredicateT & pred)335 void GC::ProcessReferences(GCPhase gc_phase, const GCTask &task, const ReferenceClearPredicateT &pred)
336 {
337 trace::ScopedTrace scoped_trace(__FUNCTION__);
338 LOG(DEBUG, REF_PROC) << "Start processing cleared references";
339 ASSERT(reference_processor_ != nullptr);
340 bool clear_soft_references = task.reason_ == GCTaskCause::OOM_CAUSE || task.reason_ == GCTaskCause::EXPLICIT_CAUSE;
341 reference_processor_->ProcessReferences(false, clear_soft_references, gc_phase, pred);
342 Reference *processed_ref = reference_processor_->CollectClearedReferences();
343 if (processed_ref != nullptr) {
344 os::memory::LockHolder holder(*cleared_references_lock_);
345 // TODO(alovkov): ged rid of cleared_references_ and just enqueue refs here?
346 cleared_references_->push_back(processed_ref);
347 }
348 }
349
GCWorkerEntry(GC * gc,PandaVM * vm)350 void GC::GCWorkerEntry(GC *gc, PandaVM *vm)
351 {
352 // We need to set VM to current_thread, since GC can call ObjectAccessor::GetBarrierSet() methods
353 Thread gc_thread(vm, Thread::ThreadType::THREAD_TYPE_GC);
354 ScopedCurrentThread sct(&gc_thread);
355 while (true) {
356 auto task = gc->gc_queue_->GetTask();
357 if (!gc->IsGCRunning()) {
358 LOG(DEBUG, GC) << "Stopping GC thread";
359 if (task != nullptr) {
360 task->Release(Runtime::GetCurrent()->GetInternalAllocator());
361 }
362 break;
363 }
364 if (task == nullptr) {
365 continue;
366 }
367 if (task->reason_ == GCTaskCause::INVALID_CAUSE) {
368 task->Release(Runtime::GetCurrent()->GetInternalAllocator());
369 continue;
370 }
371 LOG(DEBUG, GC) << "Running GC task, reason " << task->reason_;
372 task->Run(*gc);
373 task->Release(Runtime::GetCurrent()->GetInternalAllocator());
374 }
375 }
376
JoinWorker()377 void GC::JoinWorker()
378 {
379 // Atomic with seq_cst order reason: data race with gc_running_ with requirement for sequentially consistent order
380 // where threads observe all modifications in the same order
381 gc_running_.store(false, std::memory_order_seq_cst);
382 if (!gc_settings_.RunGCInPlace()) {
383 ASSERT(worker_ != nullptr);
384 }
385 if (worker_ != nullptr && !gc_settings_.RunGCInPlace()) {
386 ASSERT(gc_queue_ != nullptr);
387 gc_queue_->Signal();
388 worker_->join();
389 InternalAllocatorPtr allocator = GetInternalAllocator();
390 allocator->Delete(worker_);
391 worker_ = nullptr;
392 }
393 }
394
CreateWorker()395 void GC::CreateWorker()
396 {
397 // Atomic with seq_cst order reason: data race with gc_running_ with requirement for sequentially consistent order
398 // where threads observe all modifications in the same order
399 gc_running_.store(true, std::memory_order_seq_cst);
400 ASSERT(worker_ == nullptr);
401 if (!gc_settings_.RunGCInPlace()) {
402 ASSERT(gc_queue_ != nullptr);
403 InternalAllocatorPtr allocator = GetInternalAllocator();
404 worker_ = allocator->New<std::thread>(GC::GCWorkerEntry, this, this->GetPandaVm());
405 if (worker_ == nullptr) {
406 LOG(FATAL, COMPILER) << "Cannot create a GC thread";
407 }
408 int res = os::thread::SetThreadName(worker_->native_handle(), "GCThread");
409 if (res != 0) {
410 LOG(ERROR, RUNTIME) << "Failed to set a name for the gc thread";
411 }
412 }
413 }
414
DisableWorkerThreads()415 void GC::DisableWorkerThreads()
416 {
417 gc_settings_.SetGCWorkersCount(0);
418 gc_settings_.SetParallelMarkingEnabled(false);
419 gc_settings_.SetParallelCompactingEnabled(false);
420 }
421
EnableWorkerThreads()422 void GC::EnableWorkerThreads()
423 {
424 const RuntimeOptions &options = Runtime::GetOptions();
425 gc_settings_.SetGCWorkersCount(options.GetGcWorkersCount());
426 gc_settings_.SetParallelMarkingEnabled(options.IsGcParallelMarkingEnabled() && (options.GetGcWorkersCount() != 0));
427 gc_settings_.SetParallelCompactingEnabled(options.IsGcParallelCompactingEnabled() &&
428 (options.GetGcWorkersCount() != 0));
429 }
430
431 class GC::PostForkGCTask : public GCTask {
432 public:
PostForkGCTask(GCTaskCause reason,uint64_t target_time)433 PostForkGCTask(GCTaskCause reason, uint64_t target_time) : GCTask(reason, target_time) {}
434
Run(mem::GC & gc)435 void Run(mem::GC &gc) override
436 {
437 LOG(DEBUG, GC) << "Runing PostForkGCTask";
438 gc.GetPandaVm()->GetGCTrigger()->RestoreMinTargetFootprint();
439 gc.PostForkCallback();
440 GCTask::Run(gc);
441 }
442
443 ~PostForkGCTask() override = default;
444
445 NO_COPY_SEMANTIC(PostForkGCTask);
446 NO_MOVE_SEMANTIC(PostForkGCTask);
447 };
448
PreStartup()449 void GC::PreStartup()
450 {
451 // Add a delay GCTask.
452 if ((!Runtime::GetCurrent()->IsZygote()) && (!gc_settings_.RunGCInPlace())) {
453 // divide 2 to temporarily set target footprint to a high value to disable GC during App startup.
454 GetPandaVm()->GetGCTrigger()->SetMinTargetFootprint(Runtime::GetOptions().GetHeapSizeLimit() / 2);
455 PreStartupImp();
456 constexpr uint64_t DISABLE_GC_DURATION_NS = 2000 * 1000 * 1000;
457 auto task = MakePandaUnique<PostForkGCTask>(GCTaskCause::STARTUP_COMPLETE_CAUSE,
458 time::GetCurrentTimeInNanos() + DISABLE_GC_DURATION_NS);
459 AddGCTask(true, std::move(task), false);
460 LOG(DEBUG, GC) << "Add PostForkGCTask";
461 }
462 }
463
464 // NOLINTNEXTLINE(performance-unnecessary-value-param)
AddGCTask(bool is_managed,PandaUniquePtr<GCTask> task,bool triggered_by_threshold)465 void GC::AddGCTask(bool is_managed, PandaUniquePtr<GCTask> task, bool triggered_by_threshold)
466 {
467 if (gc_settings_.RunGCInPlace()) {
468 auto *gc_task = task.release();
469 if (IsGCRunning()) {
470 if (is_managed) {
471 WaitForGCInManaged(*gc_task);
472 } else {
473 WaitForGC(*gc_task);
474 }
475 }
476 gc_task->Release(Runtime::GetCurrent()->GetInternalAllocator());
477 } else {
478 if (triggered_by_threshold) {
479 bool expect = true;
480 if (can_add_gc_task_.compare_exchange_strong(expect, false, std::memory_order_seq_cst)) {
481 gc_queue_->AddTask(task.release());
482 }
483 } else {
484 gc_queue_->AddTask(task.release());
485 }
486 }
487 }
488
IsReference(const BaseClass * cls,const ObjectHeader * ref,const ReferenceCheckPredicateT & pred)489 bool GC::IsReference(const BaseClass *cls, const ObjectHeader *ref, const ReferenceCheckPredicateT &pred)
490 {
491 ASSERT(reference_processor_ != nullptr);
492 return reference_processor_->IsReference(cls, ref, pred);
493 }
494
EnqueueReferences()495 void GC::EnqueueReferences()
496 {
497 while (true) {
498 panda::mem::Reference *ref = nullptr;
499 {
500 os::memory::LockHolder holder(*cleared_references_lock_);
501 if (cleared_references_->empty()) {
502 break;
503 }
504 ref = cleared_references_->back();
505 cleared_references_->pop_back();
506 }
507 ASSERT(ref != nullptr);
508 ASSERT(reference_processor_ != nullptr);
509 reference_processor_->ScheduleForEnqueue(ref);
510 }
511 }
512
IsFullGC() const513 bool GC::IsFullGC() const
514 {
515 // Atomic with relaxed order reason: data race with is_full_gc_ with no synchronization or ordering
516 // constraints imposed on other reads or writes
517 return is_full_gc_.load(std::memory_order_relaxed);
518 }
519
SetFullGC(bool value)520 void GC::SetFullGC(bool value)
521 {
522 // Atomic with relaxed order reason: data race with is_full_gc_ with no synchronization or ordering
523 // constraints imposed on other reads or writes
524 is_full_gc_.store(value, std::memory_order_relaxed);
525 }
526
NotifyNativeAllocations()527 void GC::NotifyNativeAllocations()
528 {
529 // Atomic with relaxed order reason: data race with native_objects_notified_ with no synchronization or ordering
530 // constraints imposed on other reads or writes
531 native_objects_notified_.fetch_add(NOTIFY_NATIVE_INTERVAL, std::memory_order_relaxed);
532 TriggerGCForNative();
533 }
534
RegisterNativeAllocation(size_t bytes)535 void GC::RegisterNativeAllocation(size_t bytes)
536 {
537 size_t allocated;
538 do {
539 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
540 // constraints imposed on other reads or writes
541 allocated = native_bytes_registered_.load(std::memory_order_relaxed);
542 } while (!native_bytes_registered_.compare_exchange_weak(allocated, allocated + bytes));
543 if (allocated > std::numeric_limits<size_t>::max() - bytes) {
544 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
545 // constraints imposed on other reads or writes
546 native_bytes_registered_.store(std::numeric_limits<size_t>::max(), std::memory_order_relaxed);
547 }
548 TriggerGCForNative();
549 }
550
RegisterNativeFree(size_t bytes)551 void GC::RegisterNativeFree(size_t bytes)
552 {
553 size_t allocated;
554 size_t new_freed_bytes;
555 do {
556 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
557 // constraints imposed on other reads or writes
558 allocated = native_bytes_registered_.load(std::memory_order_relaxed);
559 new_freed_bytes = std::min(allocated, bytes);
560 } while (!native_bytes_registered_.compare_exchange_weak(allocated, allocated - new_freed_bytes));
561 }
562
GetNativeBytesFromMallinfoAndRegister() const563 size_t GC::GetNativeBytesFromMallinfoAndRegister() const
564 {
565 size_t mallinfo_bytes = panda::os::mem::GetNativeBytesFromMallinfo();
566 // Atomic with relaxed order reason: data race with native_bytes_registered_ with no synchronization or ordering
567 // constraints imposed on other reads or writes
568 size_t all_bytes = mallinfo_bytes + native_bytes_registered_.load(std::memory_order_relaxed);
569 return all_bytes;
570 }
571
WaitForGCInManaged(const GCTask & task)572 void GC::WaitForGCInManaged(const GCTask &task)
573 {
574 Thread *base_thread = Thread::GetCurrent();
575 if (ManagedThread::ThreadIsManagedThread(base_thread)) {
576 ManagedThread *thread = ManagedThread::CastFromThread(base_thread);
577 ASSERT(Locks::mutator_lock->HasLock());
578 [[maybe_unused]] bool is_daemon = MTManagedThread::ThreadIsMTManagedThread(base_thread) &&
579 MTManagedThread::CastFromThread(base_thread)->IsDaemon();
580 ASSERT(!is_daemon || thread->GetStatus() == ThreadStatus::RUNNING);
581 Locks::mutator_lock->Unlock();
582 thread->PrintSuspensionStackIfNeeded();
583 WaitForGC(task);
584 Locks::mutator_lock->ReadLock();
585 ASSERT(Locks::mutator_lock->HasLock());
586 }
587 }
588
ConcurrentScope(GC * gc,bool auto_start)589 ConcurrentScope::ConcurrentScope(GC *gc, bool auto_start)
590 {
591 LOG(DEBUG, GC) << "Start ConcurrentScope";
592 gc_ = gc;
593 if (auto_start) {
594 Start();
595 }
596 }
597
~ConcurrentScope()598 ConcurrentScope::~ConcurrentScope()
599 {
600 LOG(DEBUG, GC) << "Stop ConcurrentScope";
601 if (started_ && gc_->IsConcurrencyAllowed()) {
602 gc_->GetPandaVm()->GetRendezvous()->SafepointBegin();
603 gc_->GetPandaVm()->GetMemStats()->RecordGCPauseStart();
604 }
605 }
606
Start()607 NO_THREAD_SAFETY_ANALYSIS void ConcurrentScope::Start()
608 {
609 if (!started_ && gc_->IsConcurrencyAllowed()) {
610 gc_->GetPandaVm()->GetRendezvous()->SafepointEnd();
611 gc_->GetPandaVm()->GetMemStats()->RecordGCPauseEnd();
612 started_ = true;
613 }
614 }
615
WaitForGCOnPygoteFork(const GCTask & task)616 void GC::WaitForGCOnPygoteFork(const GCTask &task)
617 {
618 // do nothing if no pygote space
619 auto pygote_space_allocator = object_allocator_->GetPygoteSpaceAllocator();
620 if (pygote_space_allocator == nullptr) {
621 return;
622 }
623
624 // do nothing if not at first pygote fork
625 if (pygote_space_allocator->GetState() != PygoteSpaceState::STATE_PYGOTE_INIT) {
626 return;
627 }
628
629 LOG(DEBUG, GC) << "== GC WaitForGCOnPygoteFork Start ==";
630
631 // do we need a lock?
632 // looks all other threads have been stopped before pygote fork
633
634 // 0. indicate that we're rebuilding pygote space
635 pygote_space_allocator->SetState(PygoteSpaceState::STATE_PYGOTE_FORKING);
636
637 // 1. trigger gc
638 WaitForGC(task);
639
640 // 2. move other space to pygote space
641 MoveObjectsToPygoteSpace();
642
643 // 3. indicate that we have done
644 pygote_space_allocator->SetState(PygoteSpaceState::STATE_PYGOTE_FORKED);
645
646 // 4. disable pygote for allocation
647 object_allocator_->DisablePygoteAlloc();
648
649 LOG(DEBUG, GC) << "== GC WaitForGCOnPygoteFork End ==";
650 }
651
IsOnPygoteFork() const652 bool GC::IsOnPygoteFork() const
653 {
654 auto pygote_space_allocator = object_allocator_->GetPygoteSpaceAllocator();
655 return pygote_space_allocator != nullptr &&
656 pygote_space_allocator->GetState() == PygoteSpaceState::STATE_PYGOTE_FORKING;
657 }
658
MoveObjectsToPygoteSpace()659 void GC::MoveObjectsToPygoteSpace()
660 {
661 trace::ScopedTrace scoped_trace(__FUNCTION__);
662 LOG(DEBUG, GC) << "MoveObjectsToPygoteSpace: start";
663
664 size_t all_size_move = 0;
665 size_t moved_objects_num = 0;
666 size_t bytes_in_heap_before_move = GetPandaVm()->GetMemStats()->GetFootprintHeap();
667 auto pygote_space_allocator = object_allocator_->GetPygoteSpaceAllocator();
668 ObjectVisitor move_visitor(
669 [this, &pygote_space_allocator, &moved_objects_num, &all_size_move](ObjectHeader *src) -> void {
670 size_t size = GetObjectSize(src);
671 auto dst = reinterpret_cast<ObjectHeader *>(pygote_space_allocator->Alloc(size));
672 ASSERT(dst != nullptr);
673 memcpy_s(dst, size, src, size);
674 all_size_move += size;
675 moved_objects_num++;
676 SetForwardAddress(src, dst);
677 LOG_DEBUG_GC << "object MOVED from " << std::hex << src << " to " << dst << ", size = " << std::dec << size;
678 });
679
680 // move all small movable objects to pygote space
681 object_allocator_->IterateRegularSizeObjects(move_visitor);
682
683 LOG(DEBUG, GC) << "MoveObjectsToPygoteSpace: move_num = " << moved_objects_num << ", move_size = " << all_size_move;
684
685 if (all_size_move > 0) {
686 GetStats()->AddMemoryValue(all_size_move, MemoryTypeStats::MOVED_BYTES);
687 GetStats()->AddObjectsValue(moved_objects_num, ObjectTypeStats::MOVED_OBJECTS);
688 }
689 if (bytes_in_heap_before_move > 0) {
690 GetStats()->AddCopiedRatioValue(static_cast<double>(all_size_move) / bytes_in_heap_before_move);
691 }
692
693 // Update because we moved objects from object_allocator -> pygote space
694 UpdateRefsToMovedObjectsInPygoteSpace();
695 CommonUpdateRefsToMovedObjects();
696
697 // Clear the moved objects in old space
698 object_allocator_->FreeObjectsMovedToPygoteSpace();
699
700 LOG(DEBUG, GC) << "MoveObjectsToPygoteSpace: finish";
701 }
702
SetForwardAddress(ObjectHeader * src,ObjectHeader * dst)703 void GC::SetForwardAddress(ObjectHeader *src, ObjectHeader *dst)
704 {
705 auto base_cls = src->ClassAddr<BaseClass>();
706 if (base_cls->IsDynamicClass()) {
707 auto cls = static_cast<HClass *>(base_cls);
708 // Note: During moving phase, 'src => dst'. Consider the src is a DynClass,
709 // since 'dst' is not in GC-status the 'manage-object' inside 'dst' won't be updated to
710 // 'dst'. To fix it, we update 'manage-object' here rather than upating phase.
711 if (cls->IsHClass()) {
712 size_t offset = ObjectHeader::ObjectHeaderSize() + HClass::GetManagedObjectOffset();
713 dst->SetFieldObject<false, false, true>(GetPandaVm()->GetAssociatedThread(), offset, dst);
714 }
715 }
716
717 // Set fwd address in src
718 bool update_res = false;
719 do {
720 MarkWord mark_word = src->AtomicGetMark();
721 MarkWord fwd_mark_word =
722 mark_word.DecodeFromForwardingAddress(static_cast<MarkWord::markWordSize>(ToUintPtr(dst)));
723 update_res = src->AtomicSetMark<false>(mark_word, fwd_mark_word);
724 } while (!update_res);
725 }
726
UpdateRefsInVRegs(ManagedThread * thread)727 void GC::UpdateRefsInVRegs(ManagedThread *thread)
728 {
729 LOG_DEBUG_GC << "Update frames for thread: " << thread->GetId();
730 for (auto pframe = StackWalker::Create(thread); pframe.HasFrame(); pframe.NextFrame()) {
731 LOG_DEBUG_GC << "Frame for method " << pframe.GetMethod()->GetFullName();
732 auto iterator = [&pframe, this](auto ®_info, auto &vreg) {
733 ObjectHeader *object_header = vreg.GetReference();
734 if (object_header == nullptr) {
735 return true;
736 }
737 MarkWord mark_word = object_header->AtomicGetMark();
738 if (mark_word.GetState() == MarkWord::ObjectState::STATE_GC) {
739 MarkWord::markWordSize addr = mark_word.GetForwardingAddress();
740 LOG_DEBUG_GC << "Update vreg, vreg old val = " << std::hex << object_header << ", new val = 0x" << addr;
741 LOG_IF(reg_info.IsAccumulator(), DEBUG, GC) << "^ acc reg";
742 if (!pframe.IsCFrame() && reg_info.IsAccumulator()) {
743 LOG_DEBUG_GC << "^ acc updated";
744 vreg.SetReference(reinterpret_cast<ObjectHeader *>(addr));
745 } else {
746 pframe.template SetVRegValue<std::is_same_v<decltype(vreg), interpreter::DynamicVRegisterRef &>>(
747 reg_info, reinterpret_cast<ObjectHeader *>(addr));
748 }
749 }
750 return true;
751 };
752 pframe.IterateObjectsWithInfo(iterator);
753 }
754 }
755
PopObjectFromStack(GCMarkingStackType * objects_stack)756 const ObjectHeader *GC::PopObjectFromStack(GCMarkingStackType *objects_stack)
757 {
758 auto *object = objects_stack->PopFromStack();
759 ASSERT(object != nullptr);
760 return object;
761 }
762
IsGenerational() const763 bool GC::IsGenerational() const
764 {
765 return IsGenerationalGCType(gc_type_);
766 }
767
FireGCPhaseStarted(GCPhase phase)768 void GC::FireGCPhaseStarted(GCPhase phase)
769 {
770 for (auto listener : *gc_listeners_ptr_) {
771 if (listener != nullptr) {
772 listener->GCPhaseStarted(phase);
773 }
774 }
775 }
776
FireGCPhaseFinished(GCPhase phase)777 void GC::FireGCPhaseFinished(GCPhase phase)
778 {
779 for (auto listener : *gc_listeners_ptr_) {
780 if (listener != nullptr) {
781 listener->GCPhaseFinished(phase);
782 }
783 }
784 }
785
OnWaitForIdleFail()786 void GC::OnWaitForIdleFail() {}
787
788 TEMPLATE_GC_CREATE_GC();
789
790 } // namespace panda::mem
791