1 /**
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 #ifndef PANDA_RUNTIME_MEM_GC_G1_G1_EVACUATE_REGIONS_WORKER_STATE_INL_H
16 #define PANDA_RUNTIME_MEM_GC_G1_G1_EVACUATE_REGIONS_WORKER_STATE_INL_H
17
18 #include "runtime/mem/gc/g1/g1-evacuate-regions-worker-state.h"
19 #include "runtime/mem/object_helpers.h"
20 #include "runtime/mem/region_space.h"
21 #include "libpandabase/utils/type_converter.h"
22 #include "runtime/mem/object-references-iterator-inl.h"
23 #include "runtime/mem/gc/gc_adaptive_stack_inl.h"
24
25 namespace ark::mem {
26
27 template <typename LanguageConfig>
G1EvacuateRegionsWorkerState(G1GC<LanguageConfig> * gc,GCEvacuateRegionsTaskStack<Ref> * refStack)28 G1EvacuateRegionsWorkerState<LanguageConfig>::G1EvacuateRegionsWorkerState(G1GC<LanguageConfig> *gc,
29 GCEvacuateRegionsTaskStack<Ref> *refStack)
30 : gc_(gc),
31 objectAllocator_(gc_->GetG1ObjectAllocator()),
32 cardTable_(gc->GetCardTable()),
33 regionSizeBits_(gc_->regionSizeBits_),
34 refStack_(refStack),
35 evacuationObjectPointerHandler_(this)
36 {
37 regionTo_ = GetNextRegion();
38 }
39
40 template <typename LanguageConfig>
~G1EvacuateRegionsWorkerState()41 G1EvacuateRegionsWorkerState<LanguageConfig>::~G1EvacuateRegionsWorkerState()
42 {
43 os::memory::LockHolder lock(gc_->gcWorkerQueueLock_);
44
45 gc_->memStats_.template RecordCountMovedYoung<false>(GetCopiedObjectsYoung());
46 gc_->memStats_.template RecordCountMovedTenured<false>(GetCopiedObjectsOld());
47 gc_->copiedBytesYoung_ += GetCopiedBytesYoung();
48 gc_->copiedBytesOld_ += GetCopiedBytesOld();
49
50 VisitCards([this](auto *card) {
51 if (!card->IsMarked()) {
52 card->Mark();
53 gc_->updatedRefsQueueTemp_->push_back(card);
54 }
55 });
56 GetRegionTo()->SetLiveBytes(GetRegionTo()->GetLiveBytes() + GetLiveBytes());
57 auto *objectAllocator = gc_->GetG1ObjectAllocator();
58 objectAllocator->template PushToOldRegionQueue<true>(GetRegionTo());
59 }
60
61 template <typename LanguageConfig>
Evacuate(ObjectHeader * obj,MarkWord markWord)62 ObjectHeader *G1EvacuateRegionsWorkerState<LanguageConfig>::Evacuate(ObjectHeader *obj, MarkWord markWord)
63 {
64 auto objectSize = GetObjectSize(obj);
65 auto alignedSize = AlignUp(objectSize, DEFAULT_ALIGNMENT_IN_BYTES);
66
67 ASSERT(regionTo_ != nullptr);
68
69 void *dst = regionTo_->template Alloc<false>(alignedSize);
70 if (dst == nullptr) {
71 regionTo_->SetLiveBytes(regionTo_->GetLiveBytes() + liveBytes_);
72 liveBytes_ = 0;
73 regionTo_ = CreateNewRegion();
74 dst = regionTo_->template Alloc<false>(alignedSize);
75 }
76 // Don't initialize memory for an object here because we will use memcpy anyway
77 ASSERT(dst != nullptr);
78 [[maybe_unused]] auto copyResult = memcpy_s(dst, objectSize, obj, objectSize);
79 ASSERT(copyResult == 0);
80 // need to mark as alive moved object
81 ASSERT(regionTo_->GetLiveBitmap() != nullptr);
82
83 auto *newObj = static_cast<ObjectHeader *>(dst);
84 TSAN_ANNOTATE_IGNORE_WRITES_BEGIN();
85 // Since we forward with relaxed memory order there are no ordering with above CopyObject and other threads should
86 // not examine forwardee content without synchronization
87 auto *forwardAddr = SetForwardAddress(obj, newObj, markWord);
88 TSAN_ANNOTATE_IGNORE_WRITES_END();
89
90 if (forwardAddr == nullptr) {
91 ObjectIterator<LanguageConfig::LANG_TYPE>::template IterateAndDiscoverReferences(
92 GetGC(), newObj, &evacuationObjectPointerHandler_);
93
94 if (ObjectToRegion(obj)->HasFlag(RegionFlag::IS_EDEN)) {
95 copiedBytesYoung_ += alignedSize;
96 copiedObjectsYoung_++;
97 } else {
98 copiedBytesOld_ += alignedSize;
99 copiedObjectsOld_++;
100 }
101
102 regionTo_->IncreaseAllocatedObjects();
103 regionTo_->GetLiveBitmap()->Set(dst);
104 liveBytes_ += alignedSize;
105
106 return newObj;
107 }
108
109 regionTo_->UndoAlloc(dst);
110 return forwardAddr;
111 }
112
113 template <typename LanguageConfig>
ProcessRef(Ref p)114 void G1EvacuateRegionsWorkerState<LanguageConfig>::ProcessRef(Ref p)
115 {
116 auto o = ObjectAccessor::Load(p);
117 auto *obj = ObjectAccessor::DecodeNotNull(o);
118 if (!gc_->InGCSweepRange(obj)) {
119 // reference has been already processed
120 return;
121 }
122
123 // Atomic with relaxed order reason: memory order is not required
124 MarkWord markWord = obj->AtomicGetMark(std::memory_order_relaxed);
125 if (markWord.IsForwarded()) {
126 obj = reinterpret_cast<ObjectHeader *>(markWord.GetForwardingAddress());
127 } else {
128 obj = Evacuate(obj, markWord);
129 }
130
131 ObjectAccessor::Store(p, obj);
132
133 if (!IsSameRegion(p, obj)) {
134 EnqueueCard(p);
135 }
136 }
137
138 template <typename LanguageConfig>
SetForwardAddress(ObjectHeader * src,ObjectHeader * dst,MarkWord markWord)139 ObjectHeader *G1EvacuateRegionsWorkerState<LanguageConfig>::SetForwardAddress(ObjectHeader *src, ObjectHeader *dst,
140 MarkWord markWord)
141 {
142 MarkWord fwdMarkWord = markWord.DecodeFromForwardingAddress(static_cast<MarkWord::MarkWordSize>(ToUintPtr(dst)));
143 // Atomic with relaxed order reason: memory order is not required
144 auto success = src->AtomicSetMark<true>(markWord, fwdMarkWord, std::memory_order_relaxed);
145 if (success) {
146 if constexpr (LanguageConfig::LANG_TYPE == LANG_TYPE_DYNAMIC) {
147 auto *baseCls = src->ClassAddr<BaseClass>();
148 if (baseCls->IsDynamicClass() && static_cast<HClass *>(baseCls)->IsHClass()) {
149 // Note: During moving phase, 'src => dst'. Consider the src is a DynClass,
150 // since 'dst' is not in GC-status the 'manage-object' inside 'dst' won't be updated to
151 // 'dst'. To fix it, we update 'manage-object' here rather than upating phase.
152 size_t offset = ObjectHeader::ObjectHeaderSize() + HClass::GetManagedObjectOffset();
153 dst->SetFieldObject<false, false, true>(gc_->GetPandaVm()->GetAssociatedThread(), offset, dst);
154 }
155 }
156
157 return nullptr;
158 }
159
160 return reinterpret_cast<ObjectHeader *>(markWord.GetForwardingAddress());
161 }
162
163 template <typename LanguageConfig>
EvacuateNonHeapRoots()164 void G1EvacuateRegionsWorkerState<LanguageConfig>::EvacuateNonHeapRoots()
165 {
166 GCRootVisitor gcMarkCollectionSet = [this](const GCRoot &gcRoot) {
167 ASSERT(gcRoot.GetFromObjectHeader() == nullptr);
168 auto *rootObject = gcRoot.GetObjectHeader();
169 ASSERT(rootObject != nullptr);
170 LOG(DEBUG, GC) << "Handle root " << GetDebugInfoAboutObject(rootObject) << " from: " << gcRoot.GetType();
171 if (!gc_->InGCSweepRange(rootObject)) {
172 // Skip non-collection-set roots
173 return;
174 }
175 MarkWord markWord = rootObject->AtomicGetMark(std::memory_order_relaxed);
176 if (markWord.IsForwarded()) {
177 return;
178 }
179 LOG(DEBUG, GC) << "root " << GetDebugInfoAboutObject(rootObject);
180 Evacuate(rootObject, markWord);
181 };
182
183 gc_->VisitRoots(gcMarkCollectionSet, VisitGCRootFlags::ACCESS_ROOT_NONE);
184 }
185
186 template <typename LanguageConfig>
ScanRemset(const RemSet<> & remset)187 size_t G1EvacuateRegionsWorkerState<LanguageConfig>::ScanRemset(const RemSet<> &remset)
188 {
189 size_t remsetSize = 0;
190 auto remsetVisitor = [this, &remsetSize](Region *region, const MemRange &range) {
191 IterateRefsInMemRange(range, region);
192 remsetSize++;
193 };
194 remset.Iterate([](Region *r) { return !r->HasFlag(IS_COLLECTION_SET); }, remsetVisitor);
195 return remsetSize;
196 }
197
198 template <class LanguageConfig>
IterateRefsInMemRange(const MemRange & memRange,Region * region)199 void G1EvacuateRegionsWorkerState<LanguageConfig>::IterateRefsInMemRange(const MemRange &memRange, Region *region)
200 {
201 // Use MarkBitmap instead of LiveBitmap because it concurrently updated during evacuation
202 MarkBitmap *bitmap = region->GetMarkBitmap();
203 auto *startAddress = ToVoidPtr(memRange.GetStartAddress());
204 auto *endAddress = ToVoidPtr(memRange.GetEndAddress());
205 auto wrapper = [this, startAddress, endAddress](void *mem) {
206 ObjectIterator<LanguageConfig::LANG_TYPE>::template IterateAndDiscoverReferences(
207 GetGC(), static_cast<ObjectHeader *>(mem), &evacuationObjectPointerHandler_, startAddress, endAddress);
208 };
209 if (region->HasFlag(RegionFlag::IS_LARGE_OBJECT)) {
210 bitmap->CallForMarkedChunkInHumongousRegion<false>(ToVoidPtr(region->Begin()), wrapper);
211 } else {
212 bitmap->IterateOverMarkedChunkInRange(startAddress, endAddress, wrapper);
213 }
214 }
215
216 template <typename LanguageConfig>
EvacuateLiveObjects()217 void G1EvacuateRegionsWorkerState<LanguageConfig>::EvacuateLiveObjects()
218 {
219 refStack_->TraverseObjects(*this);
220 }
221 } // namespace ark::mem
222 #endif
223