1 /**
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 #ifndef PANDA_RUNTIME_MEM_GC_G1_G1_EVACUATE_REGIONS_WORKER_STATE_INL_H
16 #define PANDA_RUNTIME_MEM_GC_G1_G1_EVACUATE_REGIONS_WORKER_STATE_INL_H
17
18 #include "runtime/mem/gc/g1/g1-evacuate-regions-worker-state.h"
19 #include "runtime/mem/object_helpers.h"
20 #include "runtime/mem/region_space.h"
21 #include "libpandabase/utils/type_converter.h"
22 #include "runtime/mem/object-references-iterator-inl.h"
23
24 namespace ark::mem {
25
26 template <typename LanguageConfig>
G1EvacuateRegionsWorkerState(G1GC<LanguageConfig> * gc,uint id,PandaVector<Ref> * queue,SharedState * shared)27 G1EvacuateRegionsWorkerState<LanguageConfig>::G1EvacuateRegionsWorkerState(G1GC<LanguageConfig> *gc, uint id,
28 PandaVector<Ref> *queue, SharedState *shared)
29 : gc_(gc),
30 objectAllocator_(gc_->GetG1ObjectAllocator()),
31 cardTable_(gc->GetCardTable()),
32 regionSizeBits_(gc_->regionSizeBits_),
33 id_(id),
34 shared_(shared),
35 queue_(queue),
36 evacuationObjectPointerHandler_(this)
37 {
38 }
39
40 template <typename LanguageConfig>
Evacuate(ObjectHeader * obj,MarkWord markWord)41 ObjectHeader *G1EvacuateRegionsWorkerState<LanguageConfig>::Evacuate(ObjectHeader *obj, MarkWord markWord)
42 {
43 auto objectSize = GetObjectSize(obj);
44 auto alignedSize = AlignUp(objectSize, DEFAULT_ALIGNMENT_IN_BYTES);
45
46 ASSERT(regionTo_ != nullptr);
47
48 void *dst = regionTo_->template Alloc<false>(alignedSize);
49 if (dst == nullptr) {
50 regionTo_->SetLiveBytes(regionTo_->GetLiveBytes() + liveBytes_);
51 liveBytes_ = 0;
52 regionTo_ = CreateNewRegion();
53 dst = regionTo_->template Alloc<false>(alignedSize);
54 }
55 // Don't initialize memory for an object here because we will use memcpy anyway
56 ASSERT(dst != nullptr);
57 [[maybe_unused]] auto copyResult = memcpy_s(dst, objectSize, obj, objectSize);
58 ASSERT(copyResult == 0);
59 // need to mark as alive moved object
60 ASSERT(regionTo_->GetLiveBitmap() != nullptr);
61
62 auto *newObj = static_cast<ObjectHeader *>(dst);
63 TSAN_ANNOTATE_IGNORE_WRITES_BEGIN();
64 // Since we forward with relaxed memory order there are no ordering with above CopyObject and other threads should
65 // not examine forwardee content without synchronization
66 auto *forwardAddr = SetForwardAddress(obj, newObj, markWord);
67 TSAN_ANNOTATE_IGNORE_WRITES_END();
68
69 if (forwardAddr == nullptr) {
70 ObjectIterator<LanguageConfig::LANG_TYPE>::template IterateAndDiscoverReferences(
71 GetGC(), newObj, &evacuationObjectPointerHandler_);
72
73 if (ObjectToRegion(obj)->HasFlag(RegionFlag::IS_EDEN)) {
74 copiedBytesYoung_ += alignedSize;
75 copiedObjectsYoung_++;
76 } else {
77 copiedBytesOld_ += alignedSize;
78 copiedObjectsOld_++;
79 }
80
81 regionTo_->IncreaseAllocatedObjects();
82 regionTo_->GetLiveBitmap()->Set(dst);
83 liveBytes_ += alignedSize;
84
85 return newObj;
86 }
87
88 regionTo_->UndoAlloc(dst);
89 return forwardAddr;
90 }
91
92 template <typename LanguageConfig>
ProcessRef(Ref p)93 void G1EvacuateRegionsWorkerState<LanguageConfig>::ProcessRef(Ref p)
94 {
95 auto o = ObjectAccessor::Load(p);
96 auto *obj = ObjectAccessor::DecodeNotNull(o);
97 if (!gc_->InGCSweepRange(obj)) {
98 // reference has been already processed
99 return;
100 }
101
102 // Atomic with relaxed order reason: memory order is not required
103 MarkWord markWord = obj->AtomicGetMark(std::memory_order_relaxed);
104 if (markWord.GetState() == MarkWord::ObjectState::STATE_GC) {
105 obj = reinterpret_cast<ObjectHeader *>(markWord.GetForwardingAddress());
106 } else {
107 obj = Evacuate(obj, markWord);
108 }
109
110 ObjectAccessor::Store(p, obj);
111
112 if (!IsSameRegion(p, obj)) {
113 EnqueueCard(p);
114 }
115 }
116
117 template <typename LanguageConfig>
SetForwardAddress(ObjectHeader * src,ObjectHeader * dst,MarkWord markWord)118 ObjectHeader *G1EvacuateRegionsWorkerState<LanguageConfig>::SetForwardAddress(ObjectHeader *src, ObjectHeader *dst,
119 MarkWord markWord)
120 {
121 MarkWord fwdMarkWord = markWord.DecodeFromForwardingAddress(static_cast<MarkWord::MarkWordSize>(ToUintPtr(dst)));
122 // Atomic with relaxed order reason: memory order is not required
123 auto success = src->AtomicSetMark<true>(markWord, fwdMarkWord, std::memory_order_relaxed);
124 if (success) {
125 if constexpr (LanguageConfig::LANG_TYPE == LANG_TYPE_DYNAMIC) {
126 auto *baseCls = src->ClassAddr<BaseClass>();
127 if (baseCls->IsDynamicClass() && static_cast<HClass *>(baseCls)->IsHClass()) {
128 // Note: During moving phase, 'src => dst'. Consider the src is a DynClass,
129 // since 'dst' is not in GC-status the 'manage-object' inside 'dst' won't be updated to
130 // 'dst'. To fix it, we update 'manage-object' here rather than upating phase.
131 size_t offset = ObjectHeader::ObjectHeaderSize() + HClass::GetManagedObjectOffset();
132 dst->SetFieldObject<false, false, true>(gc_->GetPandaVm()->GetAssociatedThread(), offset, dst);
133 }
134 }
135
136 return nullptr;
137 }
138
139 return reinterpret_cast<ObjectHeader *>(markWord.GetForwardingAddress());
140 }
141
142 template <typename LanguageConfig>
StartWork()143 void G1EvacuateRegionsWorkerState<LanguageConfig>::StartWork()
144 {
145 taskStart_ = ark::time::GetCurrentTimeInNanos();
146 }
147
148 template <typename LanguageConfig>
EndWork()149 void G1EvacuateRegionsWorkerState<LanguageConfig>::EndWork()
150 {
151 taskEnd_ = ark::time::GetCurrentTimeInNanos();
152 }
153
154 template <typename LanguageConfig>
EvacuateNonHeapRoots()155 void G1EvacuateRegionsWorkerState<LanguageConfig>::EvacuateNonHeapRoots()
156 {
157 time::Timer timer(&scanNonHeapRootsDuration_);
158 if (GetShared()->GetWorkersCount() == 1 || GetId() == 0) {
159 // Only first worker processes roots, currently it is not parallized
160 GCRootVisitor gcMarkCollectionSet = [this](const GCRoot &gcRoot) {
161 ASSERT(gcRoot.GetFromObjectHeader() == nullptr);
162 auto *rootObject = gcRoot.GetObjectHeader();
163 ASSERT(rootObject != nullptr);
164 LOG(DEBUG, GC) << "Handle root " << GetDebugInfoAboutObject(rootObject) << " from: " << gcRoot.GetType();
165 if (!GetGC()->InGCSweepRange(rootObject)) {
166 // Skip non-collection-set roots
167 return;
168 }
169 MarkWord markWord = rootObject->AtomicGetMark(std::memory_order_relaxed);
170 if (markWord.GetState() == MarkWord::ObjectState::STATE_GC) {
171 // already evacuated by concurrent worker
172 return;
173 }
174 LOG(DEBUG, GC) << "root " << GetDebugInfoAboutObject(rootObject);
175 Evacuate(rootObject, markWord);
176 };
177 GetGC()->VisitRoots(gcMarkCollectionSet, VisitGCRootFlags::ACCESS_ROOT_NONE);
178 }
179 }
180
181 class ConcurrentRemSetDistributor {
182 public:
ConcurrentRemSetDistributor(std::atomic_size_t & counter)183 explicit ConcurrentRemSetDistributor(std::atomic_size_t &counter) : counter_(counter) {}
184
Distribute()185 bool Distribute()
186 {
187 if (claimed_) {
188 // Atomic with seq_cst order reason: full order is required to synchronize workers
189 indexToProcess_ = counter_.fetch_add(1, std::memory_order_seq_cst);
190 }
191
192 claimed_ = currentIndex_ == indexToProcess_;
193 currentIndex_++;
194 return claimed_;
195 }
196
197 private:
198 size_t currentIndex_ {0};
199 size_t indexToProcess_ {0};
200 bool claimed_ {true};
201 std::atomic_size_t &counter_;
202 };
203
204 class EvenRemSetDistributor {
205 public:
EvenRemSetDistributor(uint64_t id,size_t totalCount)206 EvenRemSetDistributor(uint64_t id, size_t totalCount) : id_(id), totalCount_(totalCount) {}
207
Distribute()208 bool Distribute()
209 {
210 auto oldvalue = currentIndex_++;
211 return oldvalue % totalCount_ == id_;
212 }
213
214 private:
215 const uint64_t id_;
216 const size_t totalCount_;
217 size_t currentIndex_ {0};
218 };
219
220 template <typename LanguageConfig>
ScanRemset()221 void G1EvacuateRegionsWorkerState<LanguageConfig>::ScanRemset()
222 {
223 time::Timer timer(&scanRemsetDuration_);
224 size_t total = 0;
225 EvenRemSetDistributor distributor(GetId(), GetShared()->GetWorkersCount());
226 auto remsetVisitor = [this, &distributor, &total](Region *r, const MemRange &range) {
227 if (distributor.Distribute()) {
228 IterateRefsInMemRange(range, r, &evacuationObjectPointerHandler_);
229 total++;
230 }
231 };
232
233 GetShared()->GetRemset()->Iterate([](Region *r) { return !r->HasFlag(IS_COLLECTION_SET); }, remsetVisitor);
234 remsetCount_ = total;
235
236 reenqueueCardsCount_ = cardQueue_.size();
237 }
238
239 template <class LanguageConfig>
240 template <typename Visitor>
IterateRefsInMemRange(const MemRange & memRange,Region * region,Visitor * visitor)241 void G1EvacuateRegionsWorkerState<LanguageConfig>::IterateRefsInMemRange(const MemRange &memRange, Region *region,
242 Visitor *visitor)
243 {
244 // Use MarkBitmap instead of LiveBitmap because it concurrently updated during evacuation
245 MarkBitmap *bitmap = region->GetMarkBitmap();
246 auto *startAddress = ToVoidPtr(memRange.GetStartAddress());
247 auto *endAddress = ToVoidPtr(memRange.GetEndAddress());
248 auto wrapper = [this, visitor, startAddress, endAddress](void *mem) {
249 ObjectIterator<LanguageConfig::LANG_TYPE>::template IterateAndDiscoverReferences(
250 GetGC(), static_cast<ObjectHeader *>(mem), visitor, startAddress, endAddress);
251 };
252 if (region->HasFlag(RegionFlag::IS_LARGE_OBJECT)) {
253 bitmap->CallForMarkedChunkInHumongousRegion<false>(ToVoidPtr(region->Begin()), wrapper);
254 } else {
255 bitmap->IterateOverMarkedChunkInRange(startAddress, endAddress, wrapper);
256 }
257 }
258
259 template <typename LanguageConfig>
EvacuateLiveObjects()260 void G1EvacuateRegionsWorkerState<LanguageConfig>::EvacuateLiveObjects()
261 {
262 time::Timer timer(&evacuationDuration_);
263 ProcessQueue();
264
265 evacuationAddCardsCount_ = cardQueue_.size() - reenqueueCardsCount_;
266 }
267 } // namespace ark::mem
268 #endif
269