1 /**
2 * Copyright (c) 2023-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "runtime/mem/gc/g1/update_remset_worker.h"
17
18 #include "mem/gc/card_table.h"
19 #include "runtime/include/language_context.h"
20 #include "runtime/include/panda_vm.h"
21 #include "runtime/mem/gc/g1/card_handler.h"
22 #include "runtime/mem/gc/g1/g1-gc.h"
23 #include "runtime/mem/object_helpers-inl.h"
24 #include "runtime/mem/rem_set-inl.h"
25
26 namespace ark::mem {
27
28 template <class LanguageConfig>
UpdateRemsetWorker(G1GC<LanguageConfig> * gc,GCG1BarrierSet::ThreadLocalCardQueues * queue,os::memory::Mutex * queueLock,size_t regionSize,bool updateConcurrent,size_t minConcurrentCardsToProcess,size_t hotCardsProcessingFrequency)29 UpdateRemsetWorker<LanguageConfig>::UpdateRemsetWorker(G1GC<LanguageConfig> *gc,
30 GCG1BarrierSet::ThreadLocalCardQueues *queue,
31 os::memory::Mutex *queueLock, size_t regionSize,
32 bool updateConcurrent, size_t minConcurrentCardsToProcess,
33 size_t hotCardsProcessingFrequency)
34 : gc_(gc),
35 regionSizeBits_(ark::helpers::math::GetIntLog2(regionSize)),
36 minConcurrentCardsToProcess_(minConcurrentCardsToProcess),
37 hotCardsProcessingFrequency_(hotCardsProcessingFrequency),
38 queue_(queue),
39 queueLock_(queueLock),
40 updateConcurrent_(updateConcurrent)
41 {
42 static constexpr size_t PREALLOCATED_CARDS_SET_SIZE = 256;
43 cards_.reserve(PREALLOCATED_CARDS_SET_SIZE);
44 }
45
46 template <class LanguageConfig>
~UpdateRemsetWorker()47 UpdateRemsetWorker<LanguageConfig>::~UpdateRemsetWorker()
48 {
49 auto allocator = gc_->GetInternalAllocator();
50 // Take the lock to satisfy TSAN.
51 // Actually at this moment all mutators should be destroyed and the lock is not needed.
52 os::memory::LockHolder holder(postBarrierBuffersLock_);
53 for (auto *buffer : postBarrierBuffers_) {
54 allocator->Delete(buffer);
55 }
56 }
57
58 template <class LanguageConfig>
CreateWorker()59 void UpdateRemsetWorker<LanguageConfig>::CreateWorker()
60 {
61 if (IsFlag(UpdateRemsetWorkerFlags::IS_STOP_WORKER)) {
62 RemoveFlag(UpdateRemsetWorkerFlags::IS_STOP_WORKER);
63 }
64 if (updateConcurrent_) {
65 this->CreateWorkerImpl();
66 }
67 }
68
69 template <class LanguageConfig>
DestroyWorker()70 void UpdateRemsetWorker<LanguageConfig>::DestroyWorker()
71 {
72 SetFlag(UpdateRemsetWorkerFlags::IS_STOP_WORKER);
73 if (updateConcurrent_) {
74 this->DestroyWorkerImpl();
75 }
76 }
77
78 template <class LanguageConfig>
AddPostBarrierBuffer(PandaVector<mem::CardTable::CardPtr> * buffer)79 void UpdateRemsetWorker<LanguageConfig>::AddPostBarrierBuffer(PandaVector<mem::CardTable::CardPtr> *buffer)
80 {
81 os::memory::LockHolder holder(postBarrierBuffersLock_);
82 postBarrierBuffers_.push_back(buffer);
83 }
84
85 template <class LanguageConfig>
FillFromDefered(PandaUnorderedSet<CardTable::CardPtr> * cards)86 void UpdateRemsetWorker<LanguageConfig>::FillFromDefered(PandaUnorderedSet<CardTable::CardPtr> *cards)
87 {
88 os::memory::LockHolder holder(*queueLock_);
89 std::copy(cards_.begin(), cards_.end(), std::inserter(*cards, cards->end()));
90 cards_.clear();
91 }
92
93 template <class LanguageConfig>
FillFromQueue(PandaUnorderedSet<CardTable::CardPtr> * cards)94 void UpdateRemsetWorker<LanguageConfig>::FillFromQueue(PandaUnorderedSet<CardTable::CardPtr> *cards)
95 {
96 os::memory::LockHolder holder(*queueLock_);
97 std::copy(queue_->begin(), queue_->end(), std::inserter(*cards, cards->end()));
98 queue_->clear();
99 }
100
101 template <class LanguageConfig>
FillFromThreads(PandaUnorderedSet<CardTable::CardPtr> * cards)102 void UpdateRemsetWorker<LanguageConfig>::FillFromThreads(PandaUnorderedSet<CardTable::CardPtr> *cards)
103 {
104 auto *vm = gc_->GetPandaVm();
105 ASSERT(vm != nullptr);
106 auto *threadManager = vm->GetThreadManager();
107 ASSERT(threadManager != nullptr);
108 threadManager->EnumerateThreads([this, cards](ManagedThread *thread) {
109 auto *buffer = thread->GetG1PostBarrierBuffer();
110 if (buffer != nullptr) {
111 FillFromPostBarrierBuffer(buffer, cards);
112 }
113 return true;
114 });
115 }
116
117 template <class LanguageConfig>
FillFromPostBarrierBuffers(PandaUnorderedSet<CardTable::CardPtr> * cards)118 void UpdateRemsetWorker<LanguageConfig>::FillFromPostBarrierBuffers(PandaUnorderedSet<CardTable::CardPtr> *cards)
119 {
120 auto allocator = gc_->GetInternalAllocator();
121 os::memory::LockHolder holder(postBarrierBuffersLock_);
122 while (!postBarrierBuffers_.empty()) {
123 auto *buffer = postBarrierBuffers_.back();
124 postBarrierBuffers_.pop_back();
125 FillFromPostBarrierBuffer(buffer, cards);
126 allocator->Delete(buffer);
127 }
128 }
129
130 template <class LanguageConfig>
FillFromHotCards(PandaUnorderedSet<CardTable::CardPtr> * cards)131 void UpdateRemsetWorker<LanguageConfig>::FillFromHotCards(PandaUnorderedSet<CardTable::CardPtr> *cards)
132 {
133 hotCards_.DrainMarkedCards(cards);
134 }
135
136 template <class LanguageConfig>
FillFromPostBarrierBuffer(GCG1BarrierSet::G1PostBarrierRingBufferType * postWrb,PandaUnorderedSet<CardTable::CardPtr> * cards)137 void UpdateRemsetWorker<LanguageConfig>::FillFromPostBarrierBuffer(GCG1BarrierSet::G1PostBarrierRingBufferType *postWrb,
138 PandaUnorderedSet<CardTable::CardPtr> *cards)
139 {
140 if (postWrb == nullptr) {
141 return;
142 }
143 bool hasElement;
144 while (true) {
145 mem::CardTable::CardPtr card;
146 hasElement = postWrb->TryPop(&card);
147 if (!hasElement) {
148 break;
149 }
150 cards->insert(card);
151 }
152 }
153
154 template <class LanguageConfig>
FillFromPostBarrierBuffer(GCG1BarrierSet::ThreadLocalCardQueues * postWrb,PandaUnorderedSet<CardTable::CardPtr> * cards)155 void UpdateRemsetWorker<LanguageConfig>::FillFromPostBarrierBuffer(GCG1BarrierSet::ThreadLocalCardQueues *postWrb,
156 PandaUnorderedSet<CardTable::CardPtr> *cards)
157 {
158 while (!postWrb->empty()) {
159 cards->insert(postWrb->back());
160 postWrb->pop_back();
161 }
162 }
163
164 template <class LanguageConfig>
NeedProcessHotCards()165 bool UpdateRemsetWorker<LanguageConfig>::NeedProcessHotCards()
166 {
167 static size_t processingCnt = 0;
168 if (processingCnt++ == hotCardsProcessingFrequency_) {
169 processingCnt = 0;
170 return true;
171 }
172 return false;
173 }
174
175 template <class LanguageConfig>
ProcessAllCards()176 size_t UpdateRemsetWorker<LanguageConfig>::ProcessAllCards()
177 {
178 size_t processedCardsCnt = ProcessCommonCards();
179 if (NeedProcessHotCards()) {
180 processedCardsCnt += ProcessHotCards();
181 }
182 return processedCardsCnt;
183 }
184
185 template <class LanguageConfig>
ProcessHotCards()186 size_t UpdateRemsetWorker<LanguageConfig>::ProcessHotCards()
187 {
188 CardHandler<LanguageConfig> cardHandler(gc_->GetCardTable(), regionSizeBits_, deferCards_);
189 auto processedCardsCnt = hotCards_.HandleCards(cardHandler);
190 return processedCardsCnt;
191 }
192
193 template <class LanguageConfig>
UpdateCardStatus(CardTable::CardPtr card)194 void UpdateRemsetWorker<LanguageConfig>::UpdateCardStatus(CardTable::CardPtr card)
195 {
196 auto cardValue = card->GetCard();
197 auto cardStatus = CardTable::Card::GetStatus(cardValue);
198
199 ASSERT_PRINT(!CardTable::Card::IsProcessed(cardStatus) && !CardTable::Card::IsYoung(cardStatus),
200 "UpdateCardStatus card: " << card << " cardValue: " << (int)cardValue);
201
202 if (!CardTable::Card::IsMarked(cardStatus)) {
203 return;
204 }
205
206 if (CardTable::Card::IsHot(cardValue)) {
207 HotCards::SetMaxHotValue(card, cardValue);
208 return;
209 }
210
211 if (CardTable::Card::IsMaxHotValue(cardValue)) {
212 HotCards::SetHot(card, cardValue);
213 hotCards_.PushCard(card);
214 return;
215 }
216
217 HotCards::IncrementHotValue(card, cardValue & HotCards::RESET_MARK_MASK);
218 }
219
220 template <class LanguageConfig>
ProcessCommonCards()221 size_t UpdateRemsetWorker<LanguageConfig>::ProcessCommonCards()
222 {
223 FillFromQueue(&cards_);
224 FillFromThreads(&cards_);
225 FillFromPostBarrierBuffers(&cards_);
226
227 std::for_each(cards_.begin(), cards_.end(), [this](auto *card) { UpdateCardStatus(card); });
228
229 // clear cards before we process it, because parallel mutator thread can make a write and we would miss it
230 arch::StoreLoadBarrier();
231
232 LOG_IF(!cards_.empty(), DEBUG, GC) << "Started process: " << cards_.size() << " cards";
233
234 size_t cardsSize = 0;
235 CardHandler<LanguageConfig> cardHandler(gc_->GetCardTable(), regionSizeBits_, deferCards_);
236 for (auto it = cards_.begin(); it != cards_.end();) {
237 auto cardPtr = *it;
238 if (!cardPtr->IsHot()) {
239 if (!cardHandler.Handle(cardPtr)) {
240 break;
241 }
242 ++cardsSize;
243 }
244 it = cards_.erase(it);
245 }
246 LOG_IF(!cards_.empty(), DEBUG, GC) << "Processed " << cardsSize << " cards";
247
248 hotCards_.DecrementHotValue();
249 return cardsSize;
250 }
251
252 template <class LanguageConfig>
DrainAllCards(PandaUnorderedSet<CardTable::CardPtr> * cards)253 void UpdateRemsetWorker<LanguageConfig>::DrainAllCards(PandaUnorderedSet<CardTable::CardPtr> *cards)
254 {
255 ASSERT(IsFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD));
256 os::memory::LockHolder holder(updateRemsetLock_);
257 FillFromDefered(cards);
258 FillFromQueue(cards);
259 FillFromThreads(cards);
260 FillFromPostBarrierBuffers(cards);
261 FillFromHotCards(cards);
262 }
263
264 template <class LanguageConfig>
GCProcessCards()265 void UpdateRemsetWorker<LanguageConfig>::GCProcessCards()
266 {
267 ASSERT(IsFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD));
268 os::memory::LockHolder holder(updateRemsetLock_);
269 ProcessCommonCards();
270 ProcessHotCards();
271 hotCards_.ClearHotCards();
272 }
273
274 template <class LanguageConfig>
DoInvalidateRegions(RegionVector * regions)275 void UpdateRemsetWorker<LanguageConfig>::DoInvalidateRegions(RegionVector *regions)
276 {
277 for (auto *region : *regions) {
278 // don't need lock because only one thread changes remsets
279 RemSet<>::template InvalidateRegion<false>(region);
280 }
281 }
282
283 template <class LanguageConfig>
GCInvalidateRegions(RegionVector * regions)284 void UpdateRemsetWorker<LanguageConfig>::GCInvalidateRegions(RegionVector *regions)
285 {
286 // Do invalidate region on pause in GCThread
287 ASSERT(IsFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD));
288 os::memory::LockHolder holder(updateRemsetLock_);
289 DoInvalidateRegions(regions);
290 }
291
292 template <class LanguageConfig>
InvalidateRegions(RegionVector * regions)293 void UpdateRemsetWorker<LanguageConfig>::InvalidateRegions(RegionVector *regions)
294 {
295 // Do invalidate region during concurrent sweep
296 ASSERT(IsFlag(UpdateRemsetWorkerFlags::IS_PROCESS_CARD));
297 SetFlag(UpdateRemsetWorkerFlags::IS_INVALIDATE_REGIONS);
298 // Atomic with relaxed order reason: memory order is not required
299 deferCards_.store(true, std::memory_order_relaxed);
300 // Aquare lock to be sure that UpdateRemsetWorker has been stopped
301 os::memory::LockHolder holder(updateRemsetLock_);
302 LOG(DEBUG, GC) << "Remset worker has been paused to invalidate region on concurrent";
303 DoInvalidateRegions(regions);
304 // Atomic with relaxed order reason: memory order is not required
305 deferCards_.store(false, std::memory_order_relaxed);
306 RemoveFlag(UpdateRemsetWorkerFlags::IS_INVALIDATE_REGIONS);
307 if (updateConcurrent_) {
308 this->ContinueProcessCards();
309 }
310 }
311
312 template <class LanguageConfig>
SuspendWorkerForGCPause()313 void UpdateRemsetWorker<LanguageConfig>::SuspendWorkerForGCPause()
314 {
315 ASSERT(!IsFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD));
316 SetFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD);
317 // Atomic with relaxed order reason: memory order is not required
318 deferCards_.store(true, std::memory_order_relaxed);
319 // Aquare lock to be sure that UpdateRemsetWorker has been stopped
320 os::memory::LockHolder holder(updateRemsetLock_);
321 #ifndef NDEBUG
322 pausedByGcThread_ = true;
323 #endif
324 LOG(DEBUG, GC) << "Remset worker has been paused for GC";
325 // Atomic with relaxed order reason: memory order is not required
326 deferCards_.store(false, std::memory_order_relaxed);
327 }
328
329 template <class LanguageConfig>
ResumeWorkerAfterGCPause()330 void UpdateRemsetWorker<LanguageConfig>::ResumeWorkerAfterGCPause()
331 {
332 ASSERT(IsFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD));
333 os::memory::LockHolder holder(updateRemsetLock_);
334 #ifndef NDEBUG
335 pausedByGcThread_ = false;
336 #endif
337 RemoveFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD);
338 if (updateConcurrent_) {
339 this->ContinueProcessCards();
340 }
341 }
342
343 TEMPLATE_CLASS_LANGUAGE_CONFIG(UpdateRemsetWorker);
344 } // namespace ark::mem
345