1 /**
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "runtime/mem/gc/g1/update_remset_worker.h"
17
18 #include "runtime/include/language_context.h"
19 #include "runtime/include/panda_vm.h"
20 #include "runtime/mem/gc/g1/card_handler.h"
21 #include "runtime/mem/gc/g1/g1-gc.h"
22 #include "runtime/mem/object_helpers-inl.h"
23 #include "runtime/mem/rem_set-inl.h"
24
25 namespace panda::mem {
26
27 template <class LanguageConfig>
UpdateRemsetWorker(G1GC<LanguageConfig> * gc,GCG1BarrierSet::ThreadLocalCardQueues * queue,os::memory::Mutex * queueLock,size_t regionSize,bool updateConcurrent,size_t minConcurrentCardsToProcess)28 UpdateRemsetWorker<LanguageConfig>::UpdateRemsetWorker(G1GC<LanguageConfig> *gc,
29 GCG1BarrierSet::ThreadLocalCardQueues *queue,
30 os::memory::Mutex *queueLock, size_t regionSize,
31 bool updateConcurrent, size_t minConcurrentCardsToProcess)
32 : gc_(gc),
33 regionSizeBits_(panda::helpers::math::GetIntLog2(regionSize)),
34 minConcurrentCardsToProcess_(minConcurrentCardsToProcess),
35 queue_(queue),
36 queueLock_(queueLock),
37 updateConcurrent_(updateConcurrent)
38 {
39 static constexpr size_t PREALLOCATED_CARDS_SET_SIZE = 256;
40 cards_.reserve(PREALLOCATED_CARDS_SET_SIZE);
41 }
42
43 template <class LanguageConfig>
~UpdateRemsetWorker()44 UpdateRemsetWorker<LanguageConfig>::~UpdateRemsetWorker()
45 {
46 auto allocator = gc_->GetInternalAllocator();
47 // Take the lock to satisfy TSAN.
48 // Actually at this moment all mutators should be destroyed and the lock is not needed.
49 os::memory::LockHolder holder(postBarrierBuffersLock_);
50 for (auto *buffer : postBarrierBuffers_) {
51 allocator->Delete(buffer);
52 }
53 }
54
55 template <class LanguageConfig>
CreateWorker()56 void UpdateRemsetWorker<LanguageConfig>::CreateWorker()
57 {
58 if (IsFlag(UpdateRemsetWorkerFlags::IS_STOP_WORKER)) {
59 RemoveFlag(UpdateRemsetWorkerFlags::IS_STOP_WORKER);
60 }
61 if (updateConcurrent_) {
62 this->CreateWorkerImpl();
63 }
64 }
65
66 template <class LanguageConfig>
DestroyWorker()67 void UpdateRemsetWorker<LanguageConfig>::DestroyWorker()
68 {
69 SetFlag(UpdateRemsetWorkerFlags::IS_STOP_WORKER);
70 if (updateConcurrent_) {
71 this->DestroyWorkerImpl();
72 }
73 }
74
75 template <class LanguageConfig>
AddPostBarrierBuffer(PandaVector<mem::CardTable::CardPtr> * buffer)76 void UpdateRemsetWorker<LanguageConfig>::AddPostBarrierBuffer(PandaVector<mem::CardTable::CardPtr> *buffer)
77 {
78 os::memory::LockHolder holder(postBarrierBuffersLock_);
79 postBarrierBuffers_.push_back(buffer);
80 }
81
82 template <class LanguageConfig>
FillFromDefered(PandaUnorderedSet<CardTable::CardPtr> * cards)83 void UpdateRemsetWorker<LanguageConfig>::FillFromDefered(PandaUnorderedSet<CardTable::CardPtr> *cards)
84 {
85 os::memory::LockHolder holder(*queueLock_);
86 std::copy(cards_.begin(), cards_.end(), std::inserter(*cards, cards->end()));
87 cards_.clear();
88 }
89
90 template <class LanguageConfig>
FillFromQueue(PandaUnorderedSet<CardTable::CardPtr> * cards)91 void UpdateRemsetWorker<LanguageConfig>::FillFromQueue(PandaUnorderedSet<CardTable::CardPtr> *cards)
92 {
93 os::memory::LockHolder holder(*queueLock_);
94 std::copy(queue_->begin(), queue_->end(), std::inserter(*cards, cards->end()));
95 queue_->clear();
96 }
97
98 template <class LanguageConfig>
FillFromThreads(PandaUnorderedSet<CardTable::CardPtr> * cards)99 void UpdateRemsetWorker<LanguageConfig>::FillFromThreads(PandaUnorderedSet<CardTable::CardPtr> *cards)
100 {
101 auto *vm = gc_->GetPandaVm();
102 ASSERT(vm != nullptr);
103 auto *threadManager = vm->GetThreadManager();
104 ASSERT(threadManager != nullptr);
105 threadManager->EnumerateThreads([this, cards](ManagedThread *thread) {
106 auto *buffer = thread->GetG1PostBarrierBuffer();
107 if (buffer != nullptr) {
108 FillFromPostBarrierBuffer(buffer, cards);
109 }
110 return true;
111 });
112 }
113
114 template <class LanguageConfig>
FillFromPostBarrierBuffers(PandaUnorderedSet<CardTable::CardPtr> * cards)115 void UpdateRemsetWorker<LanguageConfig>::FillFromPostBarrierBuffers(PandaUnorderedSet<CardTable::CardPtr> *cards)
116 {
117 auto allocator = gc_->GetInternalAllocator();
118 os::memory::LockHolder holder(postBarrierBuffersLock_);
119 while (!postBarrierBuffers_.empty()) {
120 auto *buffer = postBarrierBuffers_.back();
121 postBarrierBuffers_.pop_back();
122 FillFromPostBarrierBuffer(buffer, cards);
123 allocator->Delete(buffer);
124 }
125 }
126
127 template <class LanguageConfig>
FillFromPostBarrierBuffer(GCG1BarrierSet::G1PostBarrierRingBufferType * postWrb,PandaUnorderedSet<CardTable::CardPtr> * cards)128 void UpdateRemsetWorker<LanguageConfig>::FillFromPostBarrierBuffer(GCG1BarrierSet::G1PostBarrierRingBufferType *postWrb,
129 PandaUnorderedSet<CardTable::CardPtr> *cards)
130 {
131 if (postWrb == nullptr) {
132 return;
133 }
134 bool hasElement;
135 while (true) {
136 mem::CardTable::CardPtr card;
137 hasElement = postWrb->TryPop(&card);
138 if (!hasElement) {
139 break;
140 }
141 cards->insert(card);
142 }
143 }
144
145 template <class LanguageConfig>
FillFromPostBarrierBuffer(GCG1BarrierSet::ThreadLocalCardQueues * postWrb,PandaUnorderedSet<CardTable::CardPtr> * cards)146 void UpdateRemsetWorker<LanguageConfig>::FillFromPostBarrierBuffer(GCG1BarrierSet::ThreadLocalCardQueues *postWrb,
147 PandaUnorderedSet<CardTable::CardPtr> *cards)
148 {
149 while (!postWrb->empty()) {
150 cards->insert(postWrb->back());
151 postWrb->pop_back();
152 }
153 }
154
155 template <class LanguageConfig>
156 class RemsetCardHandler : public CardHandler {
157 public:
RemsetCardHandler(CardTable * cardTable,size_t regionSizeBits,const std::atomic<bool> & deferCards)158 RemsetCardHandler(CardTable *cardTable, size_t regionSizeBits, const std::atomic<bool> &deferCards)
159 : CardHandler(cardTable), regionSizeBits_(regionSizeBits), deferCards_(deferCards)
160 {
161 }
162
163 protected:
HandleObject(ObjectHeader * objectHeader,void * begin,void * end)164 bool HandleObject(ObjectHeader *objectHeader, void *begin, void *end) override
165 {
166 auto objRefVisitor = [this](ObjectHeader *fromObj, ObjectHeader *toObj, uint32_t offset,
167 [[maybe_unused]] bool isVolatile) {
168 ASSERT_DO(IsHeapSpace(PoolManager::GetMmapMemPool()->GetSpaceTypeForAddr(toObj)),
169 std::cerr << "Not suitable space for to_obj: " << toObj << std::endl);
170
171 // don't need lock because only one thread changes remsets
172 RemSet<>::AddRefWithAddr<false>(fromObj, offset, toObj);
173 LOG(DEBUG, GC) << "fill rem set " << fromObj << " -> " << toObj;
174 // Atomic with relaxed order reason: memory order is not required
175 return !deferCards_.load(std::memory_order_relaxed);
176 };
177 return ObjectHelpers<LanguageConfig::LANG_TYPE>::template TraverseAllObjectsWithInfo<true>(
178 objectHeader, objRefVisitor, begin, end);
179 }
180
181 private:
182 size_t regionSizeBits_;
183 const std::atomic<bool> &deferCards_;
184 };
185
186 template <class LanguageConfig>
ProcessAllCards()187 size_t UpdateRemsetWorker<LanguageConfig>::ProcessAllCards()
188 {
189 FillFromQueue(&cards_);
190 FillFromThreads(&cards_);
191 FillFromPostBarrierBuffers(&cards_);
192 LOG_IF(!cards_.empty(), DEBUG, GC) << "Started process: " << cards_.size() << " cards";
193
194 size_t cardsSize = 0;
195 RemsetCardHandler<LanguageConfig> cardHandler(gc_->GetCardTable(), regionSizeBits_, deferCards_);
196 for (auto it = cards_.begin(); it != cards_.end();) {
197 if (!cardHandler.Handle(*it)) {
198 break;
199 }
200 cardsSize++;
201
202 it = cards_.erase(it);
203 }
204 LOG_IF(!cards_.empty(), DEBUG, GC) << "Processed " << cardsSize << " cards";
205 return cardsSize;
206 }
207
208 template <class LanguageConfig>
DrainAllCards(PandaUnorderedSet<CardTable::CardPtr> * cards)209 void UpdateRemsetWorker<LanguageConfig>::DrainAllCards(PandaUnorderedSet<CardTable::CardPtr> *cards)
210 {
211 ASSERT(IsFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD));
212 os::memory::LockHolder holder(updateRemsetLock_);
213 FillFromDefered(cards);
214 FillFromQueue(cards);
215 FillFromThreads(cards);
216 FillFromPostBarrierBuffers(cards);
217 }
218
219 template <class LanguageConfig>
GCProcessCards()220 void UpdateRemsetWorker<LanguageConfig>::GCProcessCards()
221 {
222 ASSERT(IsFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD));
223 os::memory::LockHolder holder(updateRemsetLock_);
224 ProcessAllCards();
225 }
226
227 template <class LanguageConfig>
DoInvalidateRegions(RegionVector * regions)228 void UpdateRemsetWorker<LanguageConfig>::DoInvalidateRegions(RegionVector *regions)
229 {
230 for (auto *region : *regions) {
231 // don't need lock because only one thread changes remsets
232 RemSet<>::template InvalidateRegion<false>(region);
233 }
234 }
235
236 template <class LanguageConfig>
GCInvalidateRegions(RegionVector * regions)237 void UpdateRemsetWorker<LanguageConfig>::GCInvalidateRegions(RegionVector *regions)
238 {
239 // Do invalidate region on pause in GCThread
240 ASSERT(IsFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD));
241 os::memory::LockHolder holder(updateRemsetLock_);
242 DoInvalidateRegions(regions);
243 }
244
245 template <class LanguageConfig>
InvalidateRegions(RegionVector * regions)246 void UpdateRemsetWorker<LanguageConfig>::InvalidateRegions(RegionVector *regions)
247 {
248 // Do invalidate region during concurrent sweep
249 ASSERT(IsFlag(UpdateRemsetWorkerFlags::IS_PROCESS_CARD));
250 SetFlag(UpdateRemsetWorkerFlags::IS_INVALIDATE_REGIONS);
251 // Atomic with relaxed order reason: memory order is not required
252 deferCards_.store(true, std::memory_order_relaxed);
253 // Aquare lock to be sure that UpdateRemsetWorker has been stopped
254 os::memory::LockHolder holder(updateRemsetLock_);
255 LOG(DEBUG, GC) << "Remset worker has been paused to invalidate region on concurrent";
256 DoInvalidateRegions(regions);
257 // Atomic with relaxed order reason: memory order is not required
258 deferCards_.store(false, std::memory_order_relaxed);
259 RemoveFlag(UpdateRemsetWorkerFlags::IS_INVALIDATE_REGIONS);
260 if (updateConcurrent_) {
261 this->ContinueProcessCards();
262 }
263 }
264
265 template <class LanguageConfig>
SuspendWorkerForGCPause()266 void UpdateRemsetWorker<LanguageConfig>::SuspendWorkerForGCPause()
267 {
268 ASSERT(!IsFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD));
269 SetFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD);
270 // Atomic with relaxed order reason: memory order is not required
271 deferCards_.store(true, std::memory_order_relaxed);
272 // Aquare lock to be sure that UpdateRemsetWorker has been stopped
273 os::memory::LockHolder holder(updateRemsetLock_);
274 #ifndef NDEBUG
275 pausedByGcThread_ = true;
276 #endif
277 LOG(DEBUG, GC) << "Remset worker has been paused for GC";
278 // Atomic with relaxed order reason: memory order is not required
279 deferCards_.store(false, std::memory_order_relaxed);
280 }
281
282 template <class LanguageConfig>
ResumeWorkerAfterGCPause()283 void UpdateRemsetWorker<LanguageConfig>::ResumeWorkerAfterGCPause()
284 {
285 ASSERT(IsFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD));
286 os::memory::LockHolder holder(updateRemsetLock_);
287 #ifndef NDEBUG
288 pausedByGcThread_ = false;
289 #endif
290 RemoveFlag(UpdateRemsetWorkerFlags::IS_PAUSED_BY_GC_THREAD);
291 if (updateConcurrent_) {
292 this->ContinueProcessCards();
293 }
294 }
295
296 TEMPLATE_CLASS_LANGUAGE_CONFIG(UpdateRemsetWorker);
297 } // namespace panda::mem
298