1 /*
2 * Copyright (c) 2022-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef ECMASCRIPT_MEM_REGION_INL_H
17 #define ECMASCRIPT_MEM_REGION_INL_H
18
19 #include "ecmascript/mem/region.h"
20
21 #include "ecmascript/mem/mem.h"
22 #include "ecmascript/mem/native_area_allocator.h"
23
24 namespace panda::ecmascript {
CreateRememberedSet()25 inline RememberedSet *Region::CreateRememberedSet()
26 {
27 auto bitSize = GCBitset::SizeOfGCBitset(GetCapacity());
28 auto setAddr = nativeAreaAllocator_->Allocate(bitSize + RememberedSet::GCBITSET_DATA_OFFSET);
29 auto ret = new (setAddr) RememberedSet(bitSize);
30 ret->ClearAll();
31 std::atomic_thread_fence(std::memory_order_seq_cst);
32 return ret;
33 }
34
GetOrCreateCrossRegionRememberedSet()35 inline RememberedSet *Region::GetOrCreateCrossRegionRememberedSet()
36 {
37 if (UNLIKELY(crossRegionSet_ == nullptr)) {
38 LockHolder lock(*lock_);
39 if (crossRegionSet_ == nullptr) {
40 crossRegionSet_ = CreateRememberedSet();
41 }
42 }
43 return crossRegionSet_;
44 }
45
CreateOldToNewRememberedSet()46 ARK_NOINLINE RememberedSet* Region::CreateOldToNewRememberedSet()
47 {
48 LockHolder lock(*lock_);
49 if (packedData_.oldToNewSet_ == nullptr) {
50 if (sweepingOldToNewRSet_ != nullptr && IsGCFlagSet(RegionGCFlags::HAS_BEEN_SWEPT)) {
51 packedData_.oldToNewSet_ = sweepingOldToNewRSet_;
52 ClearRSetSwapFlag(RSetSwapFlag::OLD_TO_NEW_SWAPPED_MASK);
53 sweepingOldToNewRSet_ = nullptr;
54 } else {
55 packedData_.oldToNewSet_ = CreateRememberedSet();
56 }
57 }
58 return packedData_.oldToNewSet_;
59 }
60
GetOrCreateOldToNewRememberedSet()61 inline RememberedSet* Region::GetOrCreateOldToNewRememberedSet()
62 {
63 if (UNLIKELY(packedData_.oldToNewSet_ == nullptr)) {
64 return CreateOldToNewRememberedSet();
65 }
66 return packedData_.oldToNewSet_;
67 }
68
CreateLocalToShareRememberedSet()69 ARK_NOINLINE RememberedSet* Region::CreateLocalToShareRememberedSet()
70 {
71 LockHolder lock(*lock_);
72 if (packedData_.localToShareSet_ == nullptr) {
73 if (sweepingLocalToShareRSet_ != nullptr && IsGCFlagSet(RegionGCFlags::HAS_BEEN_SWEPT)) {
74 packedData_.localToShareSet_ = sweepingLocalToShareRSet_;
75 ClearRSetSwapFlag(RSetSwapFlag::LOCAL_TO_SHARE_SWAPPED_MASK);
76 sweepingLocalToShareRSet_ = nullptr;
77 } else {
78 packedData_.localToShareSet_ = CreateRememberedSet();
79 }
80 }
81 return packedData_.localToShareSet_;
82 }
83
GetOrCreateLocalToShareRememberedSet()84 inline RememberedSet *Region::GetOrCreateLocalToShareRememberedSet()
85 {
86 if (UNLIKELY(packedData_.localToShareSet_ == nullptr)) {
87 return CreateLocalToShareRememberedSet();
88 }
89 return packedData_.localToShareSet_;
90 }
91
MergeLocalToShareRSetForCS()92 inline void Region::MergeLocalToShareRSetForCS()
93 {
94 if (sweepingLocalToShareRSet_ == nullptr) {
95 return;
96 }
97 if (packedData_.localToShareSet_ == nullptr) {
98 packedData_.localToShareSet_ = sweepingLocalToShareRSet_;
99 sweepingLocalToShareRSet_ = nullptr;
100 } else {
101 packedData_.localToShareSet_->Merge(sweepingLocalToShareRSet_);
102 DeleteSweepingLocalToShareRSet();
103 sweepingLocalToShareRSet_ = nullptr;
104 }
105 ClearRSetSwapFlag(RSetSwapFlag::LOCAL_TO_SHARE_SWAPPED_MASK);
106 }
107
MergeOldToNewRSetForCS()108 inline void Region::MergeOldToNewRSetForCS()
109 {
110 if (sweepingOldToNewRSet_ == nullptr) {
111 return;
112 }
113 if (packedData_.oldToNewSet_ == nullptr) {
114 packedData_.oldToNewSet_ = sweepingOldToNewRSet_;
115 sweepingOldToNewRSet_ = nullptr;
116 } else {
117 packedData_.oldToNewSet_->Merge(sweepingOldToNewRSet_);
118 DeleteSweepingOldToNewRSet();
119 sweepingOldToNewRSet_ = nullptr;
120 }
121 ClearRSetSwapFlag(RSetSwapFlag::OLD_TO_NEW_SWAPPED_MASK);
122 }
123
MergeLocalToShareRSetForCM(RememberedSet * set)124 inline void Region::MergeLocalToShareRSetForCM(RememberedSet *set)
125 {
126 if (packedData_.localToShareSet_ == nullptr) {
127 packedData_.localToShareSet_ = set;
128 } else {
129 packedData_.localToShareSet_->Merge(set);
130 nativeAreaAllocator_->Free(set, set->Size());
131 }
132 ClearRSetSwapFlag(RSetSwapFlag::LOCAL_TO_SHARE_COLLECTED_MASK);
133 }
134
GetMarkGCBitset()135 inline GCBitset *Region::GetMarkGCBitset() const
136 {
137 return packedData_.markGCBitset_;
138 }
139
AtomicMark(void * address)140 inline bool Region::AtomicMark(void *address)
141 {
142 auto addrPtr = reinterpret_cast<uintptr_t>(address);
143 ASSERT(InRange(addrPtr));
144 return packedData_.markGCBitset_->SetBit<AccessType::ATOMIC>(
145 (addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
146 }
147
NonAtomicMark(void * address)148 inline bool Region::NonAtomicMark(void *address)
149 {
150 auto addrPtr = reinterpret_cast<uintptr_t>(address);
151 ASSERT(InRange(addrPtr));
152 return packedData_.markGCBitset_->SetBit<AccessType::NON_ATOMIC>(
153 (addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
154 }
155
ClearMark(void * address)156 inline void Region::ClearMark(void *address)
157 {
158 auto addrPtr = reinterpret_cast<uintptr_t>(address);
159 ASSERT(InRange(addrPtr));
160 packedData_.markGCBitset_->ClearBit((addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
161 }
162
Test(void * addr)163 inline bool Region::Test(void *addr) const
164 {
165 auto addrPtr = reinterpret_cast<uintptr_t>(addr);
166 return Test(addrPtr);
167 }
168
Test(uintptr_t addrPtr)169 inline bool Region::Test(uintptr_t addrPtr) const
170 {
171 ASSERT(InRange(addrPtr));
172 return packedData_.markGCBitset_->TestBit((addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
173 }
174
175 // ONLY used for heap verification.
TestOldToNew(uintptr_t addr)176 inline bool Region::TestOldToNew(uintptr_t addr)
177 {
178 ASSERT(InRange(addr));
179 // Only used for heap verification, so donot need to use lock
180 auto set = packedData_.oldToNewSet_;
181 if (set == nullptr) {
182 return false;
183 }
184 return set->TestBit(ToUintPtr(this), addr);
185 }
186
187 // ONLY used for heap verification.
TestLocalToShare(uintptr_t addr)188 inline bool Region::TestLocalToShare(uintptr_t addr)
189 {
190 ASSERT(InRange(addr));
191 // Only used for heap verification, so donot need to use lock
192 if (packedData_.localToShareSet_ == nullptr) {
193 return false;
194 }
195 return packedData_.localToShareSet_->TestBit(ToUintPtr(this), addr);
196 }
197
198 template <typename Visitor>
IterateAllMarkedBits(Visitor && visitor)199 inline void Region::IterateAllMarkedBits(Visitor &&visitor) const
200 {
201 packedData_.markGCBitset_->IterateMarkedBitsConst(
202 reinterpret_cast<uintptr_t>(this), packedData_.bitsetSize_, visitor);
203 }
204
ClearMarkGCBitset()205 inline void Region::ClearMarkGCBitset()
206 {
207 if (packedData_.markGCBitset_ != nullptr) {
208 packedData_.markGCBitset_->Clear(packedData_.bitsetSize_);
209 }
210 }
211
InsertCrossRegionRSet(uintptr_t addr)212 inline void Region::InsertCrossRegionRSet(uintptr_t addr)
213 {
214 auto set = GetOrCreateCrossRegionRememberedSet();
215 set->Insert(ToUintPtr(this), addr);
216 }
217
AtomicInsertCrossRegionRSet(uintptr_t addr)218 inline void Region::AtomicInsertCrossRegionRSet(uintptr_t addr)
219 {
220 auto set = GetOrCreateCrossRegionRememberedSet();
221 set->AtomicInsert(ToUintPtr(this), addr);
222 }
223
HasLocalToShareRememberedSet()224 inline bool Region::HasLocalToShareRememberedSet() const
225 {
226 return packedData_.localToShareSet_ != nullptr;
227 }
228
CollectLocalToShareRSet()229 inline RememberedSet *Region::CollectLocalToShareRSet()
230 {
231 RememberedSet *set = packedData_.localToShareSet_;
232 packedData_.localToShareSet_ = nullptr;
233 if (set != nullptr) {
234 SetRSetSwapFlag(RSetSwapFlag::LOCAL_TO_SHARE_COLLECTED_MASK);
235 }
236 return set;
237 }
238
InsertLocalToShareRSet(uintptr_t addr)239 inline void Region::InsertLocalToShareRSet(uintptr_t addr)
240 {
241 auto set = GetOrCreateLocalToShareRememberedSet();
242 set->Insert(ToUintPtr(this), addr);
243 }
244
245 template <Region::RegionSpaceKind kind>
GetBatchRSetUpdater(uintptr_t addr)246 Region::Updater<kind> Region::GetBatchRSetUpdater(uintptr_t addr)
247 {
248 return Region::Updater<kind>(addr, *this);
249 }
250
AtomicInsertLocalToShareRSet(uintptr_t addr)251 inline void Region::AtomicInsertLocalToShareRSet(uintptr_t addr)
252 {
253 auto set = GetOrCreateLocalToShareRememberedSet();
254 set->AtomicInsert(ToUintPtr(this), addr);
255 }
256
ClearLocalToShareRSetInRange(uintptr_t start,uintptr_t end)257 inline void Region::ClearLocalToShareRSetInRange(uintptr_t start, uintptr_t end)
258 {
259 if (packedData_.localToShareSet_ != nullptr) {
260 packedData_.localToShareSet_->ClearRange(ToUintPtr(this), start, end);
261 }
262 }
263
AtomicClearLocalToShareRSetInRange(uintptr_t start,uintptr_t end)264 inline void Region::AtomicClearLocalToShareRSetInRange(uintptr_t start, uintptr_t end)
265 {
266 if (packedData_.localToShareSet_ != nullptr) {
267 packedData_.localToShareSet_->AtomicClearRange(ToUintPtr(this), start, end);
268 }
269 }
270
DeleteLocalToShareRSet()271 inline void Region::DeleteLocalToShareRSet()
272 {
273 if (packedData_.localToShareSet_ != nullptr) {
274 nativeAreaAllocator_->Free(packedData_.localToShareSet_, packedData_.localToShareSet_->Size());
275 packedData_.localToShareSet_ = nullptr;
276 }
277 }
278
AtomicClearSweepingLocalToShareRSetInRange(uintptr_t start,uintptr_t end)279 inline void Region::AtomicClearSweepingLocalToShareRSetInRange(uintptr_t start, uintptr_t end)
280 {
281 if (sweepingLocalToShareRSet_ != nullptr) {
282 sweepingLocalToShareRSet_->AtomicClearRange(ToUintPtr(this), start, end);
283 }
284 }
285
DeleteSweepingLocalToShareRSet()286 inline void Region::DeleteSweepingLocalToShareRSet()
287 {
288 if (sweepingLocalToShareRSet_!= nullptr) {
289 nativeAreaAllocator_->Free(sweepingLocalToShareRSet_, sweepingLocalToShareRSet_->Size());
290 sweepingLocalToShareRSet_ = nullptr;
291 }
292 }
293
294 template <typename Visitor>
IterateAllLocalToShareBits(Visitor visitor)295 inline void Region::IterateAllLocalToShareBits(Visitor visitor)
296 {
297 if (packedData_.localToShareSet_ != nullptr) {
298 packedData_.localToShareSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
299 }
300 }
301
302 template <typename Visitor>
IterateAllCrossRegionBits(Visitor visitor)303 inline void Region::IterateAllCrossRegionBits(Visitor visitor) const
304 {
305 if (crossRegionSet_ != nullptr) {
306 crossRegionSet_->IterateAllMarkedBitsConst(ToUintPtr(this), visitor);
307 }
308 }
309
ClearCrossRegionRSet()310 inline void Region::ClearCrossRegionRSet()
311 {
312 if (crossRegionSet_ != nullptr) {
313 crossRegionSet_->ClearAll();
314 }
315 }
316
ClearCrossRegionRSetInRange(uintptr_t start,uintptr_t end)317 inline void Region::ClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end)
318 {
319 if (crossRegionSet_ != nullptr) {
320 crossRegionSet_->ClearRange(ToUintPtr(this), start, end);
321 }
322 }
323
AtomicClearCrossRegionRSetInRange(uintptr_t start,uintptr_t end)324 inline void Region::AtomicClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end)
325 {
326 if (crossRegionSet_ != nullptr) {
327 crossRegionSet_->AtomicClearRange(ToUintPtr(this), start, end);
328 }
329 }
330
DeleteCrossRegionRSet()331 inline void Region::DeleteCrossRegionRSet()
332 {
333 if (crossRegionSet_ != nullptr) {
334 nativeAreaAllocator_->Free(crossRegionSet_, crossRegionSet_->Size());
335 crossRegionSet_ = nullptr;
336 }
337 }
338
InsertOldToNewRSet(uintptr_t addr)339 inline void Region::InsertOldToNewRSet(uintptr_t addr)
340 {
341 auto set = GetOrCreateOldToNewRememberedSet();
342 set->Insert(ToUintPtr(this), addr);
343 }
344
ClearOldToNewRSet(uintptr_t addr)345 inline void Region::ClearOldToNewRSet(uintptr_t addr)
346 {
347 auto set = GetOrCreateOldToNewRememberedSet();
348 set->ClearBit(ToUintPtr(this), addr);
349 }
350
351 template <typename Visitor>
IterateAllOldToNewBits(Visitor visitor)352 inline void Region::IterateAllOldToNewBits(Visitor visitor)
353 {
354 if (packedData_.oldToNewSet_ != nullptr) {
355 packedData_.oldToNewSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
356 }
357 }
358
359 template <typename Visitor>
AtomicIterateAllSweepingRSetBits(Visitor visitor)360 inline void Region::AtomicIterateAllSweepingRSetBits(Visitor visitor)
361 {
362 if (sweepingOldToNewRSet_ != nullptr) {
363 sweepingOldToNewRSet_->AtomicIterateAllMarkedBits(ToUintPtr(this), visitor);
364 }
365 }
366
367 template <typename Visitor>
IterateAllSweepingRSetBits(Visitor visitor)368 inline void Region::IterateAllSweepingRSetBits(Visitor visitor)
369 {
370 if (sweepingOldToNewRSet_ != nullptr) {
371 sweepingOldToNewRSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
372 }
373 }
374
ClearOldToNewRSet()375 inline void Region::ClearOldToNewRSet()
376 {
377 if (packedData_.oldToNewSet_ != nullptr) {
378 packedData_.oldToNewSet_->ClearAll();
379 }
380 }
381
ClearOldToNewRSetInRange(uintptr_t start,uintptr_t end)382 inline void Region::ClearOldToNewRSetInRange(uintptr_t start, uintptr_t end)
383 {
384 if (packedData_.oldToNewSet_ != nullptr) {
385 packedData_.oldToNewSet_->ClearRange(ToUintPtr(this), start, end);
386 }
387 }
388
DeleteOldToNewRSet()389 inline void Region::DeleteOldToNewRSet()
390 {
391 if (packedData_.oldToNewSet_ != nullptr) {
392 nativeAreaAllocator_->Free(packedData_.oldToNewSet_, packedData_.oldToNewSet_->Size());
393 packedData_.oldToNewSet_ = nullptr;
394 }
395 }
396
AtomicClearSweepingOldToNewRSetInRange(uintptr_t start,uintptr_t end)397 inline void Region::AtomicClearSweepingOldToNewRSetInRange(uintptr_t start, uintptr_t end)
398 {
399 if (sweepingOldToNewRSet_ != nullptr) {
400 sweepingOldToNewRSet_->AtomicClearRange(ToUintPtr(this), start, end);
401 }
402 }
403
ClearSweepingOldToNewRSetInRange(uintptr_t start,uintptr_t end)404 inline void Region::ClearSweepingOldToNewRSetInRange(uintptr_t start, uintptr_t end)
405 {
406 if (sweepingOldToNewRSet_ != nullptr) {
407 sweepingOldToNewRSet_->ClearRange(ToUintPtr(this), start, end);
408 }
409 }
410
DeleteSweepingOldToNewRSet()411 inline void Region::DeleteSweepingOldToNewRSet()
412 {
413 if (sweepingOldToNewRSet_ != nullptr) {
414 nativeAreaAllocator_->Free(sweepingOldToNewRSet_, sweepingOldToNewRSet_->Size());
415 sweepingOldToNewRSet_ = nullptr;
416 }
417 }
418
GetRegionSpaceFlag()419 inline uint8_t Region::GetRegionSpaceFlag()
420 {
421 return packedData_.flags_.spaceFlag_;
422 }
423
SetRSetSwapFlag(RSetSwapFlag mask)424 inline void Region::SetRSetSwapFlag(RSetSwapFlag mask)
425 {
426 ASSERT(((packedData_.RSetSwapFlag_ & static_cast<uint8_t>(mask)) == 0) && "RSetSwapFlag should be 0");
427 packedData_.RSetSwapFlag_ |= static_cast<uint8_t>(mask);
428 }
429
ClearRSetSwapFlag(RSetSwapFlag mask)430 inline void Region::ClearRSetSwapFlag(RSetSwapFlag mask)
431 {
432 ASSERT(((packedData_.RSetSwapFlag_ & static_cast<uint8_t>(mask)) != 0) && "RSetSwapFlag should not be 0");
433 packedData_.RSetSwapFlag_ &= ~static_cast<uint8_t>(mask);
434 }
435
436 template <Region::RegionSpaceKind kind>
Flush()437 ARK_INLINE void Region::Updater<kind>::Flush()
438 {
439 uintptr_t updateAddress = 0;
440 std::array<std::bitset<GCBitset::BIT_PER_WORD>, BitSetNum> bitsets = bitsetUpdater_.GetAndResetAll(updateAddress);
441 for (size_t idx = 0; idx < BitSetNum; idx++) {
442 if (bitsets[idx].none()) {
443 continue;
444 }
445 Consume(idx, updateAddress, static_cast<uint32_t>(bitsets[idx].to_ulong()));
446 }
447 }
448
449 template <Region::RegionSpaceKind kind>
Consume(size_t idx,uintptr_t updateAddress,uint32_t mask)450 ARK_INLINE void Region::Updater<kind>::Consume(size_t idx, uintptr_t updateAddress, uint32_t mask)
451 {
452 if (idx == LocalToShareIdx) {
453 auto set = region_.GetOrCreateLocalToShareRememberedSet();
454 set->InsertRange(ToUintPtr(®ion_), updateAddress, mask);
455 }
456 if (kind == InGeneralOld && idx == OldToNewIdx) {
457 auto set = region_.GetOrCreateOldToNewRememberedSet();
458 set->InsertRange(ToUintPtr(®ion_), updateAddress, mask);
459 }
460 }
461
462 } // namespace panda::ecmascript
463 #endif // ECMASCRIPT_MEM_REGION_INL_H
464