1 /*
2 * Copyright (c) 2022-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef ECMASCRIPT_MEM_REGION_INL_H
17 #define ECMASCRIPT_MEM_REGION_INL_H
18
19 #include "ecmascript/mem/region.h"
20
21 #include "ecmascript/mem/mem.h"
22 #include "ecmascript/mem/native_area_allocator.h"
23
24 namespace panda::ecmascript {
CreateRememberedSet()25 inline RememberedSet *Region::CreateRememberedSet()
26 {
27 auto bitSize = GCBitset::SizeOfGCBitset(GetCapacity());
28 auto setAddr = nativeAreaAllocator_->Allocate(bitSize + RememberedSet::GCBITSET_DATA_OFFSET);
29 auto ret = new (setAddr) RememberedSet(bitSize);
30 ret->ClearAll();
31 std::atomic_thread_fence(std::memory_order_seq_cst);
32 return ret;
33 }
34
GetOrCreateCrossRegionRememberedSet()35 inline RememberedSet *Region::GetOrCreateCrossRegionRememberedSet()
36 {
37 if (UNLIKELY(crossRegionSet_ == nullptr)) {
38 LockHolder lock(*lock_);
39 if (crossRegionSet_ == nullptr) {
40 crossRegionSet_ = CreateRememberedSet();
41 }
42 }
43 return crossRegionSet_;
44 }
45
GetOrCreateNewToEdenRememberedSet()46 inline RememberedSet *Region::GetOrCreateNewToEdenRememberedSet()
47 {
48 if (UNLIKELY(packedData_.newToEdenSet_ == nullptr)) {
49 LockHolder lock(*lock_);
50 if (packedData_.newToEdenSet_ == nullptr) {
51 packedData_.newToEdenSet_ = CreateRememberedSet();
52 }
53 }
54 return packedData_.newToEdenSet_;
55 }
56
GetOrCreateOldToNewRememberedSet()57 inline RememberedSet *Region::GetOrCreateOldToNewRememberedSet()
58 {
59 if (UNLIKELY(packedData_.oldToNewSet_ == nullptr)) {
60 LockHolder lock(*lock_);
61 if (packedData_.oldToNewSet_ == nullptr) {
62 if (sweepingOldToNewRSet_ != nullptr && IsGCFlagSet(RegionGCFlags::HAS_BEEN_SWEPT)) {
63 packedData_.oldToNewSet_ = sweepingOldToNewRSet_;
64 sweepingOldToNewRSet_ = nullptr;
65 } else {
66 packedData_.oldToNewSet_ = CreateRememberedSet();
67 }
68 }
69 }
70 return packedData_.oldToNewSet_;
71 }
72
GetOrCreateLocalToShareRememberedSet()73 inline RememberedSet *Region::GetOrCreateLocalToShareRememberedSet()
74 {
75 if (UNLIKELY(packedData_.localToShareSet_ == nullptr)) {
76 LockHolder lock(*lock_);
77 if (packedData_.localToShareSet_ == nullptr) {
78 if (sweepingLocalToShareRSet_ != nullptr && IsGCFlagSet(RegionGCFlags::HAS_BEEN_SWEPT)) {
79 packedData_.localToShareSet_ = sweepingLocalToShareRSet_;
80 sweepingLocalToShareRSet_ = nullptr;
81 } else {
82 packedData_.localToShareSet_ = CreateRememberedSet();
83 }
84 }
85 }
86 return packedData_.localToShareSet_;
87 }
88
MergeLocalToShareRSetForCS()89 inline void Region::MergeLocalToShareRSetForCS()
90 {
91 if (sweepingLocalToShareRSet_ == nullptr) {
92 return;
93 }
94 if (packedData_.localToShareSet_ == nullptr) {
95 packedData_.localToShareSet_ = sweepingLocalToShareRSet_;
96 sweepingLocalToShareRSet_ = nullptr;
97 } else {
98 packedData_.localToShareSet_->Merge(sweepingLocalToShareRSet_);
99 DeleteSweepingLocalToShareRSet();
100 sweepingLocalToShareRSet_ = nullptr;
101 }
102 }
103
MergeOldToNewRSetForCS()104 inline void Region::MergeOldToNewRSetForCS()
105 {
106 if (sweepingOldToNewRSet_ == nullptr) {
107 return;
108 }
109 if (packedData_.oldToNewSet_ == nullptr) {
110 packedData_.oldToNewSet_ = sweepingOldToNewRSet_;
111 sweepingOldToNewRSet_ = nullptr;
112 } else {
113 packedData_.oldToNewSet_->Merge(sweepingOldToNewRSet_);
114 DeleteSweepingOldToNewRSet();
115 sweepingOldToNewRSet_ = nullptr;
116 }
117 }
118
MergeLocalToShareRSetForCM(RememberedSet * set)119 inline void Region::MergeLocalToShareRSetForCM(RememberedSet *set)
120 {
121 if (packedData_.localToShareSet_ == nullptr) {
122 packedData_.localToShareSet_ = set;
123 } else {
124 packedData_.localToShareSet_->Merge(set);
125 nativeAreaAllocator_->Free(set, set->Size());
126 }
127 }
128
GetMarkGCBitset()129 inline GCBitset *Region::GetMarkGCBitset() const
130 {
131 return packedData_.markGCBitset_;
132 }
133
AtomicMark(void * address)134 inline bool Region::AtomicMark(void *address)
135 {
136 auto addrPtr = reinterpret_cast<uintptr_t>(address);
137 ASSERT(InRange(addrPtr));
138 return packedData_.markGCBitset_->SetBit<AccessType::ATOMIC>(
139 (addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
140 }
141
NonAtomicMark(void * address)142 inline bool Region::NonAtomicMark(void *address)
143 {
144 ASSERT(IsFreshRegion());
145 auto addrPtr = reinterpret_cast<uintptr_t>(address);
146 ASSERT(InRange(addrPtr));
147 return packedData_.markGCBitset_->SetBit<AccessType::NON_ATOMIC>(
148 (addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
149 }
150
ClearMark(void * address)151 inline void Region::ClearMark(void *address)
152 {
153 auto addrPtr = reinterpret_cast<uintptr_t>(address);
154 ASSERT(InRange(addrPtr));
155 packedData_.markGCBitset_->ClearBit((addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
156 }
157
Test(void * addr)158 inline bool Region::Test(void *addr) const
159 {
160 auto addrPtr = reinterpret_cast<uintptr_t>(addr);
161 return Test(addrPtr);
162 }
163
Test(uintptr_t addrPtr)164 inline bool Region::Test(uintptr_t addrPtr) const
165 {
166 ASSERT(InRange(addrPtr));
167 return packedData_.markGCBitset_->TestBit((addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
168 }
169
170 // ONLY used for heap verification.
TestNewToEden(uintptr_t addr)171 inline bool Region::TestNewToEden(uintptr_t addr)
172 {
173 ASSERT(InRange(addr));
174 // Only used for heap verification, so donot need to use lock
175 auto set = packedData_.newToEdenSet_;
176 if (set == nullptr) {
177 return false;
178 }
179 return set->TestBit(ToUintPtr(this), addr);
180 }
181
182 // ONLY used for heap verification.
TestOldToNew(uintptr_t addr)183 inline bool Region::TestOldToNew(uintptr_t addr)
184 {
185 ASSERT(InRange(addr));
186 // Only used for heap verification, so donot need to use lock
187 auto set = packedData_.oldToNewSet_;
188 if (set == nullptr) {
189 return false;
190 }
191 return set->TestBit(ToUintPtr(this), addr);
192 }
193
194 // ONLY used for heap verification.
TestLocalToShare(uintptr_t addr)195 inline bool Region::TestLocalToShare(uintptr_t addr)
196 {
197 ASSERT(InRange(addr));
198 // Only used for heap verification, so donot need to use lock
199 if (packedData_.localToShareSet_ == nullptr) {
200 return false;
201 }
202 return packedData_.localToShareSet_->TestBit(ToUintPtr(this), addr);
203 }
204
205 template <typename Visitor>
IterateAllMarkedBits(Visitor visitor)206 inline void Region::IterateAllMarkedBits(Visitor visitor) const
207 {
208 packedData_.markGCBitset_->IterateMarkedBitsConst(
209 reinterpret_cast<uintptr_t>(this), packedData_.bitsetSize_, visitor);
210 }
211
ClearMarkGCBitset()212 inline void Region::ClearMarkGCBitset()
213 {
214 if (packedData_.markGCBitset_ != nullptr) {
215 packedData_.markGCBitset_->Clear(packedData_.bitsetSize_);
216 }
217 }
218
InsertCrossRegionRSet(uintptr_t addr)219 inline void Region::InsertCrossRegionRSet(uintptr_t addr)
220 {
221 auto set = GetOrCreateCrossRegionRememberedSet();
222 set->Insert(ToUintPtr(this), addr);
223 }
224
AtomicInsertCrossRegionRSet(uintptr_t addr)225 inline void Region::AtomicInsertCrossRegionRSet(uintptr_t addr)
226 {
227 auto set = GetOrCreateCrossRegionRememberedSet();
228 set->AtomicInsert(ToUintPtr(this), addr);
229 }
230
HasLocalToShareRememberedSet()231 inline bool Region::HasLocalToShareRememberedSet() const
232 {
233 return packedData_.localToShareSet_ != nullptr;
234 }
235
ExtractLocalToShareRSet()236 inline RememberedSet *Region::ExtractLocalToShareRSet()
237 {
238 RememberedSet *set = packedData_.localToShareSet_;
239 packedData_.localToShareSet_ = nullptr;
240 return set;
241 }
242
InsertLocalToShareRSet(uintptr_t addr)243 inline void Region::InsertLocalToShareRSet(uintptr_t addr)
244 {
245 auto set = GetOrCreateLocalToShareRememberedSet();
246 set->Insert(ToUintPtr(this), addr);
247 }
248
AtomicInsertLocalToShareRSet(uintptr_t addr)249 inline void Region::AtomicInsertLocalToShareRSet(uintptr_t addr)
250 {
251 auto set = GetOrCreateLocalToShareRememberedSet();
252 set->AtomicInsert(ToUintPtr(this), addr);
253 }
254
ClearLocalToShareRSetInRange(uintptr_t start,uintptr_t end)255 inline void Region::ClearLocalToShareRSetInRange(uintptr_t start, uintptr_t end)
256 {
257 if (packedData_.localToShareSet_ != nullptr) {
258 packedData_.localToShareSet_->ClearRange(ToUintPtr(this), start, end);
259 }
260 }
261
AtomicClearLocalToShareRSetInRange(uintptr_t start,uintptr_t end)262 inline void Region::AtomicClearLocalToShareRSetInRange(uintptr_t start, uintptr_t end)
263 {
264 if (packedData_.localToShareSet_ != nullptr) {
265 packedData_.localToShareSet_->AtomicClearRange(ToUintPtr(this), start, end);
266 }
267 }
268
DeleteLocalToShareRSet()269 inline void Region::DeleteLocalToShareRSet()
270 {
271 if (packedData_.localToShareSet_ != nullptr) {
272 nativeAreaAllocator_->Free(packedData_.localToShareSet_, packedData_.localToShareSet_->Size());
273 packedData_.localToShareSet_ = nullptr;
274 }
275 }
276
AtomicClearSweepingLocalToShareRSetInRange(uintptr_t start,uintptr_t end)277 inline void Region::AtomicClearSweepingLocalToShareRSetInRange(uintptr_t start, uintptr_t end)
278 {
279 if (sweepingLocalToShareRSet_ != nullptr) {
280 sweepingLocalToShareRSet_->AtomicClearRange(ToUintPtr(this), start, end);
281 }
282 }
283
DeleteSweepingLocalToShareRSet()284 inline void Region::DeleteSweepingLocalToShareRSet()
285 {
286 if (sweepingLocalToShareRSet_!= nullptr) {
287 nativeAreaAllocator_->Free(sweepingLocalToShareRSet_, sweepingLocalToShareRSet_->Size());
288 sweepingLocalToShareRSet_ = nullptr;
289 }
290 }
291
292 template <typename Visitor>
IterateAllLocalToShareBits(Visitor visitor)293 inline void Region::IterateAllLocalToShareBits(Visitor visitor)
294 {
295 if (packedData_.localToShareSet_ != nullptr) {
296 packedData_.localToShareSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
297 }
298 }
299
300 template <typename Visitor>
IterateAllCrossRegionBits(Visitor visitor)301 inline void Region::IterateAllCrossRegionBits(Visitor visitor) const
302 {
303 if (crossRegionSet_ != nullptr) {
304 crossRegionSet_->IterateAllMarkedBitsConst(ToUintPtr(this), visitor);
305 }
306 }
307
ClearCrossRegionRSet()308 inline void Region::ClearCrossRegionRSet()
309 {
310 if (crossRegionSet_ != nullptr) {
311 crossRegionSet_->ClearAll();
312 }
313 }
314
ClearCrossRegionRSetInRange(uintptr_t start,uintptr_t end)315 inline void Region::ClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end)
316 {
317 if (crossRegionSet_ != nullptr) {
318 crossRegionSet_->ClearRange(ToUintPtr(this), start, end);
319 }
320 }
321
AtomicClearCrossRegionRSetInRange(uintptr_t start,uintptr_t end)322 inline void Region::AtomicClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end)
323 {
324 if (crossRegionSet_ != nullptr) {
325 crossRegionSet_->AtomicClearRange(ToUintPtr(this), start, end);
326 }
327 }
328
DeleteCrossRegionRSet()329 inline void Region::DeleteCrossRegionRSet()
330 {
331 if (crossRegionSet_ != nullptr) {
332 nativeAreaAllocator_->Free(crossRegionSet_, crossRegionSet_->Size());
333 crossRegionSet_ = nullptr;
334 }
335 }
336
InsertNewToEdenRSet(uintptr_t addr)337 inline void Region::InsertNewToEdenRSet(uintptr_t addr)
338 {
339 auto set = GetOrCreateNewToEdenRememberedSet();
340 set->Insert(ToUintPtr(this), addr);
341 }
342
AtomicInsertNewToEdenRSet(uintptr_t addr)343 inline void Region::AtomicInsertNewToEdenRSet(uintptr_t addr)
344 {
345 auto set = GetOrCreateNewToEdenRememberedSet();
346 set->AtomicInsert(ToUintPtr(this), addr);
347 }
348
ClearNewToEdenRSet(uintptr_t addr)349 inline void Region::ClearNewToEdenRSet(uintptr_t addr)
350 {
351 auto set = GetOrCreateNewToEdenRememberedSet();
352 set->ClearBit(ToUintPtr(this), addr);
353 }
354
InsertOldToNewRSet(uintptr_t addr)355 inline void Region::InsertOldToNewRSet(uintptr_t addr)
356 {
357 auto set = GetOrCreateOldToNewRememberedSet();
358 set->Insert(ToUintPtr(this), addr);
359 }
360
ClearOldToNewRSet(uintptr_t addr)361 inline void Region::ClearOldToNewRSet(uintptr_t addr)
362 {
363 auto set = GetOrCreateOldToNewRememberedSet();
364 set->ClearBit(ToUintPtr(this), addr);
365 }
366
367 template <typename Visitor>
IterateAllNewToEdenBits(Visitor visitor)368 inline void Region::IterateAllNewToEdenBits(Visitor visitor)
369 {
370 if (packedData_.newToEdenSet_ != nullptr) {
371 packedData_.newToEdenSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
372 }
373 }
374
375 template <typename Visitor>
IterateAllOldToNewBits(Visitor visitor)376 inline void Region::IterateAllOldToNewBits(Visitor visitor)
377 {
378 if (packedData_.oldToNewSet_ != nullptr) {
379 packedData_.oldToNewSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
380 }
381 }
382
383 template <typename Visitor>
AtomicIterateAllSweepingRSetBits(Visitor visitor)384 inline void Region::AtomicIterateAllSweepingRSetBits(Visitor visitor)
385 {
386 if (sweepingOldToNewRSet_ != nullptr) {
387 sweepingOldToNewRSet_->AtomicIterateAllMarkedBits(ToUintPtr(this), visitor);
388 }
389 }
390
391 template <typename Visitor>
IterateAllSweepingRSetBits(Visitor visitor)392 inline void Region::IterateAllSweepingRSetBits(Visitor visitor)
393 {
394 if (sweepingOldToNewRSet_ != nullptr) {
395 sweepingOldToNewRSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
396 }
397 }
398
GetNewToEdenRSet()399 inline RememberedSet *Region::GetNewToEdenRSet()
400 {
401 return packedData_.newToEdenSet_;
402 }
403
ClearNewToEdenRSet()404 inline void Region::ClearNewToEdenRSet()
405 {
406 if (packedData_.newToEdenSet_ != nullptr) {
407 packedData_.newToEdenSet_->ClearAll();
408 }
409 }
410
ClearNewToEdenRSetInRange(uintptr_t start,uintptr_t end)411 inline void Region::ClearNewToEdenRSetInRange(uintptr_t start, uintptr_t end)
412 {
413 if (packedData_.newToEdenSet_ != nullptr) {
414 packedData_.newToEdenSet_->ClearRange(ToUintPtr(this), start, end);
415 }
416 }
417
DeleteNewToEdenRSet()418 inline void Region::DeleteNewToEdenRSet()
419 {
420 if (packedData_.newToEdenSet_ != nullptr) {
421 nativeAreaAllocator_->Free(packedData_.newToEdenSet_, packedData_.newToEdenSet_->Size());
422 packedData_.newToEdenSet_ = nullptr;
423 }
424 }
425
ClearOldToNewRSet()426 inline void Region::ClearOldToNewRSet()
427 {
428 if (packedData_.oldToNewSet_ != nullptr) {
429 packedData_.oldToNewSet_->ClearAll();
430 }
431 }
432
ClearOldToNewRSetInRange(uintptr_t start,uintptr_t end)433 inline void Region::ClearOldToNewRSetInRange(uintptr_t start, uintptr_t end)
434 {
435 if (packedData_.oldToNewSet_ != nullptr) {
436 packedData_.oldToNewSet_->ClearRange(ToUintPtr(this), start, end);
437 }
438 }
439
DeleteOldToNewRSet()440 inline void Region::DeleteOldToNewRSet()
441 {
442 if (packedData_.oldToNewSet_ != nullptr) {
443 nativeAreaAllocator_->Free(packedData_.oldToNewSet_, packedData_.oldToNewSet_->Size());
444 packedData_.oldToNewSet_ = nullptr;
445 }
446 }
447
AtomicClearSweepingOldToNewRSetInRange(uintptr_t start,uintptr_t end)448 inline void Region::AtomicClearSweepingOldToNewRSetInRange(uintptr_t start, uintptr_t end)
449 {
450 if (sweepingOldToNewRSet_ != nullptr) {
451 sweepingOldToNewRSet_->AtomicClearRange(ToUintPtr(this), start, end);
452 }
453 }
454
ClearSweepingOldToNewRSetInRange(uintptr_t start,uintptr_t end)455 inline void Region::ClearSweepingOldToNewRSetInRange(uintptr_t start, uintptr_t end)
456 {
457 if (sweepingOldToNewRSet_ != nullptr) {
458 sweepingOldToNewRSet_->ClearRange(ToUintPtr(this), start, end);
459 }
460 }
461
DeleteSweepingOldToNewRSet()462 inline void Region::DeleteSweepingOldToNewRSet()
463 {
464 if (sweepingOldToNewRSet_ != nullptr) {
465 nativeAreaAllocator_->Free(sweepingOldToNewRSet_, sweepingOldToNewRSet_->Size());
466 sweepingOldToNewRSet_ = nullptr;
467 }
468 }
469
GetRegionSpaceFlag()470 inline uint8_t Region::GetRegionSpaceFlag()
471 {
472 return packedData_.flags_.spaceFlag_;
473 }
474 } // namespace panda::ecmascript
475 #endif // ECMASCRIPT_MEM_REGION_INL_H
476