1 /*
2 * Copyright (c) 2022-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef ECMASCRIPT_MEM_REGION_H
17 #define ECMASCRIPT_MEM_REGION_H
18
19 #include "ecmascript/base/aligned_struct.h"
20 #include "ecmascript/base/asan_interface.h"
21 #include "ecmascript/js_tagged_value.h"
22 #include "ecmascript/mem/free_object_list.h"
23 #include "ecmascript/mem/gc_bitset.h"
24 #include "ecmascript/mem/remembered_set.h"
25 #include "ecmascript/mem/mem_common.h"
26 #include "ecmascript/platform/map.h"
27
28 #include "ecmascript/platform/mutex.h"
29
30 #include "securec.h"
31
32 namespace panda {
33 namespace ecmascript {
34 class JSThread;
35
36 enum RegionSpaceFlag {
37 UNINITIALIZED = 0,
38 // We should avoid using the lower 3 bits (bits 0 to 2).
39 // If ZAP_MEM is enabled, the value of the lower 3 bits conflicts with the INVALID_VALUE.
40
41 // Bits 3 to 7 are reserved to denote the space where the region is located.
42 IN_EDEN_SPACE = 0x08,
43 IN_YOUNG_SPACE = 0x09,
44 IN_SNAPSHOT_SPACE = 0x0A,
45 IN_HUGE_OBJECT_SPACE = 0x0B,
46 IN_OLD_SPACE = 0x0C,
47 IN_NON_MOVABLE_SPACE = 0x0D,
48 IN_MACHINE_CODE_SPACE = 0x0E,
49 IN_READ_ONLY_SPACE = 0X0F,
50 IN_APPSPAWN_SPACE = 0x10,
51 IN_HUGE_MACHINE_CODE_SPACE = 0x11,
52 IN_SHARED_NON_MOVABLE = 0x12,
53 IN_SHARED_OLD_SPACE = 0x13,
54 IN_SHARED_APPSPAWN_SPACE = 0X14,
55 IN_SHARED_HUGE_OBJECT_SPACE = 0x15,
56 IN_SHARED_READ_ONLY_SPACE = 0x16,
57
58 VALID_SPACE_MASK = 0xFF,
59
60 GENERAL_YOUNG_BEGIN = IN_EDEN_SPACE,
61 GENERAL_YOUNG_END = IN_YOUNG_SPACE,
62 GENERAL_OLD_BEGIN = IN_SNAPSHOT_SPACE,
63 GENERAL_OLD_END = IN_HUGE_MACHINE_CODE_SPACE,
64 SHARED_SPACE_BEGIN = IN_SHARED_NON_MOVABLE,
65 SHARED_SPACE_END = IN_SHARED_READ_ONLY_SPACE,
66 SHARED_SWEEPABLE_SPACE_BEGIN = IN_SHARED_NON_MOVABLE,
67 SHARED_SWEEPABLE_SPACE_END = IN_SHARED_HUGE_OBJECT_SPACE,
68
69 HEAP_SPACE_BEGIN = IN_EDEN_SPACE,
70 HEAP_SPACE_END = IN_SHARED_READ_ONLY_SPACE
71 };
72
73 enum RegionGCFlags {
74 // We should avoid using the lower 3 bits (bits 0 to 2).
75 // If ZAP_MEM is enabled, the value of the lower 3 bits conflicts with the INVALID_VALUE.
76
77 // Below flags are used for GC, and each flag has a dedicated bit starting from the 3rd bit.
78 NEVER_EVACUATE = 1 << 3,
79 HAS_AGE_MARK = 1 << 4,
80 BELOW_AGE_MARK = 1 << 5,
81 IN_COLLECT_SET = 1 << 6,
82 IN_NEW_TO_NEW_SET = 1 << 7,
83 // Bits 8 to 10 (the lower 3 bits for the next byte) are also excluded for the sake of
84 // INVALID_VALUE in ZAP_MEM.
85 HAS_BEEN_SWEPT = 1 << 11,
86 NEED_RELOCATE = 1 << 12,
87 // ONLY used for heap verification.
88 IN_INACTIVE_SEMI_SPACE = 1 << 13,
89 };
90
91 // Currently only use for region in LinearSpace, to check if the region is allocated during concurrent marking.
92 enum class RegionTypeFlag : uint8_t {
93 DEFAULT = 0,
94 // We should avoid using the lower 3 bits (bits 0 to 2).
95 // If ZAP_MEM is enabled, the value of the lower 3 bits conflicts with the INVALID_VALUE.
96
97 // Region is allocated before concurrent marking, but some new object may be allocated here
98 // during concurrent marking.
99 HALF_FRESH = 0x08,
100 // Region is allocated during concurrent marking.
101 FRESH = 0x09,
102 };
103
104 enum RSetType {
105 OLD_TO_NEW,
106 LOCAL_TO_SHARE,
107 };
108
ToSpaceTypeName(uint8_t space)109 static inline std::string ToSpaceTypeName(uint8_t space)
110 {
111 switch (space) {
112 case RegionSpaceFlag::IN_EDEN_SPACE:
113 return "eden space";
114 case RegionSpaceFlag::IN_YOUNG_SPACE:
115 return "young space";
116 case RegionSpaceFlag::IN_SNAPSHOT_SPACE:
117 return "snapshot space";
118 case RegionSpaceFlag::IN_HUGE_OBJECT_SPACE:
119 return "huge object space";
120 case RegionSpaceFlag::IN_OLD_SPACE:
121 return "old space";
122 case RegionSpaceFlag::IN_NON_MOVABLE_SPACE:
123 return "non movable space";
124 case RegionSpaceFlag::IN_MACHINE_CODE_SPACE:
125 return "machine code space";
126 case RegionSpaceFlag::IN_READ_ONLY_SPACE:
127 return "read only space";
128 case RegionSpaceFlag::IN_APPSPAWN_SPACE:
129 return "appspawn space";
130 case RegionSpaceFlag::IN_HUGE_MACHINE_CODE_SPACE:
131 return "huge machine code space";
132 case RegionSpaceFlag::IN_SHARED_NON_MOVABLE:
133 return "shared non movable space";
134 case RegionSpaceFlag::IN_SHARED_OLD_SPACE:
135 return "shared old space";
136 case RegionSpaceFlag::IN_SHARED_READ_ONLY_SPACE:
137 return "shared read only space";
138 case RegionSpaceFlag::IN_SHARED_HUGE_OBJECT_SPACE:
139 return "shared huge object space";
140 case RegionSpaceFlag::IN_SHARED_APPSPAWN_SPACE:
141 return "shared appspawn space";
142 default:
143 return "invalid space";
144 }
145 }
146
147 // |---------------------------------------------------------------------------------------|
148 // | Region (256 kb) |
149 // |---------------------------------|--------------------------------|--------------------|
150 // | Head (sizeof(Region)) | Mark bitset (4kb) | Data |
151 // |---------------------------------|--------------------------------|--------------------|
152
153 class Region {
154 public:
Region(NativeAreaAllocator * allocator,uintptr_t allocateBase,uintptr_t begin,uintptr_t end,RegionSpaceFlag spaceType,RegionTypeFlag typeFlag)155 Region(NativeAreaAllocator *allocator, uintptr_t allocateBase, uintptr_t begin, uintptr_t end,
156 RegionSpaceFlag spaceType, RegionTypeFlag typeFlag)
157 : packedData_(begin, end, spaceType, typeFlag),
158 nativeAreaAllocator_(allocator),
159 allocateBase_(allocateBase),
160 end_(end),
161 highWaterMark_(end),
162 aliveObject_(0),
163 wasted_(0),
164 snapshotData_(0) {}
165
166 // JitFort space is divided into regions (JitForRegion) to enable
167 // reusing free_object_list and free_object_set operations for
168 // JitFort space, and GC marking actually happens in corresponding
169 // MachineCode objects where JitFort space is allocated to. So no
170 // gc mark bits needed in JitFortRegions.
Region(NativeAreaAllocator * allocator,uintptr_t allocateBase,uintptr_t end,RegionSpaceFlag spaceType)171 Region(NativeAreaAllocator *allocator, uintptr_t allocateBase, uintptr_t end,
172 RegionSpaceFlag spaceType)
173 : packedData_(allocateBase, spaceType), // no markGCBitset_ for JitFort
174 nativeAreaAllocator_(allocator),
175 allocateBase_(allocateBase),
176 end_(end),
177 highWaterMark_(end),
178 aliveObject_(0),
179 wasted_(0),
180 snapshotData_(0) {}
181
182 ~Region() = default;
183
184 NO_COPY_SEMANTIC(Region);
185 NO_MOVE_SEMANTIC(Region);
186
Initialize()187 void Initialize()
188 {
189 lock_ = new Mutex();
190 if (InSparseSpace()) {
191 InitializeFreeObjectSets();
192 }
193 }
194
LinkNext(Region * next)195 void LinkNext(Region *next)
196 {
197 next_ = next;
198 }
199
GetNext()200 Region *GetNext() const
201 {
202 return next_;
203 }
204
LinkPrev(Region * prev)205 void LinkPrev(Region *prev)
206 {
207 prev_ = prev;
208 }
209
GetPrev()210 Region *GetPrev() const
211 {
212 return prev_;
213 }
214
GetBegin()215 uintptr_t GetBegin() const
216 {
217 return packedData_.begin_;
218 }
219
GetEnd()220 uintptr_t GetEnd() const
221 {
222 return end_;
223 }
224
GetHighWaterMark()225 uintptr_t GetHighWaterMark() const
226 {
227 return highWaterMark_;
228 }
229
GetCapacity()230 size_t GetCapacity() const
231 {
232 return end_ - allocateBase_;
233 }
234
GetSize()235 size_t GetSize() const
236 {
237 return end_ - packedData_.begin_;
238 }
239
IsGCFlagSet(RegionGCFlags flag)240 bool IsGCFlagSet(RegionGCFlags flag) const
241 {
242 return (packedData_.flags_.gcFlags_ & flag) == flag;
243 }
244
SetGCFlag(RegionGCFlags flag)245 void SetGCFlag(RegionGCFlags flag)
246 {
247 packedData_.flags_.gcFlags_ |= flag;
248 }
249
ClearGCFlag(RegionGCFlags flag)250 void ClearGCFlag(RegionGCFlags flag)
251 {
252 // NOLINTNEXTLINE(hicpp-signed-bitwise)
253 packedData_.flags_.gcFlags_ &= ~flag;
254 }
255
GetSpaceTypeName()256 std::string GetSpaceTypeName()
257 {
258 return ToSpaceTypeName(packedData_.flags_.spaceFlag_);
259 }
260
GetSpaceType()261 uint8_t GetSpaceType() const
262 {
263 return packedData_.flags_.spaceFlag_;
264 }
265
266 // Mark bitset
267 GCBitset *GetMarkGCBitset() const;
268 bool AtomicMark(void *address);
269 // Objects in fresh region should only mark in JS Thread.
270 bool NonAtomicMark(void *address);
271 void ClearMark(void *address);
272 bool Test(void *addr) const;
273 bool Test(uintptr_t addr) const;
274 // ONLY used for heap verification.
275 bool TestNewToEden(uintptr_t addr);
276 bool TestOldToNew(uintptr_t addr);
277 bool TestLocalToShare(uintptr_t addr);
278 template <typename Visitor>
279 void IterateAllMarkedBits(Visitor visitor) const;
280 void ClearMarkGCBitset();
281 // local to share remembered set
282 bool HasLocalToShareRememberedSet() const;
283 RememberedSet *ExtractLocalToShareRSet();
284 void InsertLocalToShareRSet(uintptr_t addr);
285 void AtomicInsertLocalToShareRSet(uintptr_t addr);
286 void ClearLocalToShareRSetInRange(uintptr_t start, uintptr_t end);
287 void AtomicClearLocalToShareRSetInRange(uintptr_t start, uintptr_t end);
288 void AtomicClearSweepingLocalToShareRSetInRange(uintptr_t start, uintptr_t end);
289 template <typename Visitor>
290 void IterateAllLocalToShareBits(Visitor visitor);
291 void DeleteLocalToShareRSet();
292 void DeleteSweepingLocalToShareRSet();
293 // Cross region remembered set
294 void InsertCrossRegionRSet(uintptr_t addr);
295 void AtomicInsertCrossRegionRSet(uintptr_t addr);
296 template <typename Visitor>
297 void IterateAllCrossRegionBits(Visitor visitor) const;
298 void ClearCrossRegionRSet();
299 void ClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end);
300 void AtomicClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end);
301 void DeleteCrossRegionRSet();
302 // New to eden remembered set
303 void InsertNewToEdenRSet(uintptr_t addr);
304 void AtomicInsertNewToEdenRSet(uintptr_t addr);
305 void ClearNewToEdenRSet(uintptr_t addr);
306 // Old to new remembered set
307 void InsertOldToNewRSet(uintptr_t addr);
308 void ClearOldToNewRSet(uintptr_t addr);
309
310 template <typename Visitor>
311 void IterateAllNewToEdenBits(Visitor visitor);
312 template <typename Visitor>
313 void IterateAllOldToNewBits(Visitor visitor);
314 RememberedSet* GetNewToEdenRSet();
315 void ClearNewToEdenRSet();
316 void ClearNewToEdenRSetInRange(uintptr_t start, uintptr_t end);
317 void DeleteNewToEdenRSet();
318 void ClearOldToNewRSet();
319 void ClearOldToNewRSetInRange(uintptr_t start, uintptr_t end);
320 void DeleteOldToNewRSet();
321
322 void AtomicClearSweepingOldToNewRSetInRange(uintptr_t start, uintptr_t end);
323 void ClearSweepingOldToNewRSetInRange(uintptr_t start, uintptr_t end);
324 void DeleteSweepingOldToNewRSet();
325 template <typename Visitor>
326 void AtomicIterateAllSweepingRSetBits(Visitor visitor);
327 template <typename Visitor>
328 void IterateAllSweepingRSetBits(Visitor visitor);
329
ObjectAddressToRange(TaggedObject * obj)330 static Region *ObjectAddressToRange(TaggedObject *obj)
331 {
332 return reinterpret_cast<Region *>(ToUintPtr(obj) & ~DEFAULT_REGION_MASK);
333 }
334
ObjectAddressToRange(uintptr_t objAddress)335 static Region *ObjectAddressToRange(uintptr_t objAddress)
336 {
337 return reinterpret_cast<Region *>(objAddress & ~DEFAULT_REGION_MASK);
338 }
339
GetRegionAvailableSize()340 static size_t GetRegionAvailableSize()
341 {
342 size_t regionHeaderSize = AlignUp(sizeof(Region), static_cast<size_t>(MemAlignment::MEM_ALIGN_REGION));
343 size_t bitsetSize = GCBitset::SizeOfGCBitset(DEFAULT_REGION_SIZE - regionHeaderSize);
344 return DEFAULT_REGION_SIZE - regionHeaderSize - bitsetSize;
345 }
346
ClearMembers()347 void ClearMembers()
348 {
349 if (lock_ != nullptr) {
350 delete lock_;
351 lock_ = nullptr;
352 }
353 }
354
Invalidate()355 void Invalidate()
356 {
357 ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<void *>(GetBegin()), GetSize());
358 packedData_.flags_.spaceFlag_ = RegionSpaceFlag::UNINITIALIZED;
359 }
360
361 uint8_t GetRegionSpaceFlag();
362
SetRegionSpaceFlag(RegionSpaceFlag flag)363 void SetRegionSpaceFlag(RegionSpaceFlag flag)
364 {
365 packedData_.flags_.spaceFlag_ = flag;
366 }
InEdenSpace()367 bool InEdenSpace() const
368 {
369 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_EDEN_SPACE;
370 }
371
InYoungSpace()372 bool InYoungSpace() const
373 {
374 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_YOUNG_SPACE;
375 }
376
InOldSpace()377 bool InOldSpace() const
378 {
379 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_OLD_SPACE;
380 }
381
InYoungOrOldSpace()382 bool InYoungOrOldSpace() const
383 {
384 return InGeneralNewSpace() || InOldSpace();
385 }
386
InGeneralNewSpace()387 bool InGeneralNewSpace() const
388 {
389 auto flag = packedData_.flags_.spaceFlag_;
390 return flag >= RegionSpaceFlag::GENERAL_YOUNG_BEGIN && flag <= RegionSpaceFlag::GENERAL_YOUNG_END;
391 }
392
InGeneralOldSpace()393 bool InGeneralOldSpace() const
394 {
395 ASSERT(packedData_.flags_.spaceFlag_ != 0);
396 auto flag = packedData_.flags_.spaceFlag_;
397 return flag >= RegionSpaceFlag::GENERAL_OLD_BEGIN && flag <= RegionSpaceFlag::GENERAL_OLD_END;
398 }
399
InHugeObjectSpace()400 bool InHugeObjectSpace() const
401 {
402 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_HUGE_OBJECT_SPACE;
403 }
404
InMachineCodeSpace()405 bool InMachineCodeSpace() const
406 {
407 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_MACHINE_CODE_SPACE;
408 }
409
InHugeMachineCodeSpace()410 bool InHugeMachineCodeSpace() const
411 {
412 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_HUGE_MACHINE_CODE_SPACE;
413 }
414
InNonMovableSpace()415 bool InNonMovableSpace() const
416 {
417 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_NON_MOVABLE_SPACE;
418 }
419
InSnapshotSpace()420 bool InSnapshotSpace() const
421 {
422 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_SNAPSHOT_SPACE;
423 }
424
InReadOnlySpace()425 bool InReadOnlySpace() const
426 {
427 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_READ_ONLY_SPACE;
428 }
429
InSharedOldSpace()430 bool InSharedOldSpace() const
431 {
432 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_SHARED_OLD_SPACE;
433 }
434
InSharedNonMovableSpace()435 bool InSharedNonMovableSpace() const
436 {
437 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_SHARED_NON_MOVABLE;
438 }
439
InSharedHugeObjectSpace()440 bool InSharedHugeObjectSpace() const
441 {
442 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_SHARED_HUGE_OBJECT_SPACE;
443 }
444
InSharedReadOnlySpace()445 bool InSharedReadOnlySpace() const
446 {
447 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_SHARED_READ_ONLY_SPACE;
448 }
449
InSharedAppSpawnSpace()450 bool InSharedAppSpawnSpace() const
451 {
452 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_SHARED_APPSPAWN_SPACE;
453 }
454
InAppSpawnSpace()455 bool InAppSpawnSpace() const
456 {
457 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_APPSPAWN_SPACE;
458 }
459
460 // Not including shared read only space.
InSharedSweepableSpace()461 bool InSharedSweepableSpace() const
462 {
463 auto flag = packedData_.flags_.spaceFlag_;
464 return flag >= RegionSpaceFlag::SHARED_SWEEPABLE_SPACE_BEGIN &&
465 flag <= RegionSpaceFlag::SHARED_SWEEPABLE_SPACE_END;
466 }
467
InSharedHeap()468 bool InSharedHeap() const
469 {
470 auto flag = packedData_.flags_.spaceFlag_;
471 return flag >= RegionSpaceFlag::SHARED_SPACE_BEGIN && flag <= RegionSpaceFlag::SHARED_SPACE_END;
472 }
473
InSparseSpace()474 bool InSparseSpace() const
475 {
476 auto flag = packedData_.flags_.spaceFlag_;
477 switch (flag) {
478 case RegionSpaceFlag::IN_OLD_SPACE:
479 case RegionSpaceFlag::IN_NON_MOVABLE_SPACE:
480 case RegionSpaceFlag::IN_MACHINE_CODE_SPACE:
481 case RegionSpaceFlag::IN_APPSPAWN_SPACE:
482 case RegionSpaceFlag::IN_SHARED_NON_MOVABLE:
483 case RegionSpaceFlag::IN_SHARED_OLD_SPACE:
484 return true;
485 default:
486 return false;
487 }
488 }
489
InHeapSpace()490 bool InHeapSpace() const
491 {
492 uint8_t space = packedData_.flags_.spaceFlag_;
493 return space >= RegionSpaceFlag::HEAP_SPACE_BEGIN && space <= RegionSpaceFlag::HEAP_SPACE_END;
494 }
495
InCollectSet()496 bool InCollectSet() const
497 {
498 return IsGCFlagSet(RegionGCFlags::IN_COLLECT_SET);
499 }
500
InGeneralNewSpaceOrCSet()501 bool InGeneralNewSpaceOrCSet() const
502 {
503 return InGeneralNewSpace() || InCollectSet();
504 }
505
InNewToNewSet()506 bool InNewToNewSet() const
507 {
508 return IsGCFlagSet(RegionGCFlags::IN_NEW_TO_NEW_SET);
509 }
510
HasAgeMark()511 bool HasAgeMark() const
512 {
513 return IsGCFlagSet(RegionGCFlags::HAS_AGE_MARK);
514 }
515
BelowAgeMark()516 bool BelowAgeMark() const
517 {
518 return IsGCFlagSet(RegionGCFlags::BELOW_AGE_MARK);
519 }
520
NeedRelocate()521 bool NeedRelocate() const
522 {
523 return IsGCFlagSet(RegionGCFlags::NEED_RELOCATE);
524 }
525
526 // ONLY used for heap verification.
InInactiveSemiSpace()527 bool InInactiveSemiSpace() const
528 {
529 return IsGCFlagSet(RegionGCFlags::IN_INACTIVE_SEMI_SPACE);
530 }
531
532 // ONLY used for heap verification.
InActiveSemiSpace()533 bool InActiveSemiSpace() const
534 {
535 return InYoungSpace() && !InInactiveSemiSpace();
536 }
537
GetRegionTypeFlag()538 RegionTypeFlag GetRegionTypeFlag() const
539 {
540 return packedData_.typeFlag_;
541 }
542
SetRegionTypeFlag(RegionTypeFlag typeFlag)543 void SetRegionTypeFlag(RegionTypeFlag typeFlag)
544 {
545 packedData_.typeFlag_ = typeFlag;
546 }
547
ResetRegionTypeFlag()548 void ResetRegionTypeFlag()
549 {
550 SetRegionTypeFlag(RegionTypeFlag::DEFAULT);
551 }
552
IsFreshRegion()553 bool IsFreshRegion() const
554 {
555 return GetRegionTypeFlag() == RegionTypeFlag::FRESH;
556 }
557
IsHalfFreshRegion()558 bool IsHalfFreshRegion() const
559 {
560 return GetRegionTypeFlag() == RegionTypeFlag::HALF_FRESH;
561 }
562
563 // ONLY used for heap verification.
SetInactiveSemiSpace()564 void SetInactiveSemiSpace()
565 {
566 SetGCFlag(RegionGCFlags::IN_INACTIVE_SEMI_SPACE);
567 }
568
569 // ONLY used for heap verification.
ResetInactiveSemiSpace()570 void ResetInactiveSemiSpace()
571 {
572 ClearGCFlag(RegionGCFlags::IN_INACTIVE_SEMI_SPACE);
573 }
574
SetSwept()575 void SetSwept()
576 {
577 SetGCFlag(RegionGCFlags::HAS_BEEN_SWEPT);
578 }
579
ResetSwept()580 void ResetSwept()
581 {
582 ClearGCFlag(RegionGCFlags::HAS_BEEN_SWEPT);
583 }
584
InRange(uintptr_t address)585 bool InRange(uintptr_t address) const
586 {
587 return address >= packedData_.begin_ && address <= end_;
588 }
589
GetAllocateBase()590 uintptr_t GetAllocateBase() const
591 {
592 return allocateBase_;
593 }
594
595 size_t GetAllocatedBytes(uintptr_t top = 0)
596 {
597 ASSERT(top == 0 || InRange(top));
598 return (top == 0) ? (highWaterMark_ - packedData_.begin_) : (top - packedData_.begin_);
599 }
600
SetHighWaterMark(uintptr_t mark)601 void SetHighWaterMark(uintptr_t mark)
602 {
603 ASSERT(InRange(mark));
604 highWaterMark_ = mark;
605 }
606
SetReadOnlyAndMarked()607 void SetReadOnlyAndMarked()
608 {
609 packedData_.markGCBitset_->SetAllBits(packedData_.bitsetSize_);
610 PageProtect(reinterpret_cast<void *>(allocateBase_), GetCapacity(), PAGE_PROT_READ);
611 }
612
ClearReadOnly()613 void ClearReadOnly()
614 {
615 PageProtect(reinterpret_cast<void *>(allocateBase_), GetCapacity(), PAGE_PROT_READWRITE);
616 }
617
InitializeFreeObjectSets()618 void InitializeFreeObjectSets()
619 {
620 FreeObjectSet<FreeObject> **sets = new FreeObjectSet<FreeObject> *[FreeObjectList<FreeObject>::NumberOfSets()];
621 for (int i = 0; i < FreeObjectList<FreeObject>::NumberOfSets(); i++) {
622 sets[i] = new FreeObjectSet<FreeObject>(i);
623 }
624 freeObjectSets_ = Span<FreeObjectSet<FreeObject> *>(sets, FreeObjectList<FreeObject>::NumberOfSets());
625 }
626
DestroyFreeObjectSets()627 void DestroyFreeObjectSets()
628 {
629 for (int i = 0; i < FreeObjectList<FreeObject>::NumberOfSets(); i++) {
630 delete freeObjectSets_[i];
631 freeObjectSets_[i] = nullptr;
632 }
633 delete[] freeObjectSets_.data();
634 }
635
GetFreeObjectSet(SetType type)636 FreeObjectSet<FreeObject> *GetFreeObjectSet(SetType type)
637 {
638 // Thread safe
639 if (freeObjectSets_[type] == nullptr) {
640 freeObjectSets_[type] = new FreeObjectSet<FreeObject>(type);
641 }
642 return freeObjectSets_[type];
643 }
644
645 template<class Callback>
EnumerateFreeObjectSets(Callback cb)646 void EnumerateFreeObjectSets(Callback cb)
647 {
648 for (auto set : freeObjectSets_) {
649 cb(set);
650 }
651 }
652
653 template<class Callback>
REnumerateFreeObjectSets(Callback cb)654 void REnumerateFreeObjectSets(Callback cb)
655 {
656 auto last = freeObjectSets_.crbegin();
657 auto first = freeObjectSets_.crend();
658 for (; last != first; last++) {
659 if (!cb(*last)) {
660 break;
661 }
662 }
663 }
664
IncreaseAliveObjectSafe(size_t size)665 void IncreaseAliveObjectSafe(size_t size)
666 {
667 ASSERT(aliveObject_ + size <= GetSize());
668 aliveObject_ += size;
669 }
670
IncreaseAliveObject(size_t size)671 void IncreaseAliveObject(size_t size)
672 {
673 aliveObject_.fetch_add(size, std::memory_order_relaxed);
674 }
675
SetRegionAliveSize()676 void SetRegionAliveSize()
677 {
678 gcAliveSize_ = aliveObject_;
679 }
680
ResetAliveObject()681 void ResetAliveObject()
682 {
683 aliveObject_ = 0;
684 }
685
AliveObject()686 size_t AliveObject() const
687 {
688 return aliveObject_.load(std::memory_order_relaxed);
689 }
690
GetGCAliveSize()691 size_t GetGCAliveSize() const
692 {
693 return gcAliveSize_;
694 }
695
MostObjectAlive()696 bool MostObjectAlive() const
697 {
698 return aliveObject_ > MOST_OBJECT_ALIVE_THRESHOLD_PERCENT * GetSize();
699 }
700
BelowCompressThreasholdAlive()701 bool BelowCompressThreasholdAlive() const
702 {
703 return gcAliveSize_ < COMPRESS_THREASHOLD_PERCENT * GetSize();
704 }
705
ResetWasted()706 void ResetWasted()
707 {
708 wasted_ = 0;
709 }
710
IncreaseWasted(uint64_t size)711 void IncreaseWasted(uint64_t size)
712 {
713 wasted_ += size;
714 }
715
GetWastedSize()716 uint64_t GetWastedSize()
717 {
718 return wasted_;
719 }
720
GetSnapshotData()721 uint64_t GetSnapshotData()
722 {
723 return snapshotData_;
724 }
725
SetSnapshotData(uint64_t value)726 void SetSnapshotData(uint64_t value)
727 {
728 snapshotData_ = value;
729 }
730
SwapOldToNewRSetForCS()731 void SwapOldToNewRSetForCS()
732 {
733 sweepingOldToNewRSet_ = packedData_.oldToNewSet_;
734 packedData_.oldToNewSet_ = nullptr;
735 }
736
SwapLocalToShareRSetForCS()737 void SwapLocalToShareRSetForCS()
738 {
739 sweepingLocalToShareRSet_ = packedData_.localToShareSet_;
740 packedData_.localToShareSet_ = nullptr;
741 }
742
743 // should call in js-thread
744 void MergeOldToNewRSetForCS();
745 void MergeLocalToShareRSetForCS();
746
747 // should call in daemon-thread, or in js-thread in RUNNING state
748 void MergeLocalToShareRSetForCM(RememberedSet *set);
749
750 struct alignas(JSTaggedValue::TaggedTypeSize()) PackedPtr : public base::AlignedPointer {
751 uint8_t spaceFlag_;
752 uint16_t gcFlags_;
753 };
754
755 struct PackedData : public base::AlignedStruct<JSTaggedValue::TaggedTypeSize(),
756 base::AlignedPointer,
757 base::AlignedPointer,
758 base::AlignedPointer,
759 base::AlignedPointer,
760 base::AlignedPointer,
761 base::AlignedPointer,
762 base::AlignedSize> {
763 enum class Index : size_t {
764 FlagsIndex = 0,
765 TypeFlagIndex,
766 MarkGCBitSetIndex,
767 OldToNewSetIndex,
768 LocalToShareSetIndex,
769 BeginIndex,
770 BitSetSizeIndex,
771 NumOfMembers
772 };
773
774 static_assert(static_cast<size_t>(Index::NumOfMembers) == NumOfTypes);
775
PackedDataPackedData776 inline PackedData(uintptr_t begin, uintptr_t end, RegionSpaceFlag spaceType, RegionTypeFlag typeFlag)
777 {
778 flags_.spaceFlag_ = spaceType;
779 flags_.gcFlags_ = 0;
780 typeFlag_ = typeFlag;
781 bitsetSize_ = (spaceType == RegionSpaceFlag::IN_HUGE_OBJECT_SPACE ||
782 spaceType == RegionSpaceFlag::IN_HUGE_MACHINE_CODE_SPACE ||
783 spaceType == RegionSpaceFlag::IN_SHARED_HUGE_OBJECT_SPACE) ?
784 GCBitset::BYTE_PER_WORD : GCBitset::SizeOfGCBitset(end - begin);
785 markGCBitset_ = new (ToVoidPtr(begin)) GCBitset();
786 markGCBitset_->Clear(bitsetSize_);
787 begin_ = AlignUp(begin + bitsetSize_, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
788 // The object region marked with poison until it is allocated if is_asan is true
789 #ifdef ARK_ASAN_ON
790 ASAN_POISON_MEMORY_REGION(reinterpret_cast<void *>(begin_), (end - begin_));
791 #endif
792 }
793
PackedDataPackedData794 inline PackedData(uintptr_t begin, RegionSpaceFlag spaceType)
795 {
796 flags_.spaceFlag_ = spaceType;
797 flags_.gcFlags_ = 0;
798 typeFlag_ = RegionTypeFlag::DEFAULT;
799 // no markGCBitset
800 begin_ = begin;
801 markGCBitset_ = nullptr;
802 }
803
GetFlagsOffsetPackedData804 static size_t GetFlagsOffset(bool isArch32)
805 {
806 return GetOffset<static_cast<size_t>(Index::FlagsIndex)>(isArch32);
807 }
808
GetTypeFlagOffsetPackedData809 static size_t GetTypeFlagOffset(bool isArch32)
810 {
811 return GetOffset<static_cast<size_t>(Index::TypeFlagIndex)>(isArch32);
812 }
813
GetGCBitsetOffsetPackedData814 static size_t GetGCBitsetOffset(bool isArch32)
815 {
816 return GetOffset<static_cast<size_t>(Index::MarkGCBitSetIndex)>(isArch32);
817 }
818
GetNewToEdenSetOffsetPackedData819 static size_t GetNewToEdenSetOffset(bool isArch32)
820 {
821 // NewToEdenRSet is Union with OldToNewRSet
822 return GetOffset<static_cast<size_t>(Index::OldToNewSetIndex)>(isArch32);
823 }
824
GetOldToNewSetOffsetPackedData825 static size_t GetOldToNewSetOffset(bool isArch32)
826 {
827 return GetOffset<static_cast<size_t>(Index::OldToNewSetIndex)>(isArch32);
828 }
829
GetLocalToShareSetOffsetPackedData830 static size_t GetLocalToShareSetOffset(bool isArch32)
831 {
832 return GetOffset<static_cast<size_t>(Index::LocalToShareSetIndex)>(isArch32);
833 }
834
GetBeginOffsetPackedData835 static size_t GetBeginOffset(bool isArch32)
836 {
837 return GetOffset<static_cast<size_t>(Index::BeginIndex)>(isArch32);
838 }
839
840 alignas(EAS) PackedPtr flags_;
841 // Use different UIntPtr from flags_ to prevent the potential data race.
842 // Be careful when storing to this value, currently this is only from JS_Thread during ConcurrentMarking,
843 // or from GC_Thread during GC ClearTask.
844 alignas(EAS) RegionTypeFlag typeFlag_;
845 alignas(EAS) GCBitset *markGCBitset_ {nullptr};
846 // OldToNewRSet only for general OldSpace, NewToEdenRSet only for YoungSpace. Their pointers can union
847 union {
848 alignas(EAS) RememberedSet *oldToNewSet_ {nullptr};
849 alignas(EAS) RememberedSet *newToEdenSet_;
850 };
851 alignas(EAS) RememberedSet *localToShareSet_ {nullptr};
852 alignas(EAS) uintptr_t begin_ {0};
853 alignas(EAS) size_t bitsetSize_ {0};
854 };
855 STATIC_ASSERT_EQ_ARCH(sizeof(PackedData), PackedData::SizeArch32, PackedData::SizeArch64);
856
857 static constexpr double MOST_OBJECT_ALIVE_THRESHOLD_PERCENT = 0.8;
858 static constexpr double AVERAGE_REGION_EVACUATE_SIZE = MOST_OBJECT_ALIVE_THRESHOLD_PERCENT *
859 DEFAULT_REGION_SIZE / 2; // 2 means half
860 private:
861 static constexpr double COMPRESS_THREASHOLD_PERCENT = 0.1;
862
863 RememberedSet *CreateRememberedSet();
864 RememberedSet *GetOrCreateCrossRegionRememberedSet();
865 RememberedSet *GetOrCreateNewToEdenRememberedSet();
866 RememberedSet *GetOrCreateOldToNewRememberedSet();
867 RememberedSet *GetOrCreateLocalToShareRememberedSet();
868
869 PackedData packedData_;
870 NativeAreaAllocator *nativeAreaAllocator_;
871
872 uintptr_t allocateBase_;
873 uintptr_t end_;
874 uintptr_t highWaterMark_;
875 std::atomic_size_t aliveObject_ {0};
876 size_t gcAliveSize_ {0};
877 Region *next_ {nullptr};
878 Region *prev_ {nullptr};
879
880 RememberedSet *crossRegionSet_ {nullptr};
881 RememberedSet *sweepingOldToNewRSet_ {nullptr};
882 RememberedSet *sweepingLocalToShareRSet_ {nullptr};
883 Span<FreeObjectSet<FreeObject> *> freeObjectSets_;
884 Mutex *lock_ {nullptr};
885 uint64_t wasted_;
886 // snapshotdata_ is used to encode the region for snapshot. Its upper 32 bits are used to store the size of
887 // the huge object, and the lower 32 bits are used to store the region index
888 uint64_t snapshotData_;
889
890 friend class Snapshot;
891 friend class SnapshotProcessor;
892 };
893 } // namespace ecmascript
894 } // namespace panda
895 #endif // ECMASCRIPT_MEM_REGION_H
896