1 /*
2 * Copyright (c) 2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef ECMASCRIPT_MEM_REGION_H
17 #define ECMASCRIPT_MEM_REGION_H
18
19 #include "ecmascript/base/aligned_struct.h"
20 #include "ecmascript/base/asan_interface.h"
21 #include "ecmascript/js_tagged_value.h"
22 #include "ecmascript/mem/free_object_list.h"
23 #include "ecmascript/mem/gc_bitset.h"
24 #include "ecmascript/mem/remembered_set.h"
25 #include "ecmascript/mem/mem_common.h"
26 #include "ecmascript/platform/map.h"
27
28 #include "libpandabase/os/mutex.h"
29
30 #include "securec.h"
31
32 namespace panda {
33 namespace ecmascript {
34 class JSThread;
35
36 enum RegionSpaceFlag {
37 UNINITIALIZED = 0,
38 // We should avoid using the lower 3 bits (bits 0 to 2).
39 // If ZAP_MEM is enabled, the value of the lower 3 bits conflicts with the INVALID_VALUE.
40
41 // Bits 3 to 7 are reserved to denote the space where the region is located.
42 IN_YOUNG_SPACE = 0x08,
43 IN_SNAPSHOT_SPACE = 0x09,
44 IN_HUGE_OBJECT_SPACE = 0x0A,
45 IN_OLD_SPACE = 0x0B,
46 IN_NON_MOVABLE_SPACE = 0x0C,
47 IN_MACHINE_CODE_SPACE = 0x0D,
48 IN_READ_ONLY_SPACE = 0x0E,
49 IN_APPSPAWN_SPACE = 0X0F,
50
51 VALID_SPACE_MASK = 0xFF,
52 };
53
54 enum RegionGCFlags {
55 // We should avoid using the lower 3 bits (bits 0 to 2).
56 // If ZAP_MEM is enabled, the value of the lower 3 bits conflicts with the INVALID_VALUE.
57
58 // Below flags are used for GC, and each flag has a dedicated bit starting from the 3rd bit.
59 NEVER_EVACUATE = 1 << 3,
60 HAS_AGE_MARK = 1 << 4,
61 BELOW_AGE_MARK = 1 << 5,
62 IN_COLLECT_SET = 1 << 6,
63 IN_NEW_TO_NEW_SET = 1 << 7,
64 // Bits 8 to 10 (the lower 3 bits for the next byte) are also excluded for the sake of
65 // INVALID_VALUE in ZAP_MEM.
66 HAS_BEEN_SWEPT = 1 << 11,
67 NEED_RELOCATE = 1 << 12,
68 };
69
ToSpaceTypeName(uint8_t space)70 static inline std::string ToSpaceTypeName(uint8_t space)
71 {
72 switch (space) {
73 case RegionSpaceFlag::IN_YOUNG_SPACE:
74 return "young space";
75 case RegionSpaceFlag::IN_SNAPSHOT_SPACE:
76 return "snapshot space";
77 case RegionSpaceFlag::IN_HUGE_OBJECT_SPACE:
78 return "huge object space";
79 case RegionSpaceFlag::IN_OLD_SPACE:
80 return "old space";
81 case RegionSpaceFlag::IN_NON_MOVABLE_SPACE:
82 return "non movable space";
83 case RegionSpaceFlag::IN_MACHINE_CODE_SPACE:
84 return "machine code space";
85 case RegionSpaceFlag::IN_READ_ONLY_SPACE:
86 return "read only space";
87 case RegionSpaceFlag::IN_APPSPAWN_SPACE:
88 return "appspawn space";
89 default:
90 return "invalid space";
91 }
92 }
93
94 // |---------------------------------------------------------------------------------------|
95 // | Region (256 kb) |
96 // |---------------------------------|--------------------------------|--------------------|
97 // | Head (sizeof(Region)) | Mark bitset (4kb) | Data |
98 // |---------------------------------|--------------------------------|--------------------|
99
100 class Region {
101 public:
Region(JSThread * thread,uintptr_t allocateBase,uintptr_t begin,uintptr_t end,RegionSpaceFlag spaceType)102 Region(JSThread *thread, uintptr_t allocateBase, uintptr_t begin, uintptr_t end, RegionSpaceFlag spaceType)
103 : packedData_(begin, end, spaceType),
104 thread_(thread),
105 allocateBase_(allocateBase),
106 end_(end),
107 highWaterMark_(end),
108 aliveObject_(0),
109 wasted_(0),
110 snapshotData_(0)
111 {
112 lock_ = new os::memory::Mutex();
113 }
114
115 ~Region() = default;
116
117 NO_COPY_SEMANTIC(Region);
118 NO_MOVE_SEMANTIC(Region);
119
LinkNext(Region * next)120 void LinkNext(Region *next)
121 {
122 next_ = next;
123 }
124
GetNext()125 Region *GetNext() const
126 {
127 return next_;
128 }
129
LinkPrev(Region * prev)130 void LinkPrev(Region *prev)
131 {
132 prev_ = prev;
133 }
134
GetPrev()135 Region *GetPrev() const
136 {
137 return prev_;
138 }
139
GetBegin()140 uintptr_t GetBegin() const
141 {
142 return packedData_.begin_;
143 }
144
GetEnd()145 uintptr_t GetEnd() const
146 {
147 return end_;
148 }
149
GetHighWaterMark()150 uintptr_t GetHighWaterMark() const
151 {
152 return highWaterMark_;
153 }
154
GetCapacity()155 size_t GetCapacity() const
156 {
157 return end_ - allocateBase_;
158 }
159
GetSize()160 size_t GetSize() const
161 {
162 return end_ - packedData_.begin_;
163 }
164
GetJSThread()165 JSThread *GetJSThread() const
166 {
167 return thread_;
168 }
169
IsGCFlagSet(RegionGCFlags flag)170 bool IsGCFlagSet(RegionGCFlags flag) const
171 {
172 return (packedData_.flags_.gcFlags_ & flag) == flag;
173 }
174
SetGCFlag(RegionGCFlags flag)175 void SetGCFlag(RegionGCFlags flag)
176 {
177 packedData_.flags_.gcFlags_ |= flag;
178 }
179
ClearGCFlag(RegionGCFlags flag)180 void ClearGCFlag(RegionGCFlags flag)
181 {
182 // NOLINTNEXTLINE(hicpp-signed-bitwise)
183 packedData_.flags_.gcFlags_ &= ~flag;
184 }
185
GetSpaceTypeName()186 std::string GetSpaceTypeName()
187 {
188 return ToSpaceTypeName(packedData_.flags_.spaceFlag_);
189 }
190
191 // Mark bitset
192 GCBitset *GetMarkGCBitset() const;
193 bool AtomicMark(void *address);
194 void ClearMark(void *address);
195 bool Test(void *addr) const;
196 template <typename Visitor>
197 void IterateAllMarkedBits(Visitor visitor) const;
198 void ClearMarkGCBitset();
199 // Cross region remembered set
200 void InsertCrossRegionRSet(uintptr_t addr);
201 void AtomicInsertCrossRegionRSet(uintptr_t addr);
202 template <typename Visitor>
203 void IterateAllCrossRegionBits(Visitor visitor) const;
204 void ClearCrossRegionRSet();
205 void ClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end);
206 void AtomicClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end);
207 void DeleteCrossRegionRSet();
208 // Old to new remembered set
209 void InsertOldToNewRSet(uintptr_t addr);
210 void ClearOldToNewRSet(uintptr_t addr);
211 template <typename Visitor>
212 void IterateAllOldToNewBits(Visitor visitor);
213 void ClearOldToNewRSet();
214 void ClearOldToNewRSetInRange(uintptr_t start, uintptr_t end);
215 void DeleteOldToNewRSet();
216
217 void AtomicClearSweepingRSetInRange(uintptr_t start, uintptr_t end);
218 void ClearSweepingRSetInRange(uintptr_t start, uintptr_t end);
219 void DeleteSweepingRSet();
220 template <typename Visitor>
221 void AtomicIterateAllSweepingRSetBits(Visitor visitor);
222 template <typename Visitor>
223 void IterateAllSweepingRSetBits(Visitor visitor);
224
ObjectAddressToRange(TaggedObject * obj)225 static Region *ObjectAddressToRange(TaggedObject *obj)
226 {
227 return reinterpret_cast<Region *>(ToUintPtr(obj) & ~DEFAULT_REGION_MASK);
228 }
229
ObjectAddressToRange(uintptr_t objAddress)230 static Region *ObjectAddressToRange(uintptr_t objAddress)
231 {
232 return reinterpret_cast<Region *>(objAddress & ~DEFAULT_REGION_MASK);
233 }
234
ClearMembers()235 void ClearMembers()
236 {
237 if (lock_ != nullptr) {
238 delete lock_;
239 lock_ = nullptr;
240 }
241 }
242
Invalidate()243 void Invalidate()
244 {
245 ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<void *>(GetBegin()), GetSize());
246 packedData_.flags_.spaceFlag_ = RegionSpaceFlag::UNINITIALIZED;
247 }
248
InYoungSpace()249 bool InYoungSpace() const
250 {
251 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_YOUNG_SPACE;
252 }
253
InOldSpace()254 bool InOldSpace() const
255 {
256 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_OLD_SPACE;
257 }
258
InYoungOrOldSpace()259 bool InYoungOrOldSpace() const
260 {
261 return InYoungSpace() || InOldSpace();
262 }
263
InHugeObjectSpace()264 bool InHugeObjectSpace() const
265 {
266 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_HUGE_OBJECT_SPACE;
267 }
268
InMachineCodeSpace()269 bool InMachineCodeSpace() const
270 {
271 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_MACHINE_CODE_SPACE;
272 }
273
InNonMovableSpace()274 bool InNonMovableSpace() const
275 {
276 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_NON_MOVABLE_SPACE;
277 }
278
InSnapshotSpace()279 bool InSnapshotSpace() const
280 {
281 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_SNAPSHOT_SPACE;
282 }
283
InReadOnlySpace()284 bool InReadOnlySpace() const
285 {
286 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_READ_ONLY_SPACE;
287 }
288
InAppSpawnSpace()289 bool InAppSpawnSpace() const
290 {
291 return packedData_.flags_.spaceFlag_ == RegionSpaceFlag::IN_APPSPAWN_SPACE;
292 }
293
InHeapSpace()294 bool InHeapSpace() const
295 {
296 uint8_t space = packedData_.flags_.spaceFlag_;
297 return (space == RegionSpaceFlag::IN_YOUNG_SPACE ||
298 space == RegionSpaceFlag::IN_OLD_SPACE ||
299 space == RegionSpaceFlag::IN_HUGE_OBJECT_SPACE ||
300 space == RegionSpaceFlag::IN_MACHINE_CODE_SPACE ||
301 space == RegionSpaceFlag::IN_NON_MOVABLE_SPACE ||
302 space == RegionSpaceFlag::IN_SNAPSHOT_SPACE ||
303 space == RegionSpaceFlag::IN_READ_ONLY_SPACE ||
304 space == RegionSpaceFlag::IN_APPSPAWN_SPACE);
305 }
306
InCollectSet()307 bool InCollectSet() const
308 {
309 return IsGCFlagSet(RegionGCFlags::IN_COLLECT_SET);
310 }
311
InYoungSpaceOrCSet()312 bool InYoungSpaceOrCSet() const
313 {
314 return InYoungSpace() || InCollectSet();
315 }
316
InNewToNewSet()317 bool InNewToNewSet() const
318 {
319 return IsGCFlagSet(RegionGCFlags::IN_NEW_TO_NEW_SET);
320 }
321
HasAgeMark()322 bool HasAgeMark() const
323 {
324 return IsGCFlagSet(RegionGCFlags::HAS_AGE_MARK);
325 }
326
BelowAgeMark()327 bool BelowAgeMark() const
328 {
329 return IsGCFlagSet(RegionGCFlags::BELOW_AGE_MARK);
330 }
331
NeedRelocate()332 bool NeedRelocate() const
333 {
334 return IsGCFlagSet(RegionGCFlags::NEED_RELOCATE);
335 }
336
SetSwept()337 void SetSwept()
338 {
339 SetGCFlag(RegionGCFlags::HAS_BEEN_SWEPT);
340 }
341
ResetSwept()342 void ResetSwept()
343 {
344 ClearGCFlag(RegionGCFlags::HAS_BEEN_SWEPT);
345 }
346
InRange(uintptr_t address)347 bool InRange(uintptr_t address) const
348 {
349 return address >= packedData_.begin_ && address <= end_;
350 }
351
GetAllocateBase()352 uintptr_t GetAllocateBase() const
353 {
354 return allocateBase_;
355 }
356
357 size_t GetAllocatedBytes(uintptr_t top = 0)
358 {
359 ASSERT(top == 0 || InRange(top));
360 return (top == 0) ? (highWaterMark_ - packedData_.begin_) : (top - packedData_.begin_);
361 }
362
SetHighWaterMark(uintptr_t mark)363 void SetHighWaterMark(uintptr_t mark)
364 {
365 ASSERT(InRange(mark));
366 highWaterMark_ = mark;
367 }
368
SetReadOnlyAndMarked()369 void SetReadOnlyAndMarked()
370 {
371 packedData_.markGCBitset_->SetAllBits(packedData_.bitsetSize_);
372 PageProtect(reinterpret_cast<void *>(allocateBase_), GetCapacity(), PAGE_PROT_READ);
373 }
374
ClearReadOnly()375 void ClearReadOnly()
376 {
377 PageProtect(reinterpret_cast<void *>(allocateBase_), GetCapacity(), PAGE_PROT_READWRITE);
378 }
379
InitializeFreeObjectSets()380 void InitializeFreeObjectSets()
381 {
382 freeObjectSets_ = Span<FreeObjectSet *>(new FreeObjectSet *[FreeObjectList::NumberOfSets()](),
383 FreeObjectList::NumberOfSets());
384 }
385
DestroyFreeObjectSets()386 void DestroyFreeObjectSets()
387 {
388 for (auto set : freeObjectSets_) {
389 delete set;
390 }
391 delete[] freeObjectSets_.data();
392 }
393
GetFreeObjectSet(SetType type)394 FreeObjectSet *GetFreeObjectSet(SetType type)
395 {
396 // Thread safe
397 if (freeObjectSets_[type] == nullptr) {
398 freeObjectSets_[type] = new FreeObjectSet(type);
399 }
400 return freeObjectSets_[type];
401 }
402
403 template<class Callback>
EnumerateFreeObjectSets(Callback cb)404 void EnumerateFreeObjectSets(Callback cb)
405 {
406 for (auto set : freeObjectSets_) {
407 cb(set);
408 }
409 }
410
411 template<class Callback>
REnumerateFreeObjectSets(Callback cb)412 void REnumerateFreeObjectSets(Callback cb)
413 {
414 auto last = freeObjectSets_.crbegin();
415 auto first = freeObjectSets_.crend();
416 for (; last != first; last++) {
417 if (!cb(*last)) {
418 break;
419 }
420 }
421 }
422
423 inline bool IsMarking() const;
424
IncreaseAliveObjectSafe(size_t size)425 void IncreaseAliveObjectSafe(size_t size)
426 {
427 ASSERT(aliveObject_ + size <= GetSize());
428 aliveObject_ += size;
429 }
430
IncreaseAliveObject(size_t size)431 void IncreaseAliveObject(size_t size)
432 {
433 ASSERT(aliveObject_ + size <= GetSize());
434 aliveObject_.fetch_add(size, std::memory_order_relaxed);
435 }
436
SetRegionAliveSize()437 void SetRegionAliveSize()
438 {
439 gcAliveSize_ = aliveObject_;
440 }
441
ResetAliveObject()442 void ResetAliveObject()
443 {
444 aliveObject_ = 0;
445 }
446
AliveObject()447 size_t AliveObject() const
448 {
449 return aliveObject_.load(std::memory_order_relaxed);
450 }
451
GetGCAliveSize()452 size_t GetGCAliveSize() const
453 {
454 return gcAliveSize_;
455 }
456
MostObjectAlive()457 bool MostObjectAlive() const
458 {
459 return aliveObject_ > MOST_OBJECT_ALIVE_THRESHOLD_PERCENT * GetSize();
460 }
461
BelowCompressThreasholdAlive()462 bool BelowCompressThreasholdAlive() const
463 {
464 return gcAliveSize_ < COMPRESS_THREASHOLD_PERCENT * GetSize();
465 }
466
ResetWasted()467 void ResetWasted()
468 {
469 wasted_ = 0;
470 }
471
IncreaseWasted(uint64_t size)472 void IncreaseWasted(uint64_t size)
473 {
474 wasted_ += size;
475 }
476
GetWastedSize()477 uint64_t GetWastedSize()
478 {
479 return wasted_;
480 }
481
GetSnapshotData()482 uint64_t GetSnapshotData()
483 {
484 return snapshotData_;
485 }
486
SetSnapshotData(uint64_t value)487 void SetSnapshotData(uint64_t value)
488 {
489 snapshotData_ = value;
490 }
491
SwapRSetForConcurrentSweeping()492 void SwapRSetForConcurrentSweeping()
493 {
494 sweepingRSet_ = packedData_.oldToNewSet_;
495 packedData_.oldToNewSet_ = nullptr;
496 }
497
498 // should call in js-thread
499 void MergeRSetForConcurrentSweeping();
500
501 struct alignas(JSTaggedValue::TaggedTypeSize()) PackedPtr : public base::AlignedPointer {
502 uint8_t spaceFlag_;
503 uint16_t gcFlags_;
504 };
505
506 struct PackedData : public base::AlignedStruct<JSTaggedValue::TaggedTypeSize(),
507 base::AlignedPointer,
508 base::AlignedPointer,
509 base::AlignedPointer,
510 base::AlignedPointer,
511 base::AlignedSize> {
512 enum class Index : size_t {
513 FlagIndex = 0,
514 MarkGCBitSetIndex,
515 OldToNewSetIndex,
516 BeginIndex,
517 BitSetSizeIndex,
518 NumOfMembers
519 };
520
521 static_assert(static_cast<size_t>(Index::NumOfMembers) == NumOfTypes);
522
PackedDataPackedData523 inline PackedData(uintptr_t begin, uintptr_t end, RegionSpaceFlag spaceType)
524 {
525 flags_.spaceFlag_ = spaceType;
526 flags_.gcFlags_ = 0;
527 bitsetSize_ = (spaceType == RegionSpaceFlag::IN_HUGE_OBJECT_SPACE) ?
528 GCBitset::BYTE_PER_WORD : GCBitset::SizeOfGCBitset(end - begin);
529 markGCBitset_ = new (ToVoidPtr(begin)) GCBitset();
530 markGCBitset_->Clear(bitsetSize_);
531 begin_ = AlignUp(begin + bitsetSize_, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
532 // The object region marked with poison until it is allocated if is_asan is true
533 ASAN_POISON_MEMORY_REGION(reinterpret_cast<void *>(begin_), (end - begin_));
534 }
535
GetFlagOffsetPackedData536 static size_t GetFlagOffset(bool isArch32)
537 {
538 return GetOffset<static_cast<size_t>(Index::FlagIndex)>(isArch32);
539 }
540
GetGCBitsetOffsetPackedData541 static size_t GetGCBitsetOffset(bool isArch32)
542 {
543 return GetOffset<static_cast<size_t>(Index::MarkGCBitSetIndex)>(isArch32);
544 }
545
GetOldToNewSetOffsetPackedData546 static size_t GetOldToNewSetOffset(bool isArch32)
547 {
548 return GetOffset<static_cast<size_t>(Index::OldToNewSetIndex)>(isArch32);
549 }
550
GetBeginOffsetPackedData551 static size_t GetBeginOffset(bool isArch32)
552 {
553 return GetOffset<static_cast<size_t>(Index::BeginIndex)>(isArch32);
554 }
555
556 alignas(EAS) PackedPtr flags_;
557 alignas(EAS) GCBitset *markGCBitset_ {nullptr};
558 alignas(EAS) RememberedSet *oldToNewSet_ {nullptr};
559 alignas(EAS) uintptr_t begin_ {0};
560 alignas(EAS) size_t bitsetSize_ {0};
561 };
562 STATIC_ASSERT_EQ_ARCH(sizeof(PackedData), PackedData::SizeArch32, PackedData::SizeArch64);
563
564 static constexpr double MOST_OBJECT_ALIVE_THRESHOLD_PERCENT = 0.8;
565 static constexpr double AVERAGE_REGION_EVACUATE_SIZE = MOST_OBJECT_ALIVE_THRESHOLD_PERCENT *
566 DEFAULT_REGION_SIZE / 2; // 2 means half
567 private:
568 static constexpr double COMPRESS_THREASHOLD_PERCENT = 0.1;
569
570 RememberedSet *CreateRememberedSet();
571 RememberedSet *GetOrCreateCrossRegionRememberedSet();
572 RememberedSet *GetOrCreateOldToNewRememberedSet();
573
574 PackedData packedData_;
575 /*
576 * The thread instance here is used by the GC barriers to get marking related information
577 * and perform marking related operations. The barriers will indirectly access such information
578 * via. the objects' associated regions.
579 * fixme: Figure out a more elegant solution to bridging the barrier
580 * and the information / operations it depends on. Then we can get rid of this from the region,
581 * and consequently, the region allocator, the spaces using the region allocator, etc.
582 */
583 JSThread *thread_;
584
585 uintptr_t allocateBase_;
586 uintptr_t end_;
587 uintptr_t highWaterMark_;
588 std::atomic_size_t aliveObject_ {0};
589 size_t gcAliveSize_ {0};
590 Region *next_ {nullptr};
591 Region *prev_ {nullptr};
592
593 RememberedSet *crossRegionSet_ {nullptr};
594 RememberedSet *sweepingRSet_ {nullptr};
595 Span<FreeObjectSet *> freeObjectSets_;
596 os::memory::Mutex *lock_ {nullptr};
597 uint64_t wasted_;
598 // snapshotdata_ is used to encode the region for snapshot. Its upper 32 bits are used to store the size of
599 // the huge object, and the lower 32 bits are used to store the region index
600 uint64_t snapshotData_;
601
602 friend class Snapshot;
603 friend class SnapshotProcessor;
604 };
605 } // namespace ecmascript
606 } // namespace panda
607 #endif // ECMASCRIPT_MEM_REGION_H
608