• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /**
2  * Copyright (c) 2021-2025 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef PANDA_MEM_GC_BITMAP_H
17 #define PANDA_MEM_GC_BITMAP_H
18 
19 #include <securec.h>
20 #include <cstddef>
21 #include <cstdint>
22 #include <cstring>
23 #include <atomic>
24 
25 #include "libpandabase/macros.h"
26 #include "libpandabase/mem/mem.h"
27 #include "libpandabase/utils/bit_utils.h"
28 #include "libpandabase/utils/math_helpers.h"
29 #include "libpandabase/utils/span.h"
30 
31 namespace ark::mem {
32 
33 /// Abstract base class. Constructor/destructor are protected. No virtual function to avoid dynamic polymorphism.
34 class Bitmap {
35 public:
36     using BitmapWordType = uintptr_t;
37 
Size()38     size_t Size() const
39     {
40         return bitsize_;
41     }
42 
ClearAllBits()43     void ClearAllBits()
44     {
45         memset_s(bitmap_.Data(), bitmap_.SizeBytes(), 0, bitmap_.SizeBytes());
46     }
47 
48     void CopyTo(Bitmap *dest) const;
49 
GetBitMap()50     Span<BitmapWordType> GetBitMap()
51     {
52         return bitmap_;
53     }
54 
GetSetBitCount()55     size_t GetSetBitCount()
56     {
57         size_t countMarkBits = 0;
58         for (BitmapWordType word : bitmap_) {
59             countMarkBits += static_cast<size_t>(Popcount(word));
60         }
61         return countMarkBits;
62     }
63 
64     static const size_t BITSPERBYTE = 8;
65     static const size_t BITSPERWORD = BITSPERBYTE * sizeof(BitmapWordType);
66     static constexpr size_t LOG_BITSPERBYTE = ark::helpers::math::GetIntLog2(static_cast<uint64_t>(BITSPERBYTE));
67     static constexpr size_t LOG_BITSPERWORD = ark::helpers::math::GetIntLog2(static_cast<uint64_t>(BITSPERWORD));
68 
69 protected:
70     /**
71      * @brief Set the bit indexed by bit_offset.
72      * @param bit_offset - index of the bit to set.
73      */
SetBit(size_t bitOffset)74     void SetBit(size_t bitOffset)
75     {
76         CheckBitOffset(bitOffset);
77         bitmap_[GetWordIdx(bitOffset)] |= GetBitMask(bitOffset);
78     }
79 
80     /**
81      * @brief Clear the bit indexed by bit_offset.
82      * @param bit_offset - index of the bit to clear.
83      */
ClearBit(size_t bitOffset)84     void ClearBit(size_t bitOffset)
85     {
86         CheckBitOffset(bitOffset);
87         bitmap_[GetWordIdx(bitOffset)] &= ~GetBitMask(bitOffset);
88     }
89 
90     /**
91      * @brief Test the bit indexed by bit_offset.
92      * @param bit_offset - index of the bit to test.
93      * @return Returns value of indexed bit.
94      */
TestBit(size_t bitOffset)95     bool TestBit(size_t bitOffset) const
96     {
97         CheckBitOffset(bitOffset);
98         return (bitmap_[GetWordIdx(bitOffset)] & GetBitMask(bitOffset)) != 0;
99     }
100 
101     /**
102      * @brief Atomically set bit indexed by bit_offset. If the bit is not set, set it atomically. Otherwise, do nothing.
103      * @param bit_offset - index of the bit to set.
104      * @return Returns old value of the bit.
105      */
106     bool AtomicTestAndSetBit(size_t bitOffset);
107 
108     /**
109      * @brief Atomically clear bit corresponding to addr. If the bit is set, clear it atomically. Otherwise, do nothing.
110      * @param addr - addr must be aligned to BYTESPERCHUNK.
111      * @return Returns old value of the bit.
112      */
113     bool AtomicTestAndClearBit(size_t bitOffset);
114 
115     /**
116      * @brief Atomically test bit corresponding to addr.
117      * @param addr - addr must be aligned to BYTESPERCHUNK.
118      * @return Returns the value of the bit.
119      */
120     bool AtomicTestBit(size_t bitOffset);
121 
122     /**
123      * @brief Iterates over all bits of bitmap sequentially.
124      * @tparam VisitorType
125      * @param visitor - function pointer or functor.
126      */
127     template <typename VisitorType>
IterateOverBits(const VisitorType & visitor)128     void IterateOverBits(const VisitorType &visitor)
129     {
130         IterateOverBitsInRange(0, Size(), visitor);
131     }
132 
133     /**
134      * @brief Iterates over marked bits in range [begin, end) sequentially.
135      * Finish iteration if the visitor returns false.
136      * @tparam atomic - true if want to set bit atomically.
137      * @tparam VisitorType
138      * @param begin - beginning index of the range, inclusive.
139      * @param end - end index of the range, exclusive.
140      * @param visitor - function pointer or functor.
141      */
142     // default value exists only with backward-compatibility with js_runtime, should be removed in the future
143     template <bool ATOMIC = false, typename VisitorType>
IterateOverSetBitsInRange(size_t begin,size_t end,const VisitorType & visitor)144     void IterateOverSetBitsInRange(size_t begin, size_t end, const VisitorType &visitor)
145     {
146         CheckBitRange(begin, end);
147         if (UNLIKELY(begin == end)) {
148             return;
149         }
150 
151         auto bitmapWord = GetBitmapWord<ATOMIC>(begin);
152         auto offsetWithinWord = GetBitIdxWithinWord(begin);
153         // first word, clear bits before begin
154         bitmapWord &= GetRangeBitMask(offsetWithinWord, BITSPERWORD);
155         auto offsetWordBegin = GetWordIdx(begin) * BITSPERWORD;
156         const bool rightAligned = (GetBitIdxWithinWord(end) == 0);
157         const auto offsetLastWordBegin = GetWordIdx(end) * BITSPERWORD;
158         do {
159             if (offsetWordBegin == offsetLastWordBegin && !rightAligned) {
160                 // last partial word, clear bits after right boundary
161                 auto mask = GetRangeBitMask(0, GetBitIdxWithinWord(end));
162                 bitmapWord &= mask;
163             }
164             // loop over bits of bitmap_word
165             while (offsetWithinWord < BITSPERWORD) {
166                 if (bitmapWord == 0) {
167                     break;
168                 }
169                 offsetWithinWord = static_cast<size_t>(Ctz(bitmapWord));
170                 if (!visitor(offsetWordBegin + offsetWithinWord)) {
171                     return;
172                 }
173                 bitmapWord &= ~GetBitMask(offsetWithinWord);
174             }
175 
176             offsetWordBegin += BITSPERWORD;
177             if (offsetWordBegin >= end) {
178                 break;
179             }
180 
181             bitmapWord = GetBitmapWord<ATOMIC>(offsetWordBegin);
182             offsetWithinWord = 0;
183         } while (true);
184     }
185 
186     /**
187      * @brief Iterates over marked bits of bitmap sequentially.
188      * Finish iteration if the visitor returns false.
189      * @tparam atomic - true if want to iterate with atomic reading.
190      * @tparam VisitorType
191      * @param visitor - function pointer or functor.
192      */
193     template <bool ATOMIC, typename VisitorType>
IterateOverSetBits(const VisitorType & visitor)194     void IterateOverSetBits(const VisitorType &visitor)
195     {
196         IterateOverSetBitsInRange<ATOMIC, VisitorType>(0, Size(), visitor);
197     }
198 
199     /**
200      * @brief Iterates over all bits in range [begin, end) sequentially.
201      * @tparam VisitorType
202      * @param begin - beginning index of the range, inclusive.
203      * @param end - end index of the range, exclusive.
204      * @param visitor - function pointer or functor.
205      */
206     template <typename VisitorType>
IterateOverBitsInRange(size_t begin,size_t end,const VisitorType & visitor)207     void IterateOverBitsInRange(size_t begin, size_t end, const VisitorType &visitor)
208     {
209         CheckBitRange(begin, end);
210         for (size_t i = begin; i < end; ++i) {
211             visitor(i);
212         }
213     }
214 
215     /**
216      * @brief Clear all bits in range [begin, end).
217      * @param begin - beginning index of the range, inclusive.
218      * @param end - end index of the range, exclusive.
219      */
220     void ClearBitsInRange(size_t begin, size_t end);
221 
222     /**
223      * @brief Set all bits in range [begin, end). [begin, end) must be within a BitmapWord.
224      * @param begin - beginning index of the range, inclusive.
225      * @param end - end index of the range, exclusive.
226      */
SetRangeWithinWord(size_t begin,size_t end)227     void SetRangeWithinWord(size_t begin, size_t end)
228     {
229         if (LIKELY(begin != end)) {
230             ModifyRangeWithinWord<true>(begin, end);
231         }
232     }
233 
234     /**
235      * @brief Clear all bits in range [begin, end). [begin, end) must be within a BitmapWord.
236      * @param begin - beginning index of the range, inclusive.
237      * @param end - end index of the range, exclusive.
238      */
ClearRangeWithinWord(size_t begin,size_t end)239     void ClearRangeWithinWord(size_t begin, size_t end)
240     {
241         if (LIKELY(begin != end)) {
242             ModifyRangeWithinWord<false>(begin, end);
243         }
244     }
245 
246     /**
247      * @brief Set all BitmapWords in index range [begin, end).
248      * @param begin - beginning BitmapWord index of the range, inclusive.
249      * @param end - end BitmapWord index of the range, exclusive.
250      */
SetWords(size_t wordBegin,size_t wordEnd)251     void SetWords([[maybe_unused]] size_t wordBegin, [[maybe_unused]] size_t wordEnd)
252     {
253         ASSERT(wordBegin <= wordEnd);
254         if (UNLIKELY(wordBegin == wordEnd)) {
255             return;
256         }
257         memset_s(&bitmap_[wordBegin], (wordEnd - wordBegin) * sizeof(BitmapWordType), ~static_cast<unsigned char>(0),
258                  (wordEnd - wordBegin) * sizeof(BitmapWordType));
259     }
260 
261     /**
262      * @brief Clear all BitmapWords in index range [begin, end).
263      * @param begin - beginning BitmapWord index of the range, inclusive.
264      * @param end - end BitmapWord index of the range, exclusive.
265      */
ClearWords(size_t wordBegin,size_t wordEnd)266     void ClearWords([[maybe_unused]] size_t wordBegin, [[maybe_unused]] size_t wordEnd)
267     {
268         ASSERT(wordBegin <= wordEnd);
269         if (UNLIKELY(wordBegin == wordEnd)) {
270             return;
271         }
272         memset_s(&bitmap_[wordBegin], (wordEnd - wordBegin) * sizeof(BitmapWordType), static_cast<unsigned char>(0),
273                  (wordEnd - wordBegin) * sizeof(BitmapWordType));
274     }
275 
276     template <bool ATOMIC>
FindHighestPrecedingOrSameBit(size_t begin)277     size_t FindHighestPrecedingOrSameBit(size_t begin)
278     {
279         auto wordBeginOffset = GetWordIdx(begin) * BITSPERWORD;
280         auto offsetWithinWord = GetBitIdxWithinWord(begin);
281         auto mask = ~static_cast<BitmapWordType>(0) >> (BITSPERWORD - offsetWithinWord - 1);
282         auto bitmapWord = GetBitmapWord<ATOMIC>(begin) & mask;
283 
284         while (true) {
285             if (bitmapWord != 0) {
286                 offsetWithinWord = BITSPERWORD - static_cast<size_t>(Clz(bitmapWord)) - 1;
287                 return wordBeginOffset + offsetWithinWord;
288             }
289 
290             if (wordBeginOffset < BITSPERWORD) {
291                 break;
292             }
293 
294             wordBeginOffset -= BITSPERWORD;
295             bitmapWord = GetBitmapWord<ATOMIC>(wordBeginOffset);
296         }
297 
298         return begin;
299     }
300 
Bitmap(BitmapWordType * bitmap,size_t bitsize)301     explicit Bitmap(BitmapWordType *bitmap, size_t bitsize)
302         : bitmap_(bitmap, (AlignUp(bitsize, BITSPERWORD) >> LOG_BITSPERWORD)), bitsize_(bitsize)
303     {
304     }
305     ~Bitmap() = default;
306     // do we need special copy/move constructor?
307     NO_COPY_SEMANTIC(Bitmap);
308     NO_MOVE_SEMANTIC(Bitmap);
309 
310 private:
311     Span<BitmapWordType> bitmap_;
312     size_t bitsize_ = 0;
313 
314     /**
315      * @brief Compute word index from bit index.
316      * @param bit_offset - bit index.
317      * @return Returns BitmapWord Index of bit_offset.
318      */
GetWordIdx(size_t bitOffset)319     static size_t GetWordIdx(size_t bitOffset)
320     {
321         return bitOffset >> LOG_BITSPERWORD;
322     }
323 
324     /**
325      * @brief Compute bit index within a BitmapWord from bit index.
326      * @param bit_offset - bit index.
327      * @return Returns bit index within a BitmapWord.
328      */
GetBitIdxWithinWord(size_t bitOffset)329     size_t GetBitIdxWithinWord(size_t bitOffset) const
330     {
331         CheckBitOffset(bitOffset);
332         constexpr auto BIT_INDEX_MASK = static_cast<size_t>((1ULL << LOG_BITSPERWORD) - 1);
333         return bitOffset & BIT_INDEX_MASK;
334     }
335 
336     /**
337      * @brief Compute bit mask from bit index.
338      * @param bit_offset - bit index.
339      * @return Returns bit mask of bit_offset.
340      */
GetBitMask(size_t bitOffset)341     BitmapWordType GetBitMask(size_t bitOffset) const
342     {
343         return 1ULL << GetBitIdxWithinWord(bitOffset);
344     }
345 
346     /**
347      * @brief Compute bit mask of range [begin_within_word, end_within_word).
348      * @param begin_within_word - beginning index within word, in range [0, BITSPERWORD).
349      * @param end_within_word - end index within word, in range [0, BITSPERWORD]. Make sure end_within_word is
350      * BITSPERWORD(instead of 0) if you want to cover from certain bit to last. [0, 0) is the only valid case when
351      * end_within_word is 0.
352      * @return Returns bit mask.
353      */
GetRangeBitMask(size_t beginWithinWord,size_t endWithinWord)354     BitmapWordType GetRangeBitMask(size_t beginWithinWord, size_t endWithinWord) const
355     {
356         ASSERT(beginWithinWord < BITSPERWORD);
357         ASSERT(endWithinWord <= BITSPERWORD);
358         ASSERT(beginWithinWord <= endWithinWord);
359         auto endMask = (endWithinWord == BITSPERWORD) ? ~static_cast<BitmapWordType>(0) : GetBitMask(endWithinWord) - 1;
360         return endMask - (GetBitMask(beginWithinWord) - 1);
361     }
362 
363     /// @brief Check if bit_offset is valid.
CheckBitOffset(size_t bitOffset)364     void CheckBitOffset([[maybe_unused]] size_t bitOffset) const
365     {
366         ASSERT(bitOffset <= Size());
367     }
368 
369     /**
370      * @brief According to SET, set or clear range [begin, end) within a BitmapWord.
371      * @param begin - beginning global bit index.
372      * @param end - end global bit index.
373      */
374     template <bool SET>
ModifyRangeWithinWord(size_t begin,size_t end)375     ALWAYS_INLINE void ModifyRangeWithinWord(size_t begin, size_t end)
376     {
377         CheckBitRange(begin, end);
378 
379         if (UNLIKELY(begin == end)) {
380             return;
381         }
382 
383         BitmapWordType mask;
384         if (end % BITSPERWORD == 0) {
385             ASSERT(GetWordIdx(end) - GetWordIdx(begin) == 1);
386             mask = GetRangeBitMask(GetBitIdxWithinWord(begin), BITSPERWORD);
387         } else {
388             ASSERT(GetWordIdx(end) == GetWordIdx(begin));
389             mask = GetRangeBitMask(GetBitIdxWithinWord(begin), GetBitIdxWithinWord(end));
390         }
391 
392         if (SET) {
393             bitmap_[GetWordIdx(begin)] |= mask;
394         } else {
395             bitmap_[GetWordIdx(begin)] &= ~mask;
396         }
397     }
398 
399     /// @brief Check if bit range [begin, end) is valid.
CheckBitRange(size_t begin,size_t end)400     void CheckBitRange([[maybe_unused]] size_t begin, [[maybe_unused]] size_t end) const
401     {
402         ASSERT(begin < Size());
403         ASSERT(end <= Size());
404         ASSERT(begin <= end);
405     }
406 
407     template <bool ATOMIC>
GetBitmapWord(size_t bitOffset)408     BitmapWordType GetBitmapWord(size_t bitOffset)
409     {
410         auto index = GetWordIdx(bitOffset);
411         if constexpr (!ATOMIC) {
412             return bitmap_[index];
413         }
414 
415         auto *wordAddr = reinterpret_cast<std::atomic<BitmapWordType> *>(&bitmap_[index]);
416         // Atomic with seq_cst order reason: datarace with AtomicTestAndSet
417         return wordAddr->load(std::memory_order_seq_cst);
418     }
419 };
420 
421 /**
422  * Memory bitmap, binding a continuous range of memory to a bitmap.
423  * One bit represents BYTESPERCHUNK bytes of memory.
424  */
425 template <size_t BYTESPERCHUNK = 1, typename PointerType = ObjectPointerType>
426 class MemBitmap : public Bitmap {
427 public:
MemBitmap(void * memAddr,size_t heapSize,void * bitmapAddr)428     explicit MemBitmap(void *memAddr, size_t heapSize, void *bitmapAddr)
429         : Bitmap(static_cast<BitmapWordType *>(bitmapAddr), heapSize / BYTESPERCHUNK),
430           beginAddr_(ToPointerType(memAddr)),
431           endAddr_(beginAddr_ + heapSize)
432     {
433     }
434     NO_COPY_SEMANTIC(MemBitmap);
435     NO_MOVE_SEMANTIC(MemBitmap);
436 
437     /**
438      * @brief Reinitialize the MemBitmap for new memory range.
439      * The size of range will be the same as the initial
440      * because we reuse the same bitmap storage.
441      * @param mem_addr - start addr of the new range.
442      */
ReInitializeMemoryRange(void * memAddr)443     void ReInitializeMemoryRange(void *memAddr)
444     {
445         beginAddr_ = ToPointerType(memAddr);
446         endAddr_ = beginAddr_ + MemSizeInBytes();
447         Bitmap::ClearAllBits();
448     }
449 
GetBitMapSizeInByte(size_t heapSize)450     inline static constexpr size_t GetBitMapSizeInByte(size_t heapSize)
451     {
452         ASSERT(heapSize % BYTESPERCHUNK == 0);
453         size_t bitSize = heapSize / BYTESPERCHUNK;
454         return (AlignUp(bitSize, BITSPERWORD) >> Bitmap::LOG_BITSPERWORD) * sizeof(BitmapWordType);
455     }
456 
457     ~MemBitmap() = default;
458 
MemSizeInBytes()459     size_t MemSizeInBytes() const
460     {
461         return Size() * BYTESPERCHUNK;
462     }
463 
GetHeapRange()464     inline std::pair<uintptr_t, uintptr_t> GetHeapRange()
465     {
466         return {beginAddr_, endAddr_};
467     }
468 
469     /**
470      * @brief Set bit corresponding to addr.
471      * @param addr - addr must be aligned to BYTESPERCHUNK.
472      */
Set(void * addr)473     void Set(void *addr)
474     {
475         CheckAddrValidity(addr);
476         SetBit(AddrToBitOffset(ToPointerType(addr)));
477     }
478 
479     /**
480      * @brief Clear bit corresponding to addr.
481      * @param addr - addr must be aligned to BYTESPERCHUNK.
482      */
Clear(void * addr)483     void Clear(void *addr)
484     {
485         CheckAddrValidity(addr);
486         ClearBit(AddrToBitOffset(ToPointerType(addr)));
487     }
488 
489     /// @brief Clear bits corresponding to addr range [begin, end).
ClearRange(void * begin,void * end)490     ALWAYS_INLINE void ClearRange(void *begin, void *end)
491     {
492         CheckHalfClosedHalfOpenAddressRange(begin, end);
493         ClearBitsInRange(AddrToBitOffset(ToPointerType(begin)), EndAddrToBitOffset(ToPointerType(end)));
494     }
495 
496     /**
497      * @brief Test bit corresponding to addr.
498      * @param addr - addr must be aligned to BYTESPERCHUNK.
499      */
Test(const void * addr)500     bool Test(const void *addr) const
501     {
502         CheckAddrValidity(addr);
503         return TestBit(AddrToBitOffset(ToPointerType(addr)));
504     }
505 
506     /**
507      * @brief Test bit corresponding to addr if addr is valid.
508      * @return value of indexed bit if addr is valid. If addr is invalid then false
509      */
TestIfAddrValid(const void * addr)510     bool TestIfAddrValid(const void *addr) const
511     {
512         if (IsAddrValid(addr)) {
513             return TestBit(AddrToBitOffset(ToPointerType(addr)));
514         }
515         return false;
516     }
517 
518     /**
519      * @brief Atomically set bit corresponding to addr. If the bit is not set, set it atomically. Otherwise, do nothing.
520      * @param addr - addr must be aligned to BYTESPERCHUNK.
521      * @return Returns old value of the bit.
522      */
AtomicTestAndSet(void * addr)523     bool AtomicTestAndSet(void *addr)
524     {
525         CheckAddrValidity(addr);
526         return AtomicTestAndSetBit(AddrToBitOffset(ToPointerType(addr)));
527     }
528 
529     /**
530      * @brief Atomically clear bit corresponding to addr. If the bit is set, clear it atomically. Otherwise, do nothing.
531      * @param addr - addr must be aligned to BYTESPERCHUNK.
532      * @return Returns old value of the bit.
533      */
AtomicTestAndClear(void * addr)534     bool AtomicTestAndClear(void *addr)
535     {
536         CheckAddrValidity(addr);
537         return AtomicTestAndClearBit(AddrToBitOffset(ToPointerType(addr)));
538     }
539 
540     /**
541      * @brief Atomically test bit corresponding to addr.
542      * @param addr - addr must be aligned to BYTESPERCHUNK.
543      * @return Returns the value of the bit.
544      */
AtomicTest(const void * addr)545     bool AtomicTest(const void *addr)
546     {
547         CheckAddrValidity(addr);
548         return AtomicTestBit(AddrToBitOffset(ToPointerType(addr)));
549     }
550 
551     /// @brief Find first marked chunk.
552     template <bool ATOMIC = false>
FindFirstMarkedChunks()553     void *FindFirstMarkedChunks()
554     {
555         void *firstMarked = nullptr;
556         IterateOverSetBits<ATOMIC>([&firstMarked, this](size_t bitOffset) {
557             firstMarked = BitOffsetToAddr(bitOffset);
558             return false;
559         });
560         return firstMarked;
561     }
562 
563     /// @brief Iterates over marked chunks of memory sequentially.
564     template <bool ATOMIC = false, typename MemVisitor>
IterateOverMarkedChunks(const MemVisitor & visitor)565     void IterateOverMarkedChunks(const MemVisitor &visitor)
566     {
567         IterateOverSetBits<ATOMIC>([&visitor, this](size_t bitOffset) {
568             visitor(BitOffsetToAddr(bitOffset));
569             return true;
570         });
571     }
572 
573     /// @brief Iterates over all chunks of memory sequentially.
574     template <typename MemVisitor>
IterateOverChunks(const MemVisitor & visitor)575     void IterateOverChunks(const MemVisitor &visitor)
576     {
577         IterateOverBits([&visitor, this](size_t bitOffset) { visitor(BitOffsetToAddr(bitOffset)); });
578     }
579 
580     /// @brief Iterates over marked chunks of memory in range [begin, end) sequentially.
581     template <bool ATOMIC = false, typename MemVisitor, typename T = void *>
IterateOverMarkedChunkInRange(void * begin,void * end,const MemVisitor & visitor)582     void IterateOverMarkedChunkInRange(void *begin, void *end, const MemVisitor &visitor)
583     {
584         IterateOverMarkedChunkInRangeInterruptible<ATOMIC>(begin, end, [&visitor](void *obj) {
585             visitor(obj);
586             return true;
587         });
588     }
589 
590     /**
591      * @brief Iterates over marked chunks of memory in range [begin, end) sequentially and stops iteration if visitor
592      * returns false
593      */
594     template <bool ATOMIC = false, typename MemVisitor, typename T = void *>
IterateOverMarkedChunkInRangeInterruptible(void * begin,void * end,const MemVisitor & visitor)595     void IterateOverMarkedChunkInRangeInterruptible(void *begin, void *end, const MemVisitor &visitor)
596     {
597         CheckHalfClosedHalfOpenAddressRange(begin, end);
598         IterateOverSetBitsInRange<ATOMIC>(FindHighestPrecedingOrSameBit<ATOMIC>(AddrToBitOffset(ToPointerType(begin))),
599                                           EndAddrToBitOffset(ToPointerType(end)), [&visitor, this](size_t bitOffset) {
600                                               return visitor(BitOffsetToAddr(bitOffset));
601                                           });
602     }
603 
604     /// @brief Call visitor for single allocated object in humongous region
605     template <bool ATOMIC, typename MemVisitor>
CallForMarkedChunkInHumongousRegion(void * begin,const MemVisitor & visitor)606     void CallForMarkedChunkInHumongousRegion(void *begin, const MemVisitor &visitor)
607     {
608         bool marked;
609         if constexpr (ATOMIC) {
610             marked = AtomicTest(begin);
611         } else {
612             marked = Test(begin);
613         }
614         if (marked) {
615             visitor(begin);
616         }
617     }
618 
619     /// @brief Iterates over all chunks of memory in range [begin, end) sequentially.
620     template <typename MemVisitor>
IterateOverChunkInRange(void * begin,void * end,const MemVisitor & visitor)621     void IterateOverChunkInRange(void *begin, void *end, const MemVisitor &visitor)
622     {
623         CheckHalfClosedHalfOpenAddressRange(begin, end);
624         IterateOverBitsInRange(AddrToBitOffset(ToPointerType(begin)), EndAddrToBitOffset(ToPointerType(end)),
625                                [&visitor, this](size_t bitOffset) { visitor(BitOffsetToAddr(bitOffset)); });
626     }
627 
IsAddrInRange(const void * addr)628     bool IsAddrInRange(const void *addr) const
629     {
630         return addr >= ToVoidPtr(beginAddr_) && addr < ToVoidPtr(endAddr_);
631     }
632 
633     template <class T>
ToPointerType(T * val)634     static constexpr PointerType ToPointerType(T *val)
635     {
636         return static_cast<PointerType>(ToUintPtr(val));
637     }
638 
639 private:
640     /// @brief Computes bit offset from addr.
AddrToBitOffset(PointerType addr)641     size_t AddrToBitOffset(PointerType addr) const
642     {
643         return (addr - beginAddr_) / BYTESPERCHUNK;
644     }
645 
EndAddrToBitOffset(PointerType addr)646     size_t EndAddrToBitOffset(PointerType addr) const
647     {
648         return (AlignUp(addr, BYTESPERCHUNK) - beginAddr_) / BYTESPERCHUNK;
649     }
650 
651     /// @brief Computes address from bit offset.
BitOffsetToAddr(size_t bitOffset)652     void *BitOffsetToAddr(size_t bitOffset) const
653     {
654         return ToVoidPtr(beginAddr_ + bitOffset * BYTESPERCHUNK);
655     }
656 
657     /// @brief Check if addr is valid.
CheckAddrValidity(const void * addr)658     void CheckAddrValidity([[maybe_unused]] const void *addr) const
659     {
660         ASSERT(IsAddrInRange(addr));
661         ASSERT((ToPointerType(addr) - beginAddr_) % BYTESPERCHUNK == 0);
662     }
663 
664     /**
665      * @brief Check if addr is valid with returned value.
666      * @return true if addr is valid
667      */
IsAddrValid(const void * addr)668     bool IsAddrValid(const void *addr) const
669     {
670         return IsAddrInRange(addr) && (ToPointerType(addr) - beginAddr_) % BYTESPERCHUNK == 0;
671     }
672 
673     /// @brief Check if [begin, end) is a valid address range.
CheckHalfClosedHalfOpenAddressRange(void * begin,void * end)674     void CheckHalfClosedHalfOpenAddressRange([[maybe_unused]] void *begin, [[maybe_unused]] void *end) const
675     {
676         CheckAddrValidity(begin);
677         ASSERT(ToPointerType(end) >= beginAddr_);
678         ASSERT(ToPointerType(end) <= endAddr_);
679         ASSERT(ToPointerType(begin) <= ToPointerType(end));
680     }
681 
682     PointerType beginAddr_ {0};
683     PointerType endAddr_ {0};
684 };
685 
686 using MarkBitmap = MemBitmap<DEFAULT_ALIGNMENT_IN_BYTES>;
687 
688 }  // namespace ark::mem
689 #endif  // PANDA_MEM_GC_BITMAP_H
690