1 /** 2 * Copyright (c) 2021-2025 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef PANDA_MEM_GC_BITMAP_H 17 #define PANDA_MEM_GC_BITMAP_H 18 19 #include <securec.h> 20 #include <cstddef> 21 #include <cstdint> 22 #include <cstring> 23 #include <atomic> 24 25 #include "libpandabase/macros.h" 26 #include "libpandabase/mem/mem.h" 27 #include "libpandabase/utils/bit_utils.h" 28 #include "libpandabase/utils/math_helpers.h" 29 #include "libpandabase/utils/span.h" 30 31 namespace ark::mem { 32 33 /// Abstract base class. Constructor/destructor are protected. No virtual function to avoid dynamic polymorphism. 34 class Bitmap { 35 public: 36 using BitmapWordType = uintptr_t; 37 Size()38 size_t Size() const 39 { 40 return bitsize_; 41 } 42 ClearAllBits()43 void ClearAllBits() 44 { 45 memset_s(bitmap_.Data(), bitmap_.SizeBytes(), 0, bitmap_.SizeBytes()); 46 } 47 48 void CopyTo(Bitmap *dest) const; 49 GetBitMap()50 Span<BitmapWordType> GetBitMap() 51 { 52 return bitmap_; 53 } 54 GetSetBitCount()55 size_t GetSetBitCount() 56 { 57 size_t countMarkBits = 0; 58 for (BitmapWordType word : bitmap_) { 59 countMarkBits += static_cast<size_t>(Popcount(word)); 60 } 61 return countMarkBits; 62 } 63 64 static const size_t BITSPERBYTE = 8; 65 static const size_t BITSPERWORD = BITSPERBYTE * sizeof(BitmapWordType); 66 static constexpr size_t LOG_BITSPERBYTE = ark::helpers::math::GetIntLog2(static_cast<uint64_t>(BITSPERBYTE)); 67 static constexpr size_t LOG_BITSPERWORD = ark::helpers::math::GetIntLog2(static_cast<uint64_t>(BITSPERWORD)); 68 69 protected: 70 /** 71 * @brief Set the bit indexed by bit_offset. 72 * @param bit_offset - index of the bit to set. 73 */ SetBit(size_t bitOffset)74 void SetBit(size_t bitOffset) 75 { 76 CheckBitOffset(bitOffset); 77 bitmap_[GetWordIdx(bitOffset)] |= GetBitMask(bitOffset); 78 } 79 80 /** 81 * @brief Clear the bit indexed by bit_offset. 82 * @param bit_offset - index of the bit to clear. 83 */ ClearBit(size_t bitOffset)84 void ClearBit(size_t bitOffset) 85 { 86 CheckBitOffset(bitOffset); 87 bitmap_[GetWordIdx(bitOffset)] &= ~GetBitMask(bitOffset); 88 } 89 90 /** 91 * @brief Test the bit indexed by bit_offset. 92 * @param bit_offset - index of the bit to test. 93 * @return Returns value of indexed bit. 94 */ TestBit(size_t bitOffset)95 bool TestBit(size_t bitOffset) const 96 { 97 CheckBitOffset(bitOffset); 98 return (bitmap_[GetWordIdx(bitOffset)] & GetBitMask(bitOffset)) != 0; 99 } 100 101 /** 102 * @brief Atomically set bit indexed by bit_offset. If the bit is not set, set it atomically. Otherwise, do nothing. 103 * @param bit_offset - index of the bit to set. 104 * @return Returns old value of the bit. 105 */ 106 bool AtomicTestAndSetBit(size_t bitOffset); 107 108 /** 109 * @brief Atomically clear bit corresponding to addr. If the bit is set, clear it atomically. Otherwise, do nothing. 110 * @param addr - addr must be aligned to BYTESPERCHUNK. 111 * @return Returns old value of the bit. 112 */ 113 bool AtomicTestAndClearBit(size_t bitOffset); 114 115 /** 116 * @brief Atomically test bit corresponding to addr. 117 * @param addr - addr must be aligned to BYTESPERCHUNK. 118 * @return Returns the value of the bit. 119 */ 120 bool AtomicTestBit(size_t bitOffset); 121 122 /** 123 * @brief Iterates over all bits of bitmap sequentially. 124 * @tparam VisitorType 125 * @param visitor - function pointer or functor. 126 */ 127 template <typename VisitorType> IterateOverBits(const VisitorType & visitor)128 void IterateOverBits(const VisitorType &visitor) 129 { 130 IterateOverBitsInRange(0, Size(), visitor); 131 } 132 133 /** 134 * @brief Iterates over marked bits in range [begin, end) sequentially. 135 * Finish iteration if the visitor returns false. 136 * @tparam atomic - true if want to set bit atomically. 137 * @tparam VisitorType 138 * @param begin - beginning index of the range, inclusive. 139 * @param end - end index of the range, exclusive. 140 * @param visitor - function pointer or functor. 141 */ 142 // default value exists only with backward-compatibility with js_runtime, should be removed in the future 143 template <bool ATOMIC = false, typename VisitorType> IterateOverSetBitsInRange(size_t begin,size_t end,const VisitorType & visitor)144 void IterateOverSetBitsInRange(size_t begin, size_t end, const VisitorType &visitor) 145 { 146 CheckBitRange(begin, end); 147 if (UNLIKELY(begin == end)) { 148 return; 149 } 150 151 auto bitmapWord = GetBitmapWord<ATOMIC>(begin); 152 auto offsetWithinWord = GetBitIdxWithinWord(begin); 153 // first word, clear bits before begin 154 bitmapWord &= GetRangeBitMask(offsetWithinWord, BITSPERWORD); 155 auto offsetWordBegin = GetWordIdx(begin) * BITSPERWORD; 156 const bool rightAligned = (GetBitIdxWithinWord(end) == 0); 157 const auto offsetLastWordBegin = GetWordIdx(end) * BITSPERWORD; 158 do { 159 if (offsetWordBegin == offsetLastWordBegin && !rightAligned) { 160 // last partial word, clear bits after right boundary 161 auto mask = GetRangeBitMask(0, GetBitIdxWithinWord(end)); 162 bitmapWord &= mask; 163 } 164 // loop over bits of bitmap_word 165 while (offsetWithinWord < BITSPERWORD) { 166 if (bitmapWord == 0) { 167 break; 168 } 169 offsetWithinWord = static_cast<size_t>(Ctz(bitmapWord)); 170 if (!visitor(offsetWordBegin + offsetWithinWord)) { 171 return; 172 } 173 bitmapWord &= ~GetBitMask(offsetWithinWord); 174 } 175 176 offsetWordBegin += BITSPERWORD; 177 if (offsetWordBegin >= end) { 178 break; 179 } 180 181 bitmapWord = GetBitmapWord<ATOMIC>(offsetWordBegin); 182 offsetWithinWord = 0; 183 } while (true); 184 } 185 186 /** 187 * @brief Iterates over marked bits of bitmap sequentially. 188 * Finish iteration if the visitor returns false. 189 * @tparam atomic - true if want to iterate with atomic reading. 190 * @tparam VisitorType 191 * @param visitor - function pointer or functor. 192 */ 193 template <bool ATOMIC, typename VisitorType> IterateOverSetBits(const VisitorType & visitor)194 void IterateOverSetBits(const VisitorType &visitor) 195 { 196 IterateOverSetBitsInRange<ATOMIC, VisitorType>(0, Size(), visitor); 197 } 198 199 /** 200 * @brief Iterates over all bits in range [begin, end) sequentially. 201 * @tparam VisitorType 202 * @param begin - beginning index of the range, inclusive. 203 * @param end - end index of the range, exclusive. 204 * @param visitor - function pointer or functor. 205 */ 206 template <typename VisitorType> IterateOverBitsInRange(size_t begin,size_t end,const VisitorType & visitor)207 void IterateOverBitsInRange(size_t begin, size_t end, const VisitorType &visitor) 208 { 209 CheckBitRange(begin, end); 210 for (size_t i = begin; i < end; ++i) { 211 visitor(i); 212 } 213 } 214 215 /** 216 * @brief Clear all bits in range [begin, end). 217 * @param begin - beginning index of the range, inclusive. 218 * @param end - end index of the range, exclusive. 219 */ 220 void ClearBitsInRange(size_t begin, size_t end); 221 222 /** 223 * @brief Set all bits in range [begin, end). [begin, end) must be within a BitmapWord. 224 * @param begin - beginning index of the range, inclusive. 225 * @param end - end index of the range, exclusive. 226 */ SetRangeWithinWord(size_t begin,size_t end)227 void SetRangeWithinWord(size_t begin, size_t end) 228 { 229 if (LIKELY(begin != end)) { 230 ModifyRangeWithinWord<true>(begin, end); 231 } 232 } 233 234 /** 235 * @brief Clear all bits in range [begin, end). [begin, end) must be within a BitmapWord. 236 * @param begin - beginning index of the range, inclusive. 237 * @param end - end index of the range, exclusive. 238 */ ClearRangeWithinWord(size_t begin,size_t end)239 void ClearRangeWithinWord(size_t begin, size_t end) 240 { 241 if (LIKELY(begin != end)) { 242 ModifyRangeWithinWord<false>(begin, end); 243 } 244 } 245 246 /** 247 * @brief Set all BitmapWords in index range [begin, end). 248 * @param begin - beginning BitmapWord index of the range, inclusive. 249 * @param end - end BitmapWord index of the range, exclusive. 250 */ SetWords(size_t wordBegin,size_t wordEnd)251 void SetWords([[maybe_unused]] size_t wordBegin, [[maybe_unused]] size_t wordEnd) 252 { 253 ASSERT(wordBegin <= wordEnd); 254 if (UNLIKELY(wordBegin == wordEnd)) { 255 return; 256 } 257 memset_s(&bitmap_[wordBegin], (wordEnd - wordBegin) * sizeof(BitmapWordType), ~static_cast<unsigned char>(0), 258 (wordEnd - wordBegin) * sizeof(BitmapWordType)); 259 } 260 261 /** 262 * @brief Clear all BitmapWords in index range [begin, end). 263 * @param begin - beginning BitmapWord index of the range, inclusive. 264 * @param end - end BitmapWord index of the range, exclusive. 265 */ ClearWords(size_t wordBegin,size_t wordEnd)266 void ClearWords([[maybe_unused]] size_t wordBegin, [[maybe_unused]] size_t wordEnd) 267 { 268 ASSERT(wordBegin <= wordEnd); 269 if (UNLIKELY(wordBegin == wordEnd)) { 270 return; 271 } 272 memset_s(&bitmap_[wordBegin], (wordEnd - wordBegin) * sizeof(BitmapWordType), static_cast<unsigned char>(0), 273 (wordEnd - wordBegin) * sizeof(BitmapWordType)); 274 } 275 276 template <bool ATOMIC> FindHighestPrecedingOrSameBit(size_t begin)277 size_t FindHighestPrecedingOrSameBit(size_t begin) 278 { 279 auto wordBeginOffset = GetWordIdx(begin) * BITSPERWORD; 280 auto offsetWithinWord = GetBitIdxWithinWord(begin); 281 auto mask = ~static_cast<BitmapWordType>(0) >> (BITSPERWORD - offsetWithinWord - 1); 282 auto bitmapWord = GetBitmapWord<ATOMIC>(begin) & mask; 283 284 while (true) { 285 if (bitmapWord != 0) { 286 offsetWithinWord = BITSPERWORD - static_cast<size_t>(Clz(bitmapWord)) - 1; 287 return wordBeginOffset + offsetWithinWord; 288 } 289 290 if (wordBeginOffset < BITSPERWORD) { 291 break; 292 } 293 294 wordBeginOffset -= BITSPERWORD; 295 bitmapWord = GetBitmapWord<ATOMIC>(wordBeginOffset); 296 } 297 298 return begin; 299 } 300 Bitmap(BitmapWordType * bitmap,size_t bitsize)301 explicit Bitmap(BitmapWordType *bitmap, size_t bitsize) 302 : bitmap_(bitmap, (AlignUp(bitsize, BITSPERWORD) >> LOG_BITSPERWORD)), bitsize_(bitsize) 303 { 304 } 305 ~Bitmap() = default; 306 // do we need special copy/move constructor? 307 NO_COPY_SEMANTIC(Bitmap); 308 NO_MOVE_SEMANTIC(Bitmap); 309 310 private: 311 Span<BitmapWordType> bitmap_; 312 size_t bitsize_ = 0; 313 314 /** 315 * @brief Compute word index from bit index. 316 * @param bit_offset - bit index. 317 * @return Returns BitmapWord Index of bit_offset. 318 */ GetWordIdx(size_t bitOffset)319 static size_t GetWordIdx(size_t bitOffset) 320 { 321 return bitOffset >> LOG_BITSPERWORD; 322 } 323 324 /** 325 * @brief Compute bit index within a BitmapWord from bit index. 326 * @param bit_offset - bit index. 327 * @return Returns bit index within a BitmapWord. 328 */ GetBitIdxWithinWord(size_t bitOffset)329 size_t GetBitIdxWithinWord(size_t bitOffset) const 330 { 331 CheckBitOffset(bitOffset); 332 constexpr auto BIT_INDEX_MASK = static_cast<size_t>((1ULL << LOG_BITSPERWORD) - 1); 333 return bitOffset & BIT_INDEX_MASK; 334 } 335 336 /** 337 * @brief Compute bit mask from bit index. 338 * @param bit_offset - bit index. 339 * @return Returns bit mask of bit_offset. 340 */ GetBitMask(size_t bitOffset)341 BitmapWordType GetBitMask(size_t bitOffset) const 342 { 343 return 1ULL << GetBitIdxWithinWord(bitOffset); 344 } 345 346 /** 347 * @brief Compute bit mask of range [begin_within_word, end_within_word). 348 * @param begin_within_word - beginning index within word, in range [0, BITSPERWORD). 349 * @param end_within_word - end index within word, in range [0, BITSPERWORD]. Make sure end_within_word is 350 * BITSPERWORD(instead of 0) if you want to cover from certain bit to last. [0, 0) is the only valid case when 351 * end_within_word is 0. 352 * @return Returns bit mask. 353 */ GetRangeBitMask(size_t beginWithinWord,size_t endWithinWord)354 BitmapWordType GetRangeBitMask(size_t beginWithinWord, size_t endWithinWord) const 355 { 356 ASSERT(beginWithinWord < BITSPERWORD); 357 ASSERT(endWithinWord <= BITSPERWORD); 358 ASSERT(beginWithinWord <= endWithinWord); 359 auto endMask = (endWithinWord == BITSPERWORD) ? ~static_cast<BitmapWordType>(0) : GetBitMask(endWithinWord) - 1; 360 return endMask - (GetBitMask(beginWithinWord) - 1); 361 } 362 363 /// @brief Check if bit_offset is valid. CheckBitOffset(size_t bitOffset)364 void CheckBitOffset([[maybe_unused]] size_t bitOffset) const 365 { 366 ASSERT(bitOffset <= Size()); 367 } 368 369 /** 370 * @brief According to SET, set or clear range [begin, end) within a BitmapWord. 371 * @param begin - beginning global bit index. 372 * @param end - end global bit index. 373 */ 374 template <bool SET> ModifyRangeWithinWord(size_t begin,size_t end)375 ALWAYS_INLINE void ModifyRangeWithinWord(size_t begin, size_t end) 376 { 377 CheckBitRange(begin, end); 378 379 if (UNLIKELY(begin == end)) { 380 return; 381 } 382 383 BitmapWordType mask; 384 if (end % BITSPERWORD == 0) { 385 ASSERT(GetWordIdx(end) - GetWordIdx(begin) == 1); 386 mask = GetRangeBitMask(GetBitIdxWithinWord(begin), BITSPERWORD); 387 } else { 388 ASSERT(GetWordIdx(end) == GetWordIdx(begin)); 389 mask = GetRangeBitMask(GetBitIdxWithinWord(begin), GetBitIdxWithinWord(end)); 390 } 391 392 if (SET) { 393 bitmap_[GetWordIdx(begin)] |= mask; 394 } else { 395 bitmap_[GetWordIdx(begin)] &= ~mask; 396 } 397 } 398 399 /// @brief Check if bit range [begin, end) is valid. CheckBitRange(size_t begin,size_t end)400 void CheckBitRange([[maybe_unused]] size_t begin, [[maybe_unused]] size_t end) const 401 { 402 ASSERT(begin < Size()); 403 ASSERT(end <= Size()); 404 ASSERT(begin <= end); 405 } 406 407 template <bool ATOMIC> GetBitmapWord(size_t bitOffset)408 BitmapWordType GetBitmapWord(size_t bitOffset) 409 { 410 auto index = GetWordIdx(bitOffset); 411 if constexpr (!ATOMIC) { 412 return bitmap_[index]; 413 } 414 415 auto *wordAddr = reinterpret_cast<std::atomic<BitmapWordType> *>(&bitmap_[index]); 416 // Atomic with seq_cst order reason: datarace with AtomicTestAndSet 417 return wordAddr->load(std::memory_order_seq_cst); 418 } 419 }; 420 421 /** 422 * Memory bitmap, binding a continuous range of memory to a bitmap. 423 * One bit represents BYTESPERCHUNK bytes of memory. 424 */ 425 template <size_t BYTESPERCHUNK = 1, typename PointerType = ObjectPointerType> 426 class MemBitmap : public Bitmap { 427 public: MemBitmap(void * memAddr,size_t heapSize,void * bitmapAddr)428 explicit MemBitmap(void *memAddr, size_t heapSize, void *bitmapAddr) 429 : Bitmap(static_cast<BitmapWordType *>(bitmapAddr), heapSize / BYTESPERCHUNK), 430 beginAddr_(ToPointerType(memAddr)), 431 endAddr_(beginAddr_ + heapSize) 432 { 433 } 434 NO_COPY_SEMANTIC(MemBitmap); 435 NO_MOVE_SEMANTIC(MemBitmap); 436 437 /** 438 * @brief Reinitialize the MemBitmap for new memory range. 439 * The size of range will be the same as the initial 440 * because we reuse the same bitmap storage. 441 * @param mem_addr - start addr of the new range. 442 */ ReInitializeMemoryRange(void * memAddr)443 void ReInitializeMemoryRange(void *memAddr) 444 { 445 beginAddr_ = ToPointerType(memAddr); 446 endAddr_ = beginAddr_ + MemSizeInBytes(); 447 Bitmap::ClearAllBits(); 448 } 449 GetBitMapSizeInByte(size_t heapSize)450 inline static constexpr size_t GetBitMapSizeInByte(size_t heapSize) 451 { 452 ASSERT(heapSize % BYTESPERCHUNK == 0); 453 size_t bitSize = heapSize / BYTESPERCHUNK; 454 return (AlignUp(bitSize, BITSPERWORD) >> Bitmap::LOG_BITSPERWORD) * sizeof(BitmapWordType); 455 } 456 457 ~MemBitmap() = default; 458 MemSizeInBytes()459 size_t MemSizeInBytes() const 460 { 461 return Size() * BYTESPERCHUNK; 462 } 463 GetHeapRange()464 inline std::pair<uintptr_t, uintptr_t> GetHeapRange() 465 { 466 return {beginAddr_, endAddr_}; 467 } 468 469 /** 470 * @brief Set bit corresponding to addr. 471 * @param addr - addr must be aligned to BYTESPERCHUNK. 472 */ Set(void * addr)473 void Set(void *addr) 474 { 475 CheckAddrValidity(addr); 476 SetBit(AddrToBitOffset(ToPointerType(addr))); 477 } 478 479 /** 480 * @brief Clear bit corresponding to addr. 481 * @param addr - addr must be aligned to BYTESPERCHUNK. 482 */ Clear(void * addr)483 void Clear(void *addr) 484 { 485 CheckAddrValidity(addr); 486 ClearBit(AddrToBitOffset(ToPointerType(addr))); 487 } 488 489 /// @brief Clear bits corresponding to addr range [begin, end). ClearRange(void * begin,void * end)490 ALWAYS_INLINE void ClearRange(void *begin, void *end) 491 { 492 CheckHalfClosedHalfOpenAddressRange(begin, end); 493 ClearBitsInRange(AddrToBitOffset(ToPointerType(begin)), EndAddrToBitOffset(ToPointerType(end))); 494 } 495 496 /** 497 * @brief Test bit corresponding to addr. 498 * @param addr - addr must be aligned to BYTESPERCHUNK. 499 */ Test(const void * addr)500 bool Test(const void *addr) const 501 { 502 CheckAddrValidity(addr); 503 return TestBit(AddrToBitOffset(ToPointerType(addr))); 504 } 505 506 /** 507 * @brief Test bit corresponding to addr if addr is valid. 508 * @return value of indexed bit if addr is valid. If addr is invalid then false 509 */ TestIfAddrValid(const void * addr)510 bool TestIfAddrValid(const void *addr) const 511 { 512 if (IsAddrValid(addr)) { 513 return TestBit(AddrToBitOffset(ToPointerType(addr))); 514 } 515 return false; 516 } 517 518 /** 519 * @brief Atomically set bit corresponding to addr. If the bit is not set, set it atomically. Otherwise, do nothing. 520 * @param addr - addr must be aligned to BYTESPERCHUNK. 521 * @return Returns old value of the bit. 522 */ AtomicTestAndSet(void * addr)523 bool AtomicTestAndSet(void *addr) 524 { 525 CheckAddrValidity(addr); 526 return AtomicTestAndSetBit(AddrToBitOffset(ToPointerType(addr))); 527 } 528 529 /** 530 * @brief Atomically clear bit corresponding to addr. If the bit is set, clear it atomically. Otherwise, do nothing. 531 * @param addr - addr must be aligned to BYTESPERCHUNK. 532 * @return Returns old value of the bit. 533 */ AtomicTestAndClear(void * addr)534 bool AtomicTestAndClear(void *addr) 535 { 536 CheckAddrValidity(addr); 537 return AtomicTestAndClearBit(AddrToBitOffset(ToPointerType(addr))); 538 } 539 540 /** 541 * @brief Atomically test bit corresponding to addr. 542 * @param addr - addr must be aligned to BYTESPERCHUNK. 543 * @return Returns the value of the bit. 544 */ AtomicTest(const void * addr)545 bool AtomicTest(const void *addr) 546 { 547 CheckAddrValidity(addr); 548 return AtomicTestBit(AddrToBitOffset(ToPointerType(addr))); 549 } 550 551 /// @brief Find first marked chunk. 552 template <bool ATOMIC = false> FindFirstMarkedChunks()553 void *FindFirstMarkedChunks() 554 { 555 void *firstMarked = nullptr; 556 IterateOverSetBits<ATOMIC>([&firstMarked, this](size_t bitOffset) { 557 firstMarked = BitOffsetToAddr(bitOffset); 558 return false; 559 }); 560 return firstMarked; 561 } 562 563 /// @brief Iterates over marked chunks of memory sequentially. 564 template <bool ATOMIC = false, typename MemVisitor> IterateOverMarkedChunks(const MemVisitor & visitor)565 void IterateOverMarkedChunks(const MemVisitor &visitor) 566 { 567 IterateOverSetBits<ATOMIC>([&visitor, this](size_t bitOffset) { 568 visitor(BitOffsetToAddr(bitOffset)); 569 return true; 570 }); 571 } 572 573 /// @brief Iterates over all chunks of memory sequentially. 574 template <typename MemVisitor> IterateOverChunks(const MemVisitor & visitor)575 void IterateOverChunks(const MemVisitor &visitor) 576 { 577 IterateOverBits([&visitor, this](size_t bitOffset) { visitor(BitOffsetToAddr(bitOffset)); }); 578 } 579 580 /// @brief Iterates over marked chunks of memory in range [begin, end) sequentially. 581 template <bool ATOMIC = false, typename MemVisitor, typename T = void *> IterateOverMarkedChunkInRange(void * begin,void * end,const MemVisitor & visitor)582 void IterateOverMarkedChunkInRange(void *begin, void *end, const MemVisitor &visitor) 583 { 584 IterateOverMarkedChunkInRangeInterruptible<ATOMIC>(begin, end, [&visitor](void *obj) { 585 visitor(obj); 586 return true; 587 }); 588 } 589 590 /** 591 * @brief Iterates over marked chunks of memory in range [begin, end) sequentially and stops iteration if visitor 592 * returns false 593 */ 594 template <bool ATOMIC = false, typename MemVisitor, typename T = void *> IterateOverMarkedChunkInRangeInterruptible(void * begin,void * end,const MemVisitor & visitor)595 void IterateOverMarkedChunkInRangeInterruptible(void *begin, void *end, const MemVisitor &visitor) 596 { 597 CheckHalfClosedHalfOpenAddressRange(begin, end); 598 IterateOverSetBitsInRange<ATOMIC>(FindHighestPrecedingOrSameBit<ATOMIC>(AddrToBitOffset(ToPointerType(begin))), 599 EndAddrToBitOffset(ToPointerType(end)), [&visitor, this](size_t bitOffset) { 600 return visitor(BitOffsetToAddr(bitOffset)); 601 }); 602 } 603 604 /// @brief Call visitor for single allocated object in humongous region 605 template <bool ATOMIC, typename MemVisitor> CallForMarkedChunkInHumongousRegion(void * begin,const MemVisitor & visitor)606 void CallForMarkedChunkInHumongousRegion(void *begin, const MemVisitor &visitor) 607 { 608 bool marked; 609 if constexpr (ATOMIC) { 610 marked = AtomicTest(begin); 611 } else { 612 marked = Test(begin); 613 } 614 if (marked) { 615 visitor(begin); 616 } 617 } 618 619 /// @brief Iterates over all chunks of memory in range [begin, end) sequentially. 620 template <typename MemVisitor> IterateOverChunkInRange(void * begin,void * end,const MemVisitor & visitor)621 void IterateOverChunkInRange(void *begin, void *end, const MemVisitor &visitor) 622 { 623 CheckHalfClosedHalfOpenAddressRange(begin, end); 624 IterateOverBitsInRange(AddrToBitOffset(ToPointerType(begin)), EndAddrToBitOffset(ToPointerType(end)), 625 [&visitor, this](size_t bitOffset) { visitor(BitOffsetToAddr(bitOffset)); }); 626 } 627 IsAddrInRange(const void * addr)628 bool IsAddrInRange(const void *addr) const 629 { 630 return addr >= ToVoidPtr(beginAddr_) && addr < ToVoidPtr(endAddr_); 631 } 632 633 template <class T> ToPointerType(T * val)634 static constexpr PointerType ToPointerType(T *val) 635 { 636 return static_cast<PointerType>(ToUintPtr(val)); 637 } 638 639 private: 640 /// @brief Computes bit offset from addr. AddrToBitOffset(PointerType addr)641 size_t AddrToBitOffset(PointerType addr) const 642 { 643 return (addr - beginAddr_) / BYTESPERCHUNK; 644 } 645 EndAddrToBitOffset(PointerType addr)646 size_t EndAddrToBitOffset(PointerType addr) const 647 { 648 return (AlignUp(addr, BYTESPERCHUNK) - beginAddr_) / BYTESPERCHUNK; 649 } 650 651 /// @brief Computes address from bit offset. BitOffsetToAddr(size_t bitOffset)652 void *BitOffsetToAddr(size_t bitOffset) const 653 { 654 return ToVoidPtr(beginAddr_ + bitOffset * BYTESPERCHUNK); 655 } 656 657 /// @brief Check if addr is valid. CheckAddrValidity(const void * addr)658 void CheckAddrValidity([[maybe_unused]] const void *addr) const 659 { 660 ASSERT(IsAddrInRange(addr)); 661 ASSERT((ToPointerType(addr) - beginAddr_) % BYTESPERCHUNK == 0); 662 } 663 664 /** 665 * @brief Check if addr is valid with returned value. 666 * @return true if addr is valid 667 */ IsAddrValid(const void * addr)668 bool IsAddrValid(const void *addr) const 669 { 670 return IsAddrInRange(addr) && (ToPointerType(addr) - beginAddr_) % BYTESPERCHUNK == 0; 671 } 672 673 /// @brief Check if [begin, end) is a valid address range. CheckHalfClosedHalfOpenAddressRange(void * begin,void * end)674 void CheckHalfClosedHalfOpenAddressRange([[maybe_unused]] void *begin, [[maybe_unused]] void *end) const 675 { 676 CheckAddrValidity(begin); 677 ASSERT(ToPointerType(end) >= beginAddr_); 678 ASSERT(ToPointerType(end) <= endAddr_); 679 ASSERT(ToPointerType(begin) <= ToPointerType(end)); 680 } 681 682 PointerType beginAddr_ {0}; 683 PointerType endAddr_ {0}; 684 }; 685 686 using MarkBitmap = MemBitmap<DEFAULT_ALIGNMENT_IN_BYTES>; 687 688 } // namespace ark::mem 689 #endif // PANDA_MEM_GC_BITMAP_H 690