1 /**
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef RUNTIME_MEM_GC_CARD_TABLE_INL_H
17 #define RUNTIME_MEM_GC_CARD_TABLE_INL_H
18
19 #include "runtime/mem/gc/card_table.h"
20 #include "runtime/include/mem/panda_containers.h"
21
22 #include <atomic>
23
24 namespace ark::mem {
25
GetCard()26 inline uint8_t CardTable::Card::GetCard() const
27 {
28 // Atomic with relaxed order reason: data race with value_ with no synchronization or ordering constraints imposed
29 // on other reads or writes
30 return value_.load(std::memory_order_relaxed);
31 }
32
SetCard(uint8_t newVal)33 inline void CardTable::Card::SetCard(uint8_t newVal)
34 {
35 // Atomic with relaxed order reason: data race with value_ with no synchronization or ordering constraints imposed
36 // on other reads or writes
37 value_.store(newVal, std::memory_order_relaxed);
38 }
39
GetStatus()40 inline CardTable::Card::Status CardTable::Card::GetStatus() const
41 {
42 // Atomic with relaxed order reason: data race with value_ with no synchronization or ordering constraints imposed
43 // on other reads or writes
44 return value_.load(std::memory_order_relaxed) & STATUS_MASK;
45 }
46
47 // CC-OFFNXT(G.FUD.06) perf critical
FillRanges(PandaVector<MemRange> * ranges,const Card * startCard,const Card * endCard)48 inline void CardTable::FillRanges(PandaVector<MemRange> *ranges, const Card *startCard, const Card *endCard)
49 {
50 constexpr size_t MIN_RANGE = 32;
51 constexpr size_t MAX_CARDS_COUNT = 1000; // How many cards we can process at once
52 static std::array<char, MAX_CARDS_COUNT> zeroArray {};
53
54 if (static_cast<size_t>(endCard - startCard) < MIN_RANGE) {
55 for (auto cardPtr = startCard; cardPtr <= endCard; cardPtr++) {
56 if (cardPtr->IsMarked()) {
57 ranges->emplace_back(minAddress_ + (cardPtr - cards_) * CARD_SIZE,
58 minAddress_ + (cardPtr - cards_ + 1) * CARD_SIZE - 1);
59 }
60 }
61 } else {
62 size_t diff = endCard - startCard + 1;
63 size_t splitSize = std::min(diff / 2, MAX_CARDS_COUNT); // divide 2 to get smaller split_size
64 if (memcmp(startCard, &zeroArray, splitSize) != 0) {
65 FillRanges(ranges, startCard, ToNativePtr<Card>(ToUintPtr(startCard) + splitSize - 1));
66 }
67 // NOLINTNEXTLINE(bugprone-branch-clone)
68 if (diff - splitSize > MAX_CARDS_COUNT) {
69 FillRanges(ranges, ToNativePtr<Card>(ToUintPtr(startCard) + splitSize), endCard);
70 } else if (memcmp(ToNativePtr<Card>(ToUintPtr(startCard) + splitSize), &zeroArray, diff - splitSize) != 0) {
71 FillRanges(ranges, ToNativePtr<Card>(ToUintPtr(startCard) + splitSize), endCard);
72 }
73 }
74 }
75
76 // Make sure we can treat size_t as lockfree atomic
77 static_assert(std::atomic_size_t::is_always_lock_free);
78 static_assert(sizeof(std::atomic_size_t) == sizeof(size_t));
79
80 template <typename CardVisitor>
VisitMarked(CardVisitor cardVisitor,uint32_t processedFlag)81 void CardTable::VisitMarked(CardVisitor cardVisitor, uint32_t processedFlag)
82 {
83 bool visitMarked = processedFlag & CardTableProcessedFlag::VISIT_MARKED;
84 bool visitProcessed = processedFlag & CardTableProcessedFlag::VISIT_PROCESSED;
85 bool setProcessed = processedFlag & CardTableProcessedFlag::SET_PROCESSED;
86 auto *card = cards_;
87 auto *cardEnd = cards_ + (cardsCount_ / CHUNK_CARD_NUM) * CHUNK_CARD_NUM;
88 while (card < cardEnd) {
89 // NB! In general wide load/short store on overlapping memory of different address are allowed to be reordered
90 // This optimization currently is allowed since additional VisitMarked is called after concurrent mark with
91 // global Mutator lock held, so all previous managed thread's writes should be visible by GC thread
92 // Atomic with relaxed order reason: data race with card with no synchronization or ordering constraints imposed
93 // on other reads or writes
94 if (LIKELY((reinterpret_cast<std::atomic_size_t *>(card))->load(std::memory_order_relaxed) == 0)) {
95 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
96 card += CHUNK_CARD_NUM;
97 continue;
98 }
99 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
100 auto *chunkEnd = card + CHUNK_CARD_NUM;
101 while (card < chunkEnd) {
102 auto cardStatus = card->GetStatus();
103 if (!(visitMarked && Card::IsMarked(cardStatus)) && !(visitProcessed && Card::IsProcessed(cardStatus))) {
104 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
105 ++card;
106 continue;
107 }
108
109 if (setProcessed) {
110 card->SetProcessed();
111 }
112 cardVisitor(GetMemoryRange(card));
113 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
114 ++card;
115 }
116 }
117 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
118 for (; card < cards_ + cardsCount_; ++card) {
119 auto cardStatus = card->GetStatus();
120 if ((visitMarked && Card::IsMarked(cardStatus)) || (visitProcessed && Card::IsProcessed(cardStatus))) {
121 if (setProcessed) {
122 card->SetProcessed();
123 }
124 cardVisitor(GetMemoryRange(card));
125 }
126 }
127 }
128
129 template <typename CardVisitor>
VisitMarkedCompact(CardVisitor cardVisitor)130 void CardTable::VisitMarkedCompact(CardVisitor cardVisitor)
131 {
132 constexpr size_t MAX_CARDS_COUNT = 1000;
133 size_t curPos = 0;
134 size_t endPos = 0;
135 PandaVector<MemRange> memRanges;
136
137 ASSERT(cardsCount_ > 0);
138 auto maxPoolAddress = PoolManager::GetMmapMemPool()->GetMaxObjectAddress();
139 while (curPos < cardsCount_) {
140 endPos = std::min(curPos + MAX_CARDS_COUNT - 1, cardsCount_ - 1);
141 FillRanges(&memRanges, &cards_[curPos], &cards_[endPos]);
142 curPos = endPos + 1;
143 if (GetCardStartAddress(&cards_[curPos]) > maxPoolAddress) {
144 break;
145 }
146 }
147 for (const auto &memRange : memRanges) {
148 cardVisitor(memRange);
149 }
150 }
151
152 #ifndef NDEBUG
IsClear()153 inline bool CardTable::IsClear()
154 {
155 bool clear = true;
156 VisitMarked(
157 [&clear](const MemRange &range) {
158 LOG(ERROR, GC) << "Card [" << ToVoidPtr(range.GetStartAddress()) << " - "
159 << ToVoidPtr(range.GetEndAddress()) << "] is not clear";
160 clear = false;
161 },
162 CardTableProcessedFlag::VISIT_MARKED | CardTableProcessedFlag::VISIT_PROCESSED);
163 return clear;
164 }
165 #endif
166 } // namespace ark::mem
167
168 #endif // RUNTIME_MEM_GC_CARD_TABLE_INL_H
169