1 // Copyright 2020 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef V8_HEAP_CPPGC_OBJECT_START_BITMAP_H_
6 #define V8_HEAP_CPPGC_OBJECT_START_BITMAP_H_
7
8 #include <limits.h>
9 #include <stdint.h>
10
11 #include <array>
12
13 #include "include/cppgc/internal/process-heap.h"
14 #include "src/base/atomic-utils.h"
15 #include "src/base/bits.h"
16 #include "src/base/macros.h"
17 #include "src/heap/cppgc/globals.h"
18 #include "src/heap/cppgc/heap-object-header.h"
19
20 namespace cppgc {
21 namespace internal {
22
23 // A bitmap for recording object starts. Objects have to be allocated at
24 // minimum granularity of kGranularity.
25 //
26 // Depends on internals such as:
27 // - kBlinkPageSize
28 // - kAllocationGranularity
29 //
30 // ObjectStartBitmap supports concurrent reads from multiple threads but
31 // only a single mutator thread can write to it.
32 class V8_EXPORT_PRIVATE ObjectStartBitmap {
33 public:
34 // Granularity of addresses added to the bitmap.
Granularity()35 static constexpr size_t Granularity() { return kAllocationGranularity; }
36
37 // Maximum number of entries in the bitmap.
MaxEntries()38 static constexpr size_t MaxEntries() {
39 return kReservedForBitmap * kBitsPerCell;
40 }
41
42 explicit inline ObjectStartBitmap(Address offset);
43
44 // Finds an object header based on a
45 // address_maybe_pointing_to_the_middle_of_object. Will search for an object
46 // start in decreasing address order.
47 template <AccessMode = AccessMode::kNonAtomic>
48 inline HeapObjectHeader* FindHeader(
49 ConstAddress address_maybe_pointing_to_the_middle_of_object) const;
50
51 template <AccessMode = AccessMode::kNonAtomic>
52 inline void SetBit(ConstAddress);
53 template <AccessMode = AccessMode::kNonAtomic>
54 inline void ClearBit(ConstAddress);
55 template <AccessMode = AccessMode::kNonAtomic>
56 inline bool CheckBit(ConstAddress) const;
57
58 // Iterates all object starts recorded in the bitmap.
59 //
60 // The callback is of type
61 // void(Address)
62 // and is passed the object start address as parameter.
63 template <typename Callback>
64 inline void Iterate(Callback) const;
65
66 // Clear the object start bitmap.
67 inline void Clear();
68
69 private:
70 template <AccessMode = AccessMode::kNonAtomic>
71 inline void store(size_t cell_index, uint8_t value);
72 template <AccessMode = AccessMode::kNonAtomic>
73 inline uint8_t load(size_t cell_index) const;
74
75 static constexpr size_t kBitsPerCell = sizeof(uint8_t) * CHAR_BIT;
76 static constexpr size_t kCellMask = kBitsPerCell - 1;
77 static constexpr size_t kBitmapSize =
78 (kPageSize + ((kBitsPerCell * kAllocationGranularity) - 1)) /
79 (kBitsPerCell * kAllocationGranularity);
80 static constexpr size_t kReservedForBitmap =
81 ((kBitmapSize + kAllocationMask) & ~kAllocationMask);
82
83 inline void ObjectStartIndexAndBit(ConstAddress, size_t*, size_t*) const;
84
85 const Address offset_;
86 // The bitmap contains a bit for every kGranularity aligned address on a
87 // a NormalPage, i.e., for a page of size kBlinkPageSize.
88 std::array<uint8_t, kReservedForBitmap> object_start_bit_map_;
89 };
90
ObjectStartBitmap(Address offset)91 ObjectStartBitmap::ObjectStartBitmap(Address offset) : offset_(offset) {
92 Clear();
93 }
94
95 template <AccessMode mode>
FindHeader(ConstAddress address_maybe_pointing_to_the_middle_of_object)96 HeapObjectHeader* ObjectStartBitmap::FindHeader(
97 ConstAddress address_maybe_pointing_to_the_middle_of_object) const {
98 DCHECK_LE(offset_, address_maybe_pointing_to_the_middle_of_object);
99 size_t object_offset =
100 address_maybe_pointing_to_the_middle_of_object - offset_;
101 size_t object_start_number = object_offset / kAllocationGranularity;
102 size_t cell_index = object_start_number / kBitsPerCell;
103 DCHECK_GT(object_start_bit_map_.size(), cell_index);
104 const size_t bit = object_start_number & kCellMask;
105 uint8_t byte = load<mode>(cell_index) & ((1 << (bit + 1)) - 1);
106 while (!byte && cell_index) {
107 DCHECK_LT(0u, cell_index);
108 byte = load<mode>(--cell_index);
109 }
110 const int leading_zeroes = v8::base::bits::CountLeadingZeros(byte);
111 object_start_number =
112 (cell_index * kBitsPerCell) + (kBitsPerCell - 1) - leading_zeroes;
113 object_offset = object_start_number * kAllocationGranularity;
114 return reinterpret_cast<HeapObjectHeader*>(object_offset + offset_);
115 }
116
117 template <AccessMode mode>
SetBit(ConstAddress header_address)118 void ObjectStartBitmap::SetBit(ConstAddress header_address) {
119 size_t cell_index, object_bit;
120 ObjectStartIndexAndBit(header_address, &cell_index, &object_bit);
121 // Only a single mutator thread can write to the bitmap, so no need for CAS.
122 store<mode>(cell_index,
123 static_cast<uint8_t>(load(cell_index) | (1 << object_bit)));
124 }
125
126 template <AccessMode mode>
ClearBit(ConstAddress header_address)127 void ObjectStartBitmap::ClearBit(ConstAddress header_address) {
128 size_t cell_index, object_bit;
129 ObjectStartIndexAndBit(header_address, &cell_index, &object_bit);
130 store<mode>(cell_index,
131 static_cast<uint8_t>(load(cell_index) & ~(1 << object_bit)));
132 }
133
134 template <AccessMode mode>
CheckBit(ConstAddress header_address)135 bool ObjectStartBitmap::CheckBit(ConstAddress header_address) const {
136 size_t cell_index, object_bit;
137 ObjectStartIndexAndBit(header_address, &cell_index, &object_bit);
138 return load<mode>(cell_index) & (1 << object_bit);
139 }
140
141 template <AccessMode mode>
store(size_t cell_index,uint8_t value)142 void ObjectStartBitmap::store(size_t cell_index, uint8_t value) {
143 if (mode == AccessMode::kNonAtomic) {
144 object_start_bit_map_[cell_index] = value;
145 return;
146 }
147 v8::base::AsAtomicPtr(&object_start_bit_map_[cell_index])
148 ->store(value, std::memory_order_release);
149 }
150
151 template <AccessMode mode>
load(size_t cell_index)152 uint8_t ObjectStartBitmap::load(size_t cell_index) const {
153 if (mode == AccessMode::kNonAtomic) {
154 return object_start_bit_map_[cell_index];
155 }
156 return v8::base::AsAtomicPtr(&object_start_bit_map_[cell_index])
157 ->load(std::memory_order_acquire);
158 }
159
ObjectStartIndexAndBit(ConstAddress header_address,size_t * cell_index,size_t * bit)160 void ObjectStartBitmap::ObjectStartIndexAndBit(ConstAddress header_address,
161 size_t* cell_index,
162 size_t* bit) const {
163 const size_t object_offset = header_address - offset_;
164 DCHECK(!(object_offset & kAllocationMask));
165 const size_t object_start_number = object_offset / kAllocationGranularity;
166 *cell_index = object_start_number / kBitsPerCell;
167 DCHECK_GT(kBitmapSize, *cell_index);
168 *bit = object_start_number & kCellMask;
169 }
170
171 template <typename Callback>
Iterate(Callback callback)172 inline void ObjectStartBitmap::Iterate(Callback callback) const {
173 for (size_t cell_index = 0; cell_index < kReservedForBitmap; cell_index++) {
174 if (!object_start_bit_map_[cell_index]) continue;
175
176 uint8_t value = object_start_bit_map_[cell_index];
177 while (value) {
178 const int trailing_zeroes = v8::base::bits::CountTrailingZeros(value);
179 const size_t object_start_number =
180 (cell_index * kBitsPerCell) + trailing_zeroes;
181 const Address object_address =
182 offset_ + (kAllocationGranularity * object_start_number);
183 callback(object_address);
184 // Clear current object bit in temporary value to advance iteration.
185 value &= ~(1 << (object_start_number & kCellMask));
186 }
187 }
188 }
189
Clear()190 void ObjectStartBitmap::Clear() {
191 std::fill(object_start_bit_map_.begin(), object_start_bit_map_.end(), 0);
192 }
193
194 // A platform aware version of ObjectStartBitmap to provide platform specific
195 // optimizations (e.g. Use non-atomic stores on ARMv7 when not marking).
196 class V8_EXPORT_PRIVATE PlatformAwareObjectStartBitmap
197 : public ObjectStartBitmap {
198 public:
199 explicit inline PlatformAwareObjectStartBitmap(Address offset);
200
201 template <AccessMode = AccessMode::kNonAtomic>
202 inline void SetBit(ConstAddress);
203 template <AccessMode = AccessMode::kNonAtomic>
204 inline void ClearBit(ConstAddress);
205
206 private:
207 template <AccessMode>
208 static bool ShouldForceNonAtomic();
209 };
210
PlatformAwareObjectStartBitmap(Address offset)211 PlatformAwareObjectStartBitmap::PlatformAwareObjectStartBitmap(Address offset)
212 : ObjectStartBitmap(offset) {}
213
214 // static
215 template <AccessMode mode>
ShouldForceNonAtomic()216 bool PlatformAwareObjectStartBitmap::ShouldForceNonAtomic() {
217 #if defined(V8_TARGET_ARCH_ARM)
218 // Use non-atomic accesses on ARMv7 when marking is not active.
219 if (mode == AccessMode::kAtomic) {
220 if (V8_LIKELY(!ProcessHeap::IsAnyIncrementalOrConcurrentMarking()))
221 return true;
222 }
223 #endif // defined(V8_TARGET_ARCH_ARM)
224 return false;
225 }
226
227 template <AccessMode mode>
SetBit(ConstAddress header_address)228 void PlatformAwareObjectStartBitmap::SetBit(ConstAddress header_address) {
229 if (ShouldForceNonAtomic<mode>()) {
230 ObjectStartBitmap::SetBit<AccessMode::kNonAtomic>(header_address);
231 return;
232 }
233 ObjectStartBitmap::SetBit<mode>(header_address);
234 }
235
236 template <AccessMode mode>
ClearBit(ConstAddress header_address)237 void PlatformAwareObjectStartBitmap::ClearBit(ConstAddress header_address) {
238 if (ShouldForceNonAtomic<mode>()) {
239 ObjectStartBitmap::ClearBit<AccessMode::kNonAtomic>(header_address);
240 return;
241 }
242 ObjectStartBitmap::ClearBit<mode>(header_address);
243 }
244
245 } // namespace internal
246 } // namespace cppgc
247
248 #endif // V8_HEAP_CPPGC_OBJECT_START_BITMAP_H_
249