1 /*
2 * Copyright (c) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef ECMASCRIPT_MEM_HEAP_INL_H
17 #define ECMASCRIPT_MEM_HEAP_INL_H
18
19 #include "ecmascript/mem/heap.h"
20
21 #include "ecmascript/ecma_vm.h"
22 #include "ecmascript/hprof/heap_tracker.h"
23 #include "ecmascript/mem/allocator-inl.h"
24 #include "ecmascript/mem/concurrent_sweeper.h"
25 #include "ecmascript/mem/linear_space.h"
26 #include "ecmascript/mem/mem_controller.h"
27 #include "ecmascript/mem/remembered_set.h"
28 #include "ecmascript/mem/sparse_space.h"
29 #include "ecmascript/mem/tagged_object.h"
30
31 namespace panda::ecmascript {
32 template<class Callback>
EnumerateOldSpaceRegions(const Callback & cb,Region * region)33 void Heap::EnumerateOldSpaceRegions(const Callback &cb, Region *region) const
34 {
35 oldSpace_->EnumerateRegions(cb, region);
36 nonMovableSpace_->EnumerateRegions(cb);
37 hugeObjectSpace_->EnumerateRegions(cb);
38 machineCodeSpace_->EnumerateRegions(cb);
39 }
40
41 template<class Callback>
EnumerateSnapShotSpaceRegions(const Callback & cb)42 void Heap::EnumerateSnapShotSpaceRegions(const Callback &cb) const
43 {
44 snapshotSpace_->EnumerateRegions(cb);
45 }
46
47 template<class Callback>
EnumerateNonNewSpaceRegions(const Callback & cb)48 void Heap::EnumerateNonNewSpaceRegions(const Callback &cb) const
49 {
50 oldSpace_->EnumerateRegions(cb);
51 oldSpace_->EnumerateCollectRegionSet(cb);
52 snapshotSpace_->EnumerateRegions(cb);
53 nonMovableSpace_->EnumerateRegions(cb);
54 hugeObjectSpace_->EnumerateRegions(cb);
55 machineCodeSpace_->EnumerateRegions(cb);
56 }
57
58 template<class Callback>
EnumerateNewSpaceRegions(const Callback & cb)59 void Heap::EnumerateNewSpaceRegions(const Callback &cb) const
60 {
61 toSpace_->EnumerateRegions(cb);
62 }
63
64 template<class Callback>
EnumerateNonMovableRegions(const Callback & cb)65 void Heap::EnumerateNonMovableRegions(const Callback &cb) const
66 {
67 snapshotSpace_->EnumerateRegions(cb);
68 nonMovableSpace_->EnumerateRegions(cb);
69 hugeObjectSpace_->EnumerateRegions(cb);
70 machineCodeSpace_->EnumerateRegions(cb);
71 }
72
73 template<class Callback>
EnumerateRegions(const Callback & cb)74 void Heap::EnumerateRegions(const Callback &cb) const
75 {
76 toSpace_->EnumerateRegions(cb);
77 oldSpace_->EnumerateRegions(cb);
78 oldSpace_->EnumerateCollectRegionSet(cb);
79 snapshotSpace_->EnumerateRegions(cb);
80 nonMovableSpace_->EnumerateRegions(cb);
81 hugeObjectSpace_->EnumerateRegions(cb);
82 machineCodeSpace_->EnumerateRegions(cb);
83 }
84
85 template<class Callback>
IteratorOverObjects(const Callback & cb)86 void Heap::IteratorOverObjects(const Callback &cb) const
87 {
88 toSpace_->IterateOverObjects(cb);
89 oldSpace_->IterateOverObjects(cb);
90 nonMovableSpace_->IterateOverObjects(cb);
91 hugeObjectSpace_->IterateOverObjects(cb);
92 }
93
AllocateYoungOrHugeObject(JSHClass * hclass)94 TaggedObject *Heap::AllocateYoungOrHugeObject(JSHClass *hclass)
95 {
96 size_t size = hclass->GetObjectSize();
97 return AllocateYoungOrHugeObject(hclass, size);
98 }
99
AllocateYoungOrHugeObject(JSHClass * hclass,size_t size)100 TaggedObject *Heap::AllocateYoungOrHugeObject(JSHClass *hclass, size_t size)
101 {
102 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
103 if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) {
104 return AllocateHugeObject(hclass, size);
105 }
106
107 auto object = reinterpret_cast<TaggedObject *>(toSpace_->Allocate(size));
108 if (object == nullptr) {
109 CollectGarbage(SelectGCType());
110 object = reinterpret_cast<TaggedObject *>(toSpace_->Allocate(size));
111 if (object == nullptr) {
112 CollectGarbage(SelectGCType());
113 object = reinterpret_cast<TaggedObject *>(toSpace_->Allocate(size));
114 if (UNLIKELY(object == nullptr)) {
115 ThrowOutOfMemoryError(size, "AllocateYoungObject");
116 UNREACHABLE();
117 }
118 }
119 }
120
121 object->SetClass(hclass);
122 OnAllocateEvent(reinterpret_cast<uintptr_t>(object));
123 return object;
124 }
125
AllocateYoungSync(size_t size)126 uintptr_t Heap::AllocateYoungSync(size_t size)
127 {
128 return toSpace_->AllocateSync(size);
129 }
130
MoveYoungRegionSync(Region * region)131 bool Heap::MoveYoungRegionSync(Region *region)
132 {
133 return toSpace_->SwapRegion(region, fromSpace_);
134 }
135
MergeToOldSpaceSync(LocalSpace * localSpace)136 void Heap::MergeToOldSpaceSync(LocalSpace *localSpace)
137 {
138 oldSpace_->Merge(localSpace);
139 }
140
TryAllocateYoungGeneration(JSHClass * hclass,size_t size)141 TaggedObject *Heap::TryAllocateYoungGeneration(JSHClass *hclass, size_t size)
142 {
143 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
144 if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) {
145 return nullptr;
146 }
147 auto object = reinterpret_cast<TaggedObject *>(toSpace_->Allocate(size));
148 if (object != nullptr) {
149 object->SetClass(hclass);
150 }
151 return object;
152 }
153
AllocateOldOrHugeObject(JSHClass * hclass)154 TaggedObject *Heap::AllocateOldOrHugeObject(JSHClass *hclass)
155 {
156 size_t size = hclass->GetObjectSize();
157 return AllocateOldOrHugeObject(hclass, size);
158 }
159
AllocateOldOrHugeObject(JSHClass * hclass,size_t size)160 TaggedObject *Heap::AllocateOldOrHugeObject(JSHClass *hclass, size_t size)
161 {
162 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
163 if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) {
164 return AllocateHugeObject(hclass, size);
165 }
166 auto object = reinterpret_cast<TaggedObject *>(oldSpace_->Allocate(size));
167 if (UNLIKELY(object == 0)) {
168 ThrowOutOfMemoryError(size, "AllocateOldGenerationOrHugeObject");
169 UNREACHABLE();
170 }
171 object->SetClass(hclass);
172 OnAllocateEvent(reinterpret_cast<uintptr_t>(object));
173 return object;
174 }
175
AllocateNonMovableOrHugeObject(JSHClass * hclass)176 TaggedObject *Heap::AllocateNonMovableOrHugeObject(JSHClass *hclass)
177 {
178 size_t size = hclass->GetObjectSize();
179 return AllocateNonMovableOrHugeObject(hclass, size);
180 }
181
AllocateNonMovableOrHugeObject(JSHClass * hclass,size_t size)182 TaggedObject *Heap::AllocateNonMovableOrHugeObject(JSHClass *hclass, size_t size)
183 {
184 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
185 if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) {
186 return AllocateHugeObject(hclass, size);
187 }
188 auto object = reinterpret_cast<TaggedObject *>(nonMovableSpace_->Allocate(size));
189 if (UNLIKELY(object == nullptr)) {
190 ThrowOutOfMemoryError(size, "AllocateNonMovableOrHugeObject");
191 UNREACHABLE();
192 }
193 object->SetClass(hclass);
194 OnAllocateEvent(reinterpret_cast<uintptr_t>(object));
195 return object;
196 }
197
AllocateDynClassClass(JSHClass * hclass,size_t size)198 TaggedObject *Heap::AllocateDynClassClass(JSHClass *hclass, size_t size)
199 {
200 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
201 auto object = reinterpret_cast<TaggedObject *>(nonMovableSpace_->Allocate(size));
202 if (UNLIKELY(object == nullptr)) {
203 LOG_ECMA_MEM(FATAL) << "Heap::AllocateDynClassClass can not allocate any space";
204 }
205 *reinterpret_cast<MarkWordType *>(ToUintPtr(object)) = reinterpret_cast<MarkWordType>(hclass);
206 OnAllocateEvent(reinterpret_cast<uintptr_t>(object));
207 return object;
208 }
209
AllocateHugeObject(JSHClass * hclass,size_t size)210 TaggedObject *Heap::AllocateHugeObject(JSHClass *hclass, size_t size)
211 {
212 auto *object = reinterpret_cast<TaggedObject *>(hugeObjectSpace_->Allocate(size));
213 if (UNLIKELY(object == nullptr)) {
214 CollectGarbage(TriggerGCType::OLD_GC);
215 object = reinterpret_cast<TaggedObject *>(hugeObjectSpace_->Allocate(size));
216 if (UNLIKELY(object == nullptr)) {
217 ThrowOutOfMemoryError(size, "Heap::AllocateHugeObject");
218 }
219 }
220 object->SetClass(hclass);
221 OnAllocateEvent(reinterpret_cast<uintptr_t>(object));
222 return object;
223 }
224
AllocateMachineCodeObject(JSHClass * hclass,size_t size)225 TaggedObject *Heap::AllocateMachineCodeObject(JSHClass *hclass, size_t size)
226 {
227 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
228 auto object = reinterpret_cast<TaggedObject *>(machineCodeSpace_->Allocate(size));
229 if (UNLIKELY(object == nullptr)) {
230 ThrowOutOfMemoryError(size, "Heap::AllocateMachineCodeObject");
231 return nullptr;
232 }
233 object->SetClass(hclass);
234 OnAllocateEvent(reinterpret_cast<uintptr_t>(object));
235 return object;
236 }
237
AllocateSnapShotSpace(size_t size)238 uintptr_t Heap::AllocateSnapShotSpace(size_t size)
239 {
240 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
241 uintptr_t object = snapshotSpace_->Allocate(size);
242 if (UNLIKELY(object == 0)) {
243 LOG_ECMA_MEM(FATAL) << "alloc failed";
244 UNREACHABLE();
245 }
246 return object;
247 }
248
OnAllocateEvent(uintptr_t address)249 void Heap::OnAllocateEvent(uintptr_t address)
250 {
251 if (tracker_ != nullptr) {
252 tracker_->AllocationEvent(address);
253 }
254 }
255
OnMoveEvent(uintptr_t address,uintptr_t forwardAddress)256 void Heap::OnMoveEvent(uintptr_t address, uintptr_t forwardAddress)
257 {
258 if (tracker_ != nullptr) {
259 tracker_->MoveEvent(address, forwardAddress);
260 }
261 }
262
SwapNewSpace()263 void Heap::SwapNewSpace()
264 {
265 toSpace_->Stop();
266 fromSpace_->Restart();
267
268 SemiSpace *newSpace = fromSpace_;
269 fromSpace_ = toSpace_;
270 toSpace_ = newSpace;
271 }
272
ReclaimRegions(TriggerGCType gcType)273 void Heap::ReclaimRegions(TriggerGCType gcType)
274 {
275 toSpace_->EnumerateRegions([] (Region *region) {
276 region->ClearMarkBitmap();
277 region->ClearCrossRegionRememberedSet();
278 region->ResetAliveObject();
279 region->ClearFlag(RegionFlags::IS_IN_NEW_TO_NEW_SET);
280 });
281 if (gcType == TriggerGCType::FULL_GC) {
282 compressSpace_->Reset();
283 } else if (gcType == TriggerGCType::OLD_GC) {
284 oldSpace_->ReclaimCSet();
285 }
286 fromSpace_->ReclaimRegions();
287
288 sweeper_->WaitAllTaskFinished();
289 EnumerateNonNewSpaceRegions([] (Region *region) {
290 region->ClearMarkBitmap();
291 region->ClearCrossRegionRememberedSet();
292 });
293 if (!isClearTaskFinished_) {
294 os::memory::LockHolder holder(waitClearTaskFinishedMutex_);
295 isClearTaskFinished_ = true;
296 waitClearTaskFinishedCV_.SignalAll();
297 }
298 }
299
ClearSlotsRange(Region * current,uintptr_t freeStart,uintptr_t freeEnd)300 void Heap::ClearSlotsRange(Region *current, uintptr_t freeStart, uintptr_t freeEnd)
301 {
302 auto set = current->GetOldToNewRememberedSet();
303 if (set != nullptr) {
304 set->ClearRange(freeStart, freeEnd);
305 }
306 set = current->GetCrossRegionRememberedSet();
307 if (set != nullptr) {
308 set->ClearRange(freeStart, freeEnd);
309 }
310 }
311
GetCommittedSize()312 size_t Heap::GetCommittedSize() const
313 {
314 size_t result = toSpace_->GetCommittedSize() + oldSpace_->GetCommittedSize() + hugeObjectSpace_->GetCommittedSize()
315 + nonMovableSpace_->GetCommittedSize() + machineCodeSpace_->GetCommittedSize();
316 return result;
317 }
318
GetHeapObjectSize()319 size_t Heap::GetHeapObjectSize() const
320 {
321 size_t result = toSpace_->GetHeapObjectSize() + oldSpace_->GetHeapObjectSize()
322 + hugeObjectSpace_->GetHeapObjectSize() + nonMovableSpace_->GetHeapObjectSize()
323 + machineCodeSpace_->GetCommittedSize();
324 return result;
325 }
326 } // namespace panda::ecmascript
327
328 #endif // ECMASCRIPT_MEM_HEAP_INL_H
329