1 /*
2 * Copyright (c) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef ECMASCRIPT_MEM_HEAP_INL_H
17 #define ECMASCRIPT_MEM_HEAP_INL_H
18
19 #include "ecmascript/mem/heap.h"
20
21 #include "ecmascript/base/block_hook_scope.h"
22 #include "ecmascript/dfx/hprof/heap_tracker.h"
23 #include "ecmascript/ecma_vm.h"
24 #include "ecmascript/mem/allocator-inl.h"
25 #include "ecmascript/mem/concurrent_sweeper.h"
26 #include "ecmascript/mem/linear_space.h"
27 #include "ecmascript/mem/mem_controller.h"
28 #include "ecmascript/mem/sparse_space.h"
29 #include "ecmascript/mem/tagged_object.h"
30 #include "ecmascript/mem/barriers-inl.h"
31 #include "ecmascript/mem/mem_map_allocator.h"
32
33 namespace panda::ecmascript {
34 using BlockHookScope = base::BlockHookScope;
35 #define CHECK_OBJ_AND_THROW_OOM_ERROR(object, size, space, message) \
36 if (UNLIKELY((object) == nullptr)) { \
37 size_t oomOvershootSize = GetEcmaVM()->GetEcmaParamConfiguration().GetOutOfMemoryOvershootSize(); \
38 (space)->IncreaseOutOfMemoryOvershootSize(oomOvershootSize); \
39 object = reinterpret_cast<TaggedObject *>((space)->Allocate(size)); \
40 ThrowOutOfMemoryError(size, message); \
41 } \
42
43 template<class Callback>
EnumerateOldSpaceRegions(const Callback & cb,Region * region)44 void Heap::EnumerateOldSpaceRegions(const Callback &cb, Region *region) const
45 {
46 oldSpace_->EnumerateRegions(cb, region);
47 appSpawnSpace_->EnumerateRegions(cb);
48 nonMovableSpace_->EnumerateRegions(cb);
49 hugeObjectSpace_->EnumerateRegions(cb);
50 machineCodeSpace_->EnumerateRegions(cb);
51 }
52
53 template<class Callback>
EnumerateSnapshotSpaceRegions(const Callback & cb)54 void Heap::EnumerateSnapshotSpaceRegions(const Callback &cb) const
55 {
56 snapshotSpace_->EnumerateRegions(cb);
57 }
58
59 template<class Callback>
EnumerateNonNewSpaceRegions(const Callback & cb)60 void Heap::EnumerateNonNewSpaceRegions(const Callback &cb) const
61 {
62 oldSpace_->EnumerateRegions(cb);
63 oldSpace_->EnumerateCollectRegionSet(cb);
64 appSpawnSpace_->EnumerateRegions(cb);
65 snapshotSpace_->EnumerateRegions(cb);
66 nonMovableSpace_->EnumerateRegions(cb);
67 hugeObjectSpace_->EnumerateRegions(cb);
68 machineCodeSpace_->EnumerateRegions(cb);
69 }
70
71 template<class Callback>
EnumerateNonNewSpaceRegionsWithRecord(const Callback & cb)72 void Heap::EnumerateNonNewSpaceRegionsWithRecord(const Callback &cb) const
73 {
74 oldSpace_->EnumerateRegionsWithRecord(cb);
75 snapshotSpace_->EnumerateRegionsWithRecord(cb);
76 nonMovableSpace_->EnumerateRegionsWithRecord(cb);
77 hugeObjectSpace_->EnumerateRegionsWithRecord(cb);
78 machineCodeSpace_->EnumerateRegionsWithRecord(cb);
79 }
80
81 template<class Callback>
EnumerateNewSpaceRegions(const Callback & cb)82 void Heap::EnumerateNewSpaceRegions(const Callback &cb) const
83 {
84 activeSemiSpace_->EnumerateRegions(cb);
85 }
86
87 template<class Callback>
EnumerateNonMovableRegions(const Callback & cb)88 void Heap::EnumerateNonMovableRegions(const Callback &cb) const
89 {
90 snapshotSpace_->EnumerateRegions(cb);
91 appSpawnSpace_->EnumerateRegions(cb);
92 nonMovableSpace_->EnumerateRegions(cb);
93 hugeObjectSpace_->EnumerateRegions(cb);
94 machineCodeSpace_->EnumerateRegions(cb);
95 }
96
97 template<class Callback>
EnumerateRegions(const Callback & cb)98 void Heap::EnumerateRegions(const Callback &cb) const
99 {
100 activeSemiSpace_->EnumerateRegions(cb);
101 oldSpace_->EnumerateRegions(cb);
102 oldSpace_->EnumerateCollectRegionSet(cb);
103 appSpawnSpace_->EnumerateRegions(cb);
104 snapshotSpace_->EnumerateRegions(cb);
105 nonMovableSpace_->EnumerateRegions(cb);
106 hugeObjectSpace_->EnumerateRegions(cb);
107 machineCodeSpace_->EnumerateRegions(cb);
108 }
109
110 template<class Callback>
IterateOverObjects(const Callback & cb)111 void Heap::IterateOverObjects(const Callback &cb) const
112 {
113 activeSemiSpace_->IterateOverObjects(cb);
114 oldSpace_->IterateOverObjects(cb);
115 appSpawnSpace_->IterateOverMarkedObjects(cb);
116 nonMovableSpace_->IterateOverObjects(cb);
117 hugeObjectSpace_->IterateOverObjects(cb);
118 }
119
AllocateYoungOrHugeObject(JSHClass * hclass)120 TaggedObject *Heap::AllocateYoungOrHugeObject(JSHClass *hclass)
121 {
122 size_t size = hclass->GetObjectSize();
123 return AllocateYoungOrHugeObject(hclass, size);
124 }
125
AllocateYoungOrHugeObject(size_t size)126 TaggedObject *Heap::AllocateYoungOrHugeObject(size_t size)
127 {
128 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
129 if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) {
130 return AllocateHugeObject(size);
131 }
132
133 auto object = reinterpret_cast<TaggedObject *>(activeSemiSpace_->Allocate(size));
134 if (object == nullptr) {
135 CollectGarbage(SelectGCType());
136 object = reinterpret_cast<TaggedObject *>(activeSemiSpace_->Allocate(size));
137 if (object == nullptr) {
138 CollectGarbage(SelectGCType());
139 object = reinterpret_cast<TaggedObject *>(activeSemiSpace_->Allocate(size));
140 CHECK_OBJ_AND_THROW_OOM_ERROR(object, size, activeSemiSpace_, "Heap::AllocateYoungOrHugeObject");
141 }
142 }
143 return object;
144 }
145
AllocateYoungOrHugeObject(JSHClass * hclass,size_t size)146 TaggedObject *Heap::AllocateYoungOrHugeObject(JSHClass *hclass, size_t size)
147 {
148 auto object = AllocateYoungOrHugeObject(size);
149 object->SetClass(hclass);
150 OnAllocateEvent(reinterpret_cast<TaggedObject*>(object), size);
151 return object;
152 }
153
AllocateYoungSync(size_t size)154 uintptr_t Heap::AllocateYoungSync(size_t size)
155 {
156 return activeSemiSpace_->AllocateSync(size);
157 }
158
MoveYoungRegionSync(Region * region)159 bool Heap::MoveYoungRegionSync(Region *region)
160 {
161 return activeSemiSpace_->SwapRegion(region, inactiveSemiSpace_);
162 }
163
MergeToOldSpaceSync(LocalSpace * localSpace)164 void Heap::MergeToOldSpaceSync(LocalSpace *localSpace)
165 {
166 oldSpace_->Merge(localSpace);
167 }
168
TryAllocateYoungGeneration(JSHClass * hclass,size_t size)169 TaggedObject *Heap::TryAllocateYoungGeneration(JSHClass *hclass, size_t size)
170 {
171 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
172 if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) {
173 return nullptr;
174 }
175 auto object = reinterpret_cast<TaggedObject *>(activeSemiSpace_->Allocate(size));
176 if (object != nullptr) {
177 object->SetClass(hclass);
178 }
179 return object;
180 }
181
AllocateOldOrHugeObject(JSHClass * hclass)182 TaggedObject *Heap::AllocateOldOrHugeObject(JSHClass *hclass)
183 {
184 size_t size = hclass->GetObjectSize();
185 return AllocateOldOrHugeObject(hclass, size);
186 }
187
AllocateOldOrHugeObject(JSHClass * hclass,size_t size)188 TaggedObject *Heap::AllocateOldOrHugeObject(JSHClass *hclass, size_t size)
189 {
190 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
191 if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) {
192 return AllocateHugeObject(hclass, size);
193 }
194 auto object = reinterpret_cast<TaggedObject *>(oldSpace_->Allocate(size));
195 CHECK_OBJ_AND_THROW_OOM_ERROR(object, size, oldSpace_, "Heap::AllocateOldOrHugeObject");
196 object->SetClass(hclass);
197 OnAllocateEvent(reinterpret_cast<TaggedObject*>(object), size);
198 return object;
199 }
200
AllocateReadOnlyOrHugeObject(JSHClass * hclass)201 TaggedObject *Heap::AllocateReadOnlyOrHugeObject(JSHClass *hclass)
202 {
203 size_t size = hclass->GetObjectSize();
204 return AllocateReadOnlyOrHugeObject(hclass, size);
205 }
206
AllocateReadOnlyOrHugeObject(JSHClass * hclass,size_t size)207 TaggedObject *Heap::AllocateReadOnlyOrHugeObject(JSHClass *hclass, size_t size)
208 {
209 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
210 if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) {
211 return AllocateHugeObject(hclass, size);
212 }
213 auto object = reinterpret_cast<TaggedObject *>(readOnlySpace_->Allocate(size));
214 CHECK_OBJ_AND_THROW_OOM_ERROR(object, size, readOnlySpace_, "Heap::AllocateReadOnlyOrHugeObject");
215 object->SetClass(hclass);
216 OnAllocateEvent(reinterpret_cast<TaggedObject*>(object), size);
217 return object;
218 }
219
AllocateNonMovableOrHugeObject(JSHClass * hclass)220 TaggedObject *Heap::AllocateNonMovableOrHugeObject(JSHClass *hclass)
221 {
222 size_t size = hclass->GetObjectSize();
223 return AllocateNonMovableOrHugeObject(hclass, size);
224 }
225
AllocateNonMovableOrHugeObject(JSHClass * hclass,size_t size)226 TaggedObject *Heap::AllocateNonMovableOrHugeObject(JSHClass *hclass, size_t size)
227 {
228 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
229 if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) {
230 return AllocateHugeObject(hclass, size);
231 }
232 auto object = reinterpret_cast<TaggedObject *>(nonMovableSpace_->Allocate(size));
233 CHECK_OBJ_AND_THROW_OOM_ERROR(object, size, nonMovableSpace_, "Heap::AllocateNonMovableOrHugeObject");
234 object->SetClass(hclass);
235 OnAllocateEvent(reinterpret_cast<TaggedObject*>(object), size);
236 return object;
237 }
238
AllocateClassClass(JSHClass * hclass,size_t size)239 TaggedObject *Heap::AllocateClassClass(JSHClass *hclass, size_t size)
240 {
241 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
242 auto object = reinterpret_cast<TaggedObject *>(nonMovableSpace_->Allocate(size));
243 if (UNLIKELY(object == nullptr)) {
244 LOG_ECMA_MEM(FATAL) << "Heap::AllocateClassClass can not allocate any space";
245 UNREACHABLE();
246 }
247 *reinterpret_cast<MarkWordType *>(ToUintPtr(object)) = reinterpret_cast<MarkWordType>(hclass);
248 OnAllocateEvent(reinterpret_cast<TaggedObject*>(object), size);
249 return object;
250 }
251
AllocateHugeObject(size_t size)252 TaggedObject *Heap::AllocateHugeObject(size_t size)
253 {
254 // Check whether it is necessary to trigger Old GC before expanding to avoid OOM risk.
255 CheckAndTriggerOldGC(size);
256
257 auto *object = reinterpret_cast<TaggedObject *>(hugeObjectSpace_->Allocate(size, thread_));
258 if (UNLIKELY(object == nullptr)) {
259 CollectGarbage(TriggerGCType::OLD_GC);
260 object = reinterpret_cast<TaggedObject *>(hugeObjectSpace_->Allocate(size, thread_));
261 if (UNLIKELY(object == nullptr)) {
262 // if allocate huge object OOM, temporarily increase space size to avoid vm crash
263 size_t oomOvershootSize = GetEcmaVM()->GetEcmaParamConfiguration().GetOutOfMemoryOvershootSize();
264 oldSpace_->IncreaseOutOfMemoryOvershootSize(oomOvershootSize);
265 object = reinterpret_cast<TaggedObject *>(hugeObjectSpace_->Allocate(size, thread_));
266 if (UNLIKELY(object == nullptr)) {
267 FatalOutOfMemoryError(size, "Heap::AllocateHugeObject");
268 }
269 ThrowOutOfMemoryError(size, "Heap::AllocateHugeObject");
270 }
271 }
272 return object;
273 }
274
AllocateHugeObject(JSHClass * hclass,size_t size)275 TaggedObject *Heap::AllocateHugeObject(JSHClass *hclass, size_t size)
276 {
277 // Check whether it is necessary to trigger Old GC before expanding to avoid OOM risk.
278 CheckAndTriggerOldGC(size);
279 auto object = AllocateHugeObject(size);
280 object->SetClass(hclass);
281 OnAllocateEvent(reinterpret_cast<TaggedObject*>(object), size);
282 return object;
283 }
284
AllocateMachineCodeObject(JSHClass * hclass,size_t size)285 TaggedObject *Heap::AllocateMachineCodeObject(JSHClass *hclass, size_t size)
286 {
287 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
288 auto object = reinterpret_cast<TaggedObject *>(machineCodeSpace_->Allocate(size));
289 CHECK_OBJ_AND_THROW_OOM_ERROR(object, size, machineCodeSpace_, "Heap::AllocateMachineCodeObject");
290 object->SetClass(hclass);
291 OnAllocateEvent(reinterpret_cast<TaggedObject*>(object), size);
292 return object;
293 }
294
AllocateSnapshotSpace(size_t size)295 uintptr_t Heap::AllocateSnapshotSpace(size_t size)
296 {
297 size = AlignUp(size, static_cast<size_t>(MemAlignment::MEM_ALIGN_OBJECT));
298 uintptr_t object = snapshotSpace_->Allocate(size);
299 if (UNLIKELY(object == 0)) {
300 FatalOutOfMemoryError(size, "Heap::AllocateSnapshotSpaceObject");
301 }
302 return object;
303 }
304
OnAllocateEvent(TaggedObject * address,size_t size)305 void Heap::OnAllocateEvent([[maybe_unused]] TaggedObject* address, [[maybe_unused]] size_t size)
306 {
307 #if defined(ECMASCRIPT_SUPPORT_SNAPSHOT)
308 if (tracker_ != nullptr) {
309 BlockHookScope blockScope;
310 tracker_->AllocationEvent(address, size);
311 }
312 #endif
313 }
314
OnMoveEvent(uintptr_t address,TaggedObject * forwardAddress,size_t size)315 void Heap::OnMoveEvent([[maybe_unused]] uintptr_t address, [[maybe_unused]] TaggedObject* forwardAddress,
316 [[maybe_unused]] size_t size)
317 {
318 #if defined(ECMASCRIPT_SUPPORT_SNAPSHOT)
319 if (tracker_ != nullptr) {
320 BlockHookScope blockScope;
321 tracker_->MoveEvent(address, forwardAddress, size);
322 }
323 #endif
324 }
325
SwapNewSpace()326 void Heap::SwapNewSpace()
327 {
328 activeSemiSpace_->Stop();
329 inactiveSemiSpace_->Restart();
330
331 SemiSpace *newSpace = inactiveSemiSpace_;
332 inactiveSemiSpace_ = activeSemiSpace_;
333 activeSemiSpace_ = newSpace;
334 auto topAddress = activeSemiSpace_->GetAllocationTopAddress();
335 auto endAddress = activeSemiSpace_->GetAllocationEndAddress();
336 thread_->ReSetNewSpaceAllocationAddress(topAddress, endAddress);
337 }
338
ReclaimRegions(TriggerGCType gcType)339 void Heap::ReclaimRegions(TriggerGCType gcType)
340 {
341 activeSemiSpace_->EnumerateRegionsWithRecord([] (Region *region) {
342 region->ClearMarkGCBitset();
343 region->ClearCrossRegionRSet();
344 region->ResetAliveObject();
345 region->ClearGCFlag(RegionGCFlags::IN_NEW_TO_NEW_SET);
346 });
347 if (gcType == TriggerGCType::FULL_GC) {
348 compressSpace_->Reset();
349 } else if (gcType == TriggerGCType::OLD_GC) {
350 oldSpace_->ReclaimCSet();
351 }
352 inactiveSemiSpace_->ReclaimRegions();
353
354 sweeper_->WaitAllTaskFinished();
355 EnumerateNonNewSpaceRegionsWithRecord([] (Region *region) {
356 region->ClearMarkGCBitset();
357 region->ClearCrossRegionRSet();
358 });
359 if (!clearTaskFinished_) {
360 os::memory::LockHolder holder(waitClearTaskFinishedMutex_);
361 clearTaskFinished_ = true;
362 waitClearTaskFinishedCV_.SignalAll();
363 }
364 }
365
366 // only call in js-thread
ClearSlotsRange(Region * current,uintptr_t freeStart,uintptr_t freeEnd)367 void Heap::ClearSlotsRange(Region *current, uintptr_t freeStart, uintptr_t freeEnd)
368 {
369 current->AtomicClearSweepingRSetInRange(freeStart, freeEnd);
370 current->ClearOldToNewRSetInRange(freeStart, freeEnd);
371 current->AtomicClearCrossRegionRSetInRange(freeStart, freeEnd);
372 }
373
GetCommittedSize()374 size_t Heap::GetCommittedSize() const
375 {
376 size_t result = activeSemiSpace_->GetCommittedSize()
377 + oldSpace_->GetCommittedSize()
378 + hugeObjectSpace_->GetCommittedSize()
379 + nonMovableSpace_->GetCommittedSize()
380 + machineCodeSpace_->GetCommittedSize()
381 + snapshotSpace_->GetCommittedSize();
382 return result;
383 }
384
GetHeapObjectSize()385 size_t Heap::GetHeapObjectSize() const
386 {
387 size_t result = activeSemiSpace_->GetHeapObjectSize()
388 + oldSpace_->GetHeapObjectSize()
389 + hugeObjectSpace_->GetHeapObjectSize()
390 + nonMovableSpace_->GetHeapObjectSize()
391 + machineCodeSpace_->GetCommittedSize()
392 + snapshotSpace_->GetHeapObjectSize();
393 return result;
394 }
395
GetHeapObjectCount()396 int32_t Heap::GetHeapObjectCount() const
397 {
398 int32_t count = 0;
399 sweeper_->EnsureAllTaskFinished();
400 this->IterateOverObjects([&count]([[maybe_unused]]TaggedObject *obj) {
401 ++count;
402 });
403 return count;
404 }
405 } // namespace panda::ecmascript
406
407 #endif // ECMASCRIPT_MEM_HEAP_INL_H
408