1 /*
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "ecmascript/js_tagged_value-inl.h"
17 #include "ecmascript/mem/mem_controller.h"
18 #include "ecmascript/mem/region-inl.h"
19 #include "ecmascript/mem/space.h"
20 #include "ecmascript/platform/os.h"
21
22 namespace panda::ecmascript {
Space(BaseHeap * heap,HeapRegionAllocator * heapRegionAllocator,MemSpaceType spaceType,size_t initialCapacity,size_t maximumCapacity)23 Space::Space(BaseHeap* heap, HeapRegionAllocator *heapRegionAllocator,
24 MemSpaceType spaceType, size_t initialCapacity,
25 size_t maximumCapacity)
26 : heap_(heap),
27 heapRegionAllocator_(heapRegionAllocator),
28 spaceType_(spaceType),
29 initialCapacity_(initialCapacity),
30 maximumCapacity_(maximumCapacity),
31 committedSize_(0)
32 {
33 ASSERT(heap != nullptr);
34 ASSERT(heapRegionAllocator != nullptr);
35 }
36
AddAllocationInspector(AllocationInspector * inspector)37 void Space::AddAllocationInspector(AllocationInspector* inspector)
38 {
39 ASSERT(inspector != nullptr);
40 allocationCounter_.AddAllocationInspector(inspector);
41 }
42
ClearAllocationInspector()43 void Space::ClearAllocationInspector()
44 {
45 allocationCounter_.ClearAllocationInspector();
46 }
47
SwapAllocationCounter(Space * space)48 void Space::SwapAllocationCounter(Space *space)
49 {
50 ASSERT(space != nullptr);
51 std::swap(allocationCounter_, space->allocationCounter_);
52 }
53
Destroy()54 void Space::Destroy()
55 {
56 ReclaimRegions();
57 }
58
ReclaimRegions(size_t cachedSize)59 void Space::ReclaimRegions(size_t cachedSize)
60 {
61 ASSERT(cachedSize >= 0);
62 EnumerateRegions([this, &cachedSize](Region *current) { ClearAndFreeRegion(current, cachedSize); });
63 regionList_.Clear();
64 committedSize_ = 0;
65 }
66
ClearAndFreeRegion(Region * region,size_t cachedSize)67 void Space::ClearAndFreeRegion(Region *region, size_t cachedSize)
68 {
69 ASSERT(region != nullptr);
70 LOG_ECMA_MEM(DEBUG) << "Clear region from:" << region << " to " << ToSpaceTypeName(spaceType_);
71 region->DeleteCrossRegionRSet();
72 region->DeleteOldToNewRSet();
73 region->DeleteLocalToShareRSet();
74 region->DeleteSweepingOldToNewRSet();
75 region->DeleteSweepingLocalToShareRSet();
76 DecreaseCommitted(region->GetCapacity());
77 DecreaseObjectSize(region->GetSize());
78 if (spaceType_ == MemSpaceType::OLD_SPACE || spaceType_ == MemSpaceType::NON_MOVABLE ||
79 spaceType_ == MemSpaceType::MACHINE_CODE_SPACE || spaceType_ == MemSpaceType::LOCAL_SPACE ||
80 spaceType_ == MemSpaceType::APPSPAWN_SPACE || spaceType_ == MemSpaceType::SHARED_NON_MOVABLE ||
81 spaceType_ == MemSpaceType::SHARED_OLD_SPACE || spaceType_ == MemSpaceType::SHARED_LOCAL_SPACE) {
82 region->DestroyFreeObjectSets();
83 }
84 heapRegionAllocator_->FreeRegion(region, cachedSize);
85 }
86
HugeObjectSpace(Heap * heap,HeapRegionAllocator * heapRegionAllocator,size_t initialCapacity,size_t maximumCapacity)87 HugeObjectSpace::HugeObjectSpace(Heap *heap, HeapRegionAllocator *heapRegionAllocator,
88 size_t initialCapacity, size_t maximumCapacity)
89 : Space(heap, heapRegionAllocator, MemSpaceType::HUGE_OBJECT_SPACE, initialCapacity, maximumCapacity)
90 {
91 }
92
HugeObjectSpace(Heap * heap,HeapRegionAllocator * heapRegionAllocator,size_t initialCapacity,size_t maximumCapacity,MemSpaceType spaceType)93 HugeObjectSpace::HugeObjectSpace(Heap *heap, HeapRegionAllocator *heapRegionAllocator,
94 size_t initialCapacity, size_t maximumCapacity, MemSpaceType spaceType)
95 : Space(heap, heapRegionAllocator, spaceType, initialCapacity, maximumCapacity)
96 {
97 }
98
HugeMachineCodeSpace(Heap * heap,HeapRegionAllocator * heapRegionAllocator,size_t initialCapacity,size_t maximumCapacity)99 HugeMachineCodeSpace::HugeMachineCodeSpace(Heap *heap, HeapRegionAllocator *heapRegionAllocator,
100 size_t initialCapacity, size_t maximumCapacity)
101 : HugeObjectSpace(heap, heapRegionAllocator, initialCapacity,
102 maximumCapacity, MemSpaceType::HUGE_MACHINE_CODE_SPACE)
103 {
104 }
105
GetMachineCodeObject(uintptr_t pc) const106 uintptr_t HugeMachineCodeSpace::GetMachineCodeObject(uintptr_t pc) const
107 {
108 uintptr_t machineCode = 0;
109 EnumerateRegions([&](Region *region) {
110 if (machineCode != 0) {
111 return;
112 }
113 if (!region->InRange(pc)) {
114 return;
115 }
116 uintptr_t curPtr = region->GetBegin();
117 auto obj = MachineCode::Cast(reinterpret_cast<TaggedObject*>(curPtr));
118 if (obj->IsInText(pc)) {
119 machineCode = curPtr;
120 }
121 });
122 return machineCode;
123 }
124
AllocateFort(size_t objectSize,JSThread * thread,void * pDesc)125 Region *HugeMachineCodeSpace::AllocateFort(size_t objectSize, JSThread *thread, void *pDesc)
126 {
127 // A Huge machine code object is consisted of contiguous 256Kb aligned blocks.
128 // For JitFort, a huge machine code object starts with a page aligned mutable area
129 // (which holds Region and MachineCode object header, FuncEntryDesc and StackMap), followed
130 // by a page aligned immutable (JitFort space) area for JIT generated native instructions code.
131 //
132 // allocation sizes for Huge Machine Code:
133 // a: mutable area size (aligned up to PageSize()) =
134 // sizeof(Region) + HUGE_OBJECT_BITSET_SIZE + MachineCode::SIZE + payLoadSize - instructionsSize
135 // (note: payLoadSize = funcDesc size + stackMap size + instructionsSize)
136 // b: immutable area (starts on native page boundary) size = instructionsSize
137 // c: size to mmap for huge machine code object = Alignup(a + b, 256 Kbyte)
138 //
139 // mmap to enable JIT_FORT rights control:
140 // 1. first mmap (without JIT_FORT option flag) region of size c above
141 // 2. then mmap immutable area with MAP_FIXED and JIT_FORT option flag (to be used by codesigner verify/copy)
142 ASSERT(thread != nullptr);
143 ASSERT(pDesc != nullptr);
144 MachineCodeDesc *desc = reinterpret_cast<MachineCodeDesc *>(pDesc);
145 size_t mutableSize = AlignUp(
146 objectSize + sizeof(Region) + HUGE_OBJECT_BITSET_SIZE - desc->instructionsSize, PageSize());
147 size_t allocSize = AlignUp(mutableSize + desc->instructionsSize, PANDA_POOL_ALIGNMENT_IN_BYTES);
148 if (heap_->OldSpaceExceedCapacity(allocSize)) {
149 LOG_ECMA_MEM(INFO) << "Committed size " << committedSize_ << " of huge object space is too big.";
150 return 0;
151 }
152 Region *region = heapRegionAllocator_->AllocateAlignedRegion(this, allocSize, thread, heap_);
153 desc->instructionsAddr = region->GetAllocateBase() + mutableSize;
154
155 // Enabe JitFort rights control
156 [[maybe_unused]] void *addr = PageMapExecFortSpace((void *)desc->instructionsAddr, allocSize - mutableSize,
157 PageProtectProt(reinterpret_cast<Heap *>(heap_)->GetEcmaVM()->GetJSOptions().GetDisableCodeSign() ||
158 !JitFort::IsResourceAvailable()));
159
160 ASSERT(addr == (void *)desc->instructionsAddr);
161 return region;
162 }
163
164
Allocate(size_t objectSize,JSThread * thread,void * pDesc,AllocateEventType allocType)165 uintptr_t HugeMachineCodeSpace::Allocate(size_t objectSize, JSThread *thread, void *pDesc,
166 AllocateEventType allocType)
167 {
168 ASSERT(thread != nullptr);
169 ASSERT(pDesc != nullptr);
170 // JitFort path
171 #if ECMASCRIPT_ENABLE_THREAD_STATE_CHECK
172 if (UNLIKELY(!thread->IsInRunningStateOrProfiling())) {
173 LOG_ECMA(FATAL) << "Allocate must be in jsthread running state";
174 UNREACHABLE();
175 }
176 #endif
177 if (allocType == AllocateEventType::NORMAL) {
178 thread->CheckSafepointIfSuspended();
179 }
180 Region *region;
181 if (reinterpret_cast<Heap *>(heap_)->GetEcmaVM()->GetJSOptions().GetEnableAsyncCopyToFort() &&
182 reinterpret_cast<MachineCodeDesc *>(pDesc)->isAsyncCompileMode) {
183 region = reinterpret_cast<Region *>(reinterpret_cast<MachineCodeDesc *>(pDesc)->hugeObjRegion);
184 } else {
185 region = AllocateFort(objectSize, thread, pDesc);
186 }
187 if (UNLIKELY(region == nullptr)) { // LCOV_EXCL_BR_LINE
188 LOG_GC(ERROR) << "HugeMachineCodeSpace::Allocate: region is nullptr";
189 return 0;
190 }
191 AddRegion(region);
192 // It need to mark unpoison when huge object being allocated.
193 ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<void *>(region->GetBegin()), objectSize);
194 #ifdef ECMASCRIPT_SUPPORT_HEAPSAMPLING
195 InvokeAllocationInspector(region->GetBegin(), objectSize);
196 #endif
197 return region->GetBegin();
198 }
199
Allocate(size_t objectSize,JSThread * thread)200 uintptr_t HugeMachineCodeSpace::Allocate(size_t objectSize, JSThread *thread)
201 {
202 // non JitFort path
203 return HugeObjectSpace::Allocate(objectSize, thread);
204 }
205
Allocate(size_t objectSize,JSThread * thread,AllocateEventType allocType)206 uintptr_t HugeObjectSpace::Allocate(size_t objectSize, JSThread *thread, AllocateEventType allocType)
207 {
208 #if ECMASCRIPT_ENABLE_THREAD_STATE_CHECK
209 if (UNLIKELY(!thread->IsInRunningStateOrProfiling())) {
210 LOG_ECMA(FATAL) << "Allocate must be in jsthread running state";
211 UNREACHABLE();
212 }
213 #endif
214 if (allocType == AllocateEventType::NORMAL) {
215 thread->CheckSafepointIfSuspended();
216 }
217 // In HugeObject allocation, we have a revervation of 8 bytes for markBitSet in objectSize.
218 // In case Region is not aligned by 16 bytes, HUGE_OBJECT_BITSET_SIZE is 8 bytes more.
219 size_t alignedSize = AlignUp(objectSize + sizeof(Region) + HUGE_OBJECT_BITSET_SIZE, PANDA_POOL_ALIGNMENT_IN_BYTES);
220 if (heap_->OldSpaceExceedCapacity(alignedSize)) {
221 LOG_ECMA_MEM(INFO) << "Committed size " << committedSize_ << " of huge object space is too big.";
222 return 0;
223 }
224 Region *region = heapRegionAllocator_->AllocateAlignedRegion(this, alignedSize, thread, heap_);
225 AddRegion(region);
226 // It need to mark unpoison when huge object being allocated.
227 ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<void *>(region->GetBegin()), objectSize);
228 #ifdef ECMASCRIPT_SUPPORT_HEAPSAMPLING
229 InvokeAllocationInspector(region->GetBegin(), objectSize);
230 #endif
231 return region->GetBegin();
232 }
233
Sweep()234 void HugeObjectSpace::Sweep()
235 {
236 Region *currentRegion = GetRegionList().GetFirst();
237 while (currentRegion != nullptr) {
238 Region *next = currentRegion->GetNext();
239 bool isMarked = false;
240 currentRegion->IterateAllMarkedBits([&isMarked]([[maybe_unused]] void *mem) { isMarked = true; });
241 if (!isMarked) {
242 GetRegionList().RemoveNode(currentRegion);
243 hugeNeedFreeList_.AddNode(currentRegion);
244 }
245 currentRegion = next;
246 }
247 }
248
GetHeapObjectSize() const249 size_t HugeObjectSpace::GetHeapObjectSize() const
250 {
251 return committedSize_;
252 }
253
IterateOverObjects(const std::function<void (TaggedObject * object)> & objectVisitor) const254 void HugeObjectSpace::IterateOverObjects(const std::function<void(TaggedObject *object)> &objectVisitor) const
255 {
256 EnumerateRegions([&](Region *region) {
257 uintptr_t curPtr = region->GetBegin();
258 objectVisitor(reinterpret_cast<TaggedObject *>(curPtr));
259 });
260 }
261
ReclaimHugeRegion()262 void HugeObjectSpace::ReclaimHugeRegion()
263 {
264 if (hugeNeedFreeList_.IsEmpty()) {
265 return;
266 }
267 do {
268 Region *last = hugeNeedFreeList_.PopBack();
269 ClearAndFreeRegion(last);
270 } while (!hugeNeedFreeList_.IsEmpty());
271 }
272
InvokeAllocationInspector(Address object,size_t objectSize)273 void HugeObjectSpace::InvokeAllocationInspector(Address object, size_t objectSize)
274 {
275 if (LIKELY(!allocationCounter_.IsActive())) { // LCOV_EXCL_BR_LINE
276 return;
277 }
278 if (objectSize >= allocationCounter_.NextBytes()) {
279 allocationCounter_.InvokeAllocationInspector(object, objectSize, objectSize);
280 }
281 allocationCounter_.AdvanceAllocationInspector(objectSize);
282 }
283 } // namespace panda::ecmascript
284