1 /*
2 * Copyright (c) 2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "ecmascript/mem/heap_region_allocator.h"
17
18 #include "ecmascript/jit/jit.h"
19 #include "ecmascript/mem/mem_map_allocator.h"
20
21 namespace panda::ecmascript {
22 constexpr size_t PANDA_POOL_ALIGNMENT_IN_BYTES = 256_KB;
23
HeapRegionAllocator(JSRuntimeOptions & option)24 HeapRegionAllocator::HeapRegionAllocator(JSRuntimeOptions &option)
25 {
26 enablePageTagThreadId_ = option.EnablePageTagThreadId();
27 }
28
AllocateAlignedRegion(Space * space,size_t capacity,JSThread * thread,BaseHeap * heap,bool isFresh)29 Region *HeapRegionAllocator::AllocateAlignedRegion(Space *space, size_t capacity, JSThread* thread, BaseHeap *heap,
30 bool isFresh)
31 {
32 if (capacity == 0) { // LOCV_EXCL_BR_LINE
33 LOG_ECMA_MEM(FATAL) << "capacity must have a size bigger than 0";
34 UNREACHABLE();
35 }
36 RegionSpaceFlag flags = space->GetRegionFlag();
37 RegionTypeFlag typeFlag = isFresh ? RegionTypeFlag::FRESH : RegionTypeFlag::DEFAULT;
38 bool isRegular = (flags != RegionSpaceFlag::IN_HUGE_OBJECT_SPACE &&
39 flags != RegionSpaceFlag::IN_HUGE_MACHINE_CODE_SPACE &&
40 flags != RegionSpaceFlag::IN_SHARED_HUGE_OBJECT_SPACE);
41 bool isMachineCode = (flags == RegionSpaceFlag::IN_MACHINE_CODE_SPACE ||
42 flags == RegionSpaceFlag::IN_HUGE_MACHINE_CODE_SPACE);
43 JSThread::ThreadId tid = 0;
44 bool shouldPageTag = AllocateRegionShouldPageTag(space);
45 if (enablePageTagThreadId_) {
46 tid = thread ? thread->GetThreadId() : JSThread::GetCurrentThreadId();
47 }
48 auto pool = MemMapAllocator::GetInstance()->Allocate(tid, capacity, DEFAULT_REGION_SIZE,
49 ToSpaceTypeName(space->GetSpaceType()), isRegular, isMachineCode, Jit::GetInstance()->IsEnableJitFort(),
50 shouldPageTag);
51 void *mapMem = pool.GetMem();
52 if (mapMem == nullptr) { // LOCV_EXCL_BR_LINE
53 if (heap->InGC()) {
54 // Donot crash in GC.
55 TemporarilyEnsureAllocateionAlwaysSuccess(heap);
56 pool = MemMapAllocator::GetInstance()->Allocate(tid, capacity, DEFAULT_REGION_SIZE,
57 ToSpaceTypeName(space->GetSpaceType()), isRegular, isMachineCode,
58 Jit::GetInstance()->IsEnableJitFort(), shouldPageTag);
59 mapMem = pool.GetMem();
60 if (mapMem == nullptr) {
61 // This should not happen
62 LOG_ECMA_MEM(FATAL) << "pool is empty in GC unexpectedly";
63 UNREACHABLE();
64 }
65 } else {
66 if (thread != nullptr && thread->GetEcmaVM()->IsInitialized()) {
67 Heap *localHeap = const_cast<Heap *>(thread->GetEcmaVM()->GetHeap());
68 localHeap->DumpHeapSnapshotBeforeOOM();
69 heap->ThrowOutOfMemoryErrorForDefault(thread, DEFAULT_REGION_SIZE,
70 "HeapRegionAllocator::AllocateAlignedRegion", false);
71 }
72 LOG_ECMA_MEM(FATAL) << "pool is empty " << annoMemoryUsage_.load(std::memory_order_relaxed);
73 UNREACHABLE();
74 }
75 }
76 #if ECMASCRIPT_ENABLE_ZAP_MEM
77 if (memset_s(mapMem, capacity, 0, capacity) != EOK) { // LOCV_EXCL_BR_LINE
78 LOG_FULL(FATAL) << "memset_s failed";
79 UNREACHABLE();
80 }
81 #endif
82 IncreaseAnnoMemoryUsage(capacity);
83
84 uintptr_t mem = ToUintPtr(mapMem);
85 // Check that the address is 256K byte aligned
86 LOG_ECMA_IF(AlignUp(mem, PANDA_POOL_ALIGNMENT_IN_BYTES) != mem, FATAL) << "region not align by 256KB";
87 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
88 uintptr_t begin = AlignUp(mem + sizeof(Region), static_cast<size_t>(MemAlignment::MEM_ALIGN_REGION));
89 uintptr_t end = mem + capacity;
90
91 Region *region = new (ToVoidPtr(mem)) Region(heap->GetNativeAreaAllocator(), mem, begin, end, flags, typeFlag);
92 region->Initialize();
93 std::atomic_thread_fence(std::memory_order_seq_cst);
94 return region;
95 }
96
FreeRegion(Region * region,size_t cachedSize,bool skipCache)97 void HeapRegionAllocator::FreeRegion(Region *region, size_t cachedSize, bool skipCache)
98 {
99 auto size = region->GetCapacity();
100 bool isRegular = !region->InHugeObjectSpace() && !region->InHugeMachineCodeSpace() &&
101 !region->InSharedHugeObjectSpace();
102 auto allocateBase = region->GetAllocateBase();
103 bool shouldPageTag = FreeRegionShouldPageTag(region);
104 bool inHugeMachineCodeSpace = region->InHugeMachineCodeSpace();
105
106 DecreaseAnnoMemoryUsage(size);
107 region->Invalidate();
108 region->ClearMembers();
109
110 // Use the mmap interface to clean up the MAP_JIT tag bits on memory.
111 // The MAP_JIT flag prevents the mprotect interface from setting PROT_WRITE for memory.
112 if (inHugeMachineCodeSpace) {
113 MemMap memMap = PageMap(size, PAGE_PROT_NONE, 0, ToVoidPtr(allocateBase), PAGE_FLAG_MAP_FIXED);
114 if (memMap.GetMem() == nullptr) {
115 LOG_ECMA_MEM(FATAL) << "Failed to clear the MAP_JIT flag for the huge machine code space.";
116 }
117 }
118 #if ECMASCRIPT_ENABLE_ZAP_MEM
119 if (memset_s(ToVoidPtr(allocateBase), size, INVALID_VALUE, size) != EOK) { // LOCV_EXCL_BR_LINE
120 LOG_FULL(FATAL) << "memset_s failed";
121 UNREACHABLE();
122 }
123 #endif
124 MemMapAllocator::GetInstance()->CacheOrFree(ToVoidPtr(allocateBase), size, isRegular, cachedSize,
125 shouldPageTag, skipCache);
126 }
127
TemporarilyEnsureAllocateionAlwaysSuccess(BaseHeap * heap)128 void HeapRegionAllocator::TemporarilyEnsureAllocateionAlwaysSuccess(BaseHeap *heap)
129 {
130 // Make MemMapAllocator infinite, and record to Dump&Fatal when GC completed.
131 heap->ShouldForceThrowOOMError();
132 MemMapAllocator::GetInstance()->TransferToInfiniteModeForGC();
133 }
134
AllocateRegionShouldPageTag(Space * space) const135 bool HeapRegionAllocator::AllocateRegionShouldPageTag(Space *space) const
136 {
137 if (enablePageTagThreadId_) {
138 return true;
139 }
140 MemSpaceType type = space->GetSpaceType();
141 // Both LocalSpace and OldSpace belong to OldSpace.
142 return type != MemSpaceType::OLD_SPACE && type != MemSpaceType::LOCAL_SPACE;
143 }
144
FreeRegionShouldPageTag(Region * region) const145 bool HeapRegionAllocator::FreeRegionShouldPageTag(Region *region) const
146 {
147 if (enablePageTagThreadId_) {
148 return true;
149 }
150 // There is no LocalSpace tag in region.
151 return !region->InOldSpace();
152 }
153 } // namespace panda::ecmascript
154