1 /*
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "ecmascript/mem/heap-inl.h"
17 #include "ecmascript/mem/jit_fort.h"
18 #include "ecmascript/jit/jit.h"
19 #if defined(JIT_ENABLE_CODE_SIGN) && !defined(JIT_FORT_DISABLE)
20 #include <sys/ioctl.h>
21 #include <sys/prctl.h>
22
23 #define XPM_JITFORT_ENABLE_OPCODE 3
24 #define XPM_MAGIC 'x'
25 #define XPM_SET_JITFORT_ENABLE _IOW(XPM_MAGIC, XPM_JITFORT_ENABLE_OPCODE, unsigned long)
26 #endif
27
28 namespace panda::ecmascript {
29
30 template <>
FreeListAllocator(BaseHeap * heap,MemDescPool * pool,JitFort * fort)31 FreeListAllocator<MemDesc>::FreeListAllocator(BaseHeap *heap, MemDescPool *pool, JitFort *fort)
32 : memDescPool_(pool), heap_(heap)
33 {
34 freeList_ = std::make_unique<FreeObjectList<MemDesc>>(fort);
35 }
36
JitFort()37 JitFort::JitFort()
38 {
39 jitFortMem_ = PageMap(JIT_FORT_REG_SPACE_MAX,
40 PageProtectProt(Jit::GetInstance()->IsDisableCodeSign() || !IsResourceAvailable()),
41 DEFAULT_REGION_SIZE, nullptr, MAP_JITFORT);
42 jitFortBegin_ = reinterpret_cast<uintptr_t>(jitFortMem_.GetMem());
43 jitFortSize_ = JIT_FORT_REG_SPACE_MAX;
44 memDescPool_ = new MemDescPool(jitFortBegin_, jitFortSize_);
45 allocator_ = new FreeListAllocator<MemDesc>(nullptr, memDescPool_, this);
46 InitRegions();
47 LOG_JIT(DEBUG) << "JitFort Begin " << (void *)JitFortBegin() << " end " << (void *)(JitFortBegin() + JitFortSize());
48 }
49
~JitFort()50 JitFort::~JitFort()
51 {
52 constexpr size_t numRegions = JIT_FORT_REG_SPACE_MAX / DEFAULT_REGION_SIZE;
53 for (size_t i = 0; i < numRegions; i++) {
54 regions_[i]->DestroyFreeObjectSets();
55 delete regions_[i];
56 }
57 delete allocator_;
58 delete memDescPool_;
59 PageUnmap(jitFortMem_);
60 }
61
InitRegions()62 void JitFort::InitRegions()
63 {
64 constexpr size_t numRegions = JIT_FORT_REG_SPACE_MAX / DEFAULT_REGION_SIZE;
65 for (size_t i = 0; i < numRegions; i++) {
66 uintptr_t mem = reinterpret_cast<uintptr_t>(jitFortMem_.GetMem()) + i*DEFAULT_REGION_SIZE;
67 uintptr_t end = mem + DEFAULT_REGION_SIZE;
68 JitFortRegion *region = new JitFortRegion(nullptr, mem, end, RegionSpaceFlag::IN_MACHINE_CODE_SPACE,
69 memDescPool_);
70 region->InitializeFreeObjectSets();
71 regions_[i] = region;
72 }
73 AddRegion();
74 }
75
AddRegion()76 bool JitFort::AddRegion()
77 {
78 if (nextFreeRegionIdx_ < MAX_JIT_FORT_REGIONS) {
79 allocator_->AddFree(regions_[nextFreeRegionIdx_]);
80 regionList_.AddNode(regions_[nextFreeRegionIdx_]);
81 nextFreeRegionIdx_++;
82 return true;
83 }
84 return false;
85 }
86
Allocate(MachineCodeDesc * desc)87 uintptr_t JitFort::Allocate(MachineCodeDesc *desc)
88 {
89 LockHolder lock(mutex_);
90
91 size_t size = desc->instructionsSize;
92 auto ret = allocator_->Allocate(size);
93 if (ret == ToUintPtr(nullptr)) {
94 if (AddRegion()) {
95 LOG_JIT(DEBUG) << "JitFort: Allocate - AddRegion";
96 ret = allocator_->Allocate(size);
97 }
98 }
99 if (ret == ToUintPtr(nullptr)) {
100 LOG_JIT(DEBUG) << "JitFort:: Allocate return nullptr for size " << size;
101 return ret;
102 }
103 // Record allocation to keep it from being collected by the next
104 // JitFort::UpdateFreeSpace in case corresponding Machine code object is not
105 // marked for sweep yet by then.
106 desc->memDesc = RecordLiveJitCodeNoLock(ret, size);
107 LOG_JIT(DEBUG) << "JitFort:: Allocate " << (void *)ret << " - " << (void *)(ret+size-1) <<
108 " size " << size << " MachineCodeGC " << IsMachineCodeGC();
109 return ret;
110 }
111
RecordLiveJitCodeNoLock(uintptr_t addr,size_t size,bool installed)112 MemDesc *JitFort::RecordLiveJitCodeNoLock(uintptr_t addr, size_t size, bool installed)
113 {
114 // check duplicate
115 for (size_t i = 0; i < liveJitCodeBlks_.size(); ++i) {
116 if (liveJitCodeBlks_[i]->GetBegin() == addr && liveJitCodeBlks_[i]->Available() == size) {
117 LOG_JIT(DEBUG) << "RecordLiveJitCode duplicate " << (void *)addr << " size " << size;
118 return nullptr;
119 }
120 if (liveJitCodeBlks_[i]->GetBegin() == addr) {
121 LOG_JIT(FATAL) << "RecordLiveJitCode duplicate addr " << std::hex << addr << std::dec << " size " <<
122 size << " existing entry size " << liveJitCodeBlks_[i]->Available() << std::endl;
123 return nullptr;
124 }
125 }
126 MemDesc *desc = memDescPool_->GetDescFromPool();
127 ASSERT(desc != NULL);
128 desc->SetMem(addr);
129 desc->SetSize(size);
130 desc->SetInstalled(installed);
131 liveJitCodeBlks_.emplace_back(desc);
132 return desc;
133 }
134
RecordLiveJitCode(uintptr_t addr,size_t size,bool installed)135 MemDesc *JitFort::RecordLiveJitCode(uintptr_t addr, size_t size, bool installed)
136 {
137 LockHolder lock(mutex_);
138 return RecordLiveJitCodeNoLock(addr, size, installed);
139 }
140
SortLiveMemDescList()141 void JitFort::SortLiveMemDescList()
142 {
143 if (liveJitCodeBlks_.size()) {
144 std::sort(liveJitCodeBlks_.begin(), liveJitCodeBlks_.end(), [](MemDesc *first, MemDesc *second) {
145 return first->GetBegin() < second->GetBegin(); // ascending order
146 });
147 }
148 }
149
150 /*
151 * UpdateFreeSpace updates JitFort allocator free object list by go through mem blocks
152 * in use (liveJitCodeBlks_) in Jit fort space and putting free space in between into
153 * allocator free list .
154 *
155 * This is to be done once when an old or full GC Sweep finishes, and needs to be mutext
156 * protected because if concurrent sweep is enabled, this func may be called simulatneously
157 * from a GC worker thread when Old/Full GC Seep finishes (AsyncClearTask), or from main/JS
158 * thread AllocateMachineCode if an Old/Full GC is in progress.
159 *
160 * The following must be done before calling UpdateFreeSpace:
161 * - MachineCodeSpace::FreeRegion completed (whether sync or aync) on all regions
162 */
UpdateFreeSpace()163 void JitFort::UpdateFreeSpace()
164 {
165 LockHolder lock(mutex_);
166
167 if (!regionList_.GetLength()) {
168 return;
169 }
170
171 if (!IsMachineCodeGC()) {
172 return;
173 }
174
175 LOG_JIT(DEBUG) << "UpdateFreeSpace enter: " << "Fort space allocated: "
176 << allocator_->GetAllocatedSize()
177 << " available: " << allocator_->GetAvailableSize()
178 << " liveJitCodeBlks: " << liveJitCodeBlks_.size();
179 allocator_->RebuildFreeList();
180 SortLiveMemDescList();
181 auto region = regionList_.GetFirst();
182 while (region) {
183 CollectFreeRanges(region);
184 region = region->GetNext();
185 }
186 LOG_JIT(DEBUG) << "UpdateFreeSpace exit: allocator_->GetAvailableSize "
187 << allocator_->GetAvailableSize();
188
189 SetMachineCodeGC(false);
190 }
191
CollectFreeRanges(JitFortRegion * region)192 void JitFort::CollectFreeRanges(JitFortRegion *region)
193 {
194 LOG_JIT(DEBUG) << "region " << (void*)(region->GetBegin());
195 uintptr_t freeStart = region->GetBegin();
196 for (auto it = liveJitCodeBlks_.begin(); it != liveJitCodeBlks_.end();) {
197 MemDesc* desc = *it;
198 if (desc->GetBegin() < region->GetBegin()) {
199 // Skip entries for fort mem awaiting installation that's already processed
200 // by a previous call to CollectFreeRange. Do not use desc->IsInstalled()
201 // to check here because an entry's IsInstalled flag can change from false to
202 // true (if JSThread installs its corresponding MachineCode object) during
203 // the time a GC thread runs UpdateFreeSpace
204 it++;
205 continue;
206 }
207 if (desc->GetBegin() >= region->GetBegin() && desc->GetBegin() < region->GetEnd()) {
208 uintptr_t freeEnd = desc->GetBegin();
209 LOG_JIT(DEBUG) << " freeStart = " << (void *)freeStart
210 << " freeEnd = "<< (void*)freeEnd
211 << " desc->GetBegin() = " << (void *)(desc->GetBegin())
212 << " desc->GetEnd() = " << (void *)(desc->GetEnd());
213 if (freeStart != freeEnd && freeEnd <= freeStart) {
214 LOG_JIT(FATAL) << "CollectFreeRanges Abort: freeEnd smaller than freeStart";
215 return;
216 }
217 if (freeStart != freeEnd) {
218 allocator_->Free(freeStart, freeEnd - freeStart, true);
219 }
220 freeStart = freeEnd + desc->Available();
221 if (desc->IsInstalled()) {
222 it = liveJitCodeBlks_.erase(it);
223 memDescPool_->ReturnDescToPool(desc);
224 } else {
225 // retain liveJitCodeBlks entry for fort mem awaiting installation to keep from being freed
226 it++;
227 }
228 } else {
229 break;
230 }
231 }
232 uintptr_t freeEnd = region->GetEnd();
233 if (freeStart != freeEnd) {
234 allocator_->Free(freeStart, freeEnd - freeStart, true);
235 LOG_JIT(DEBUG) << " freeStart = " << (void *)freeStart << " freeEnd = " << (void*)freeEnd;
236 }
237 }
238
InRange(uintptr_t address) const239 bool JitFort::InRange(uintptr_t address) const
240 {
241 return address >= jitFortBegin_ && address <= (jitFortBegin_ + jitFortSize_ - 1);
242 }
243
244 // Used by JitFort::UpdateFreeSpace call path to find corresponding
245 // JitFortRegion for a free block in JitFort space, in order to put the blk into
246 // the corresponding free set of the JitFortRegion the free block belongs.
ObjectAddressToRange(uintptr_t objAddress)247 JitFortRegion *JitFort::ObjectAddressToRange(uintptr_t objAddress)
248 {
249 JitFortRegion *region = GetRegionList();
250 while (region != nullptr) {
251 if (objAddress >= region->GetBegin() && objAddress < region->GetEnd()) {
252 return region;
253 }
254 region = region->GetNext();
255 }
256 return region;
257 }
258
259 bool JitFort::isResourceAvailable_ = true;
IsResourceAvailable()260 bool JitFort::IsResourceAvailable()
261 {
262 return isResourceAvailable_;
263 }
InitJitFortResource()264 void JitFort::InitJitFortResource()
265 {
266 #if defined(JIT_ENABLE_CODE_SIGN) && !defined(JIT_FORT_DISABLE)
267 ECMA_BYTRACE_NAME(HITRACE_TAG_ARK, "JIT::InitJitFortResource");
268 if (!Jit::GetInstance()->IsAppJit()) {
269 int fd = open("/dev/xpm", O_RDWR);
270 if (fd < 0) {
271 isResourceAvailable_ = false;
272 LOG_JIT(ERROR) << "Failed to init jitfort resource, open xpm failed: " << strerror(errno);
273 return;
274 }
275 int rc = ioctl(fd, XPM_SET_JITFORT_ENABLE, 0);
276 if (rc < 0) {
277 isResourceAvailable_ = false;
278 LOG_JIT(ERROR) << "Failed to init jitfort resource, enable xpm failed: " << strerror(errno);
279 close(fd);
280 return;
281 }
282 close(fd);
283 }
284 constexpr int prSetJitFort = 0x6a6974;
285 constexpr int jitFortInit = 5;
286 int res = prctl(prSetJitFort, jitFortInit, 0);
287 if (res < 0) {
288 isResourceAvailable_ = false;
289 LOG_JIT(ERROR) << "Failed to init jitfort resource: " << strerror(errno);
290 return;
291 }
292 res = prctl(prSetJitFort, jitFortInit, 0);
293 if (res >= 0 || errno != EEXIST) {
294 isResourceAvailable_ = false;
295 LOG_JIT(ERROR) << "jitfort not support";
296 }
297 #endif
298 }
299
MemDescPool(uintptr_t fortBegin,size_t fortSize)300 MemDescPool::MemDescPool(uintptr_t fortBegin, size_t fortSize)
301 : fortBegin_(fortBegin), fortSize_(fortSize)
302 {
303 Expand();
304 }
305
~MemDescPool()306 MemDescPool::~MemDescPool()
307 {
308 for (const auto& block : memDescBlocks_) {
309 if (block) {
310 free(block);
311 }
312 }
313 }
314
GetDesc()315 MemDesc *MemDescPool::GetDesc()
316 {
317 if (IsEmpty(freeList_)) {
318 Expand();
319 }
320 if (!IsEmpty(freeList_)) {
321 MemDesc *res = freeList_;
322 freeList_ = freeList_->GetNext();
323 allocated_++;
324 if (allocated_-returned_ > highwater_) {
325 highwater_ = allocated_ - returned_;
326 }
327 return res;
328 }
329 return nullptr;
330 }
331
Expand()332 void MemDescPool::Expand()
333 {
334 void *block = malloc(sizeof(MemDesc) * MEMDESCS_PER_BLOCK);
335 if (block) {
336 memDescBlocks_.push_back(block);
337 for (size_t i = 0; i < MEMDESCS_PER_BLOCK; ++i) {
338 Add(new (ToVoidPtr(reinterpret_cast<uintptr_t>(block) + i*sizeof(MemDesc))) MemDesc());
339 }
340 }
341 }
342
Add(MemDesc * desc)343 void MemDescPool::Add(MemDesc *desc)
344 {
345 desc->SetNext(freeList_);
346 freeList_ = desc;
347 }
348
349 } // namespace panda::ecmascript
350