1 //===------ TargetProcessControl.cpp -- Target process control APIs -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/TPCIndirectionUtils.h"
10
11 #include "llvm/ExecutionEngine/Orc/TargetProcessControl.h"
12 #include "llvm/Support/MathExtras.h"
13
14 #include <future>
15
16 using namespace llvm;
17 using namespace llvm::orc;
18
19 namespace llvm {
20 namespace orc {
21
22 class TPCIndirectionUtilsAccess {
23 public:
24 using IndirectStubInfo = TPCIndirectionUtils::IndirectStubInfo;
25 using IndirectStubInfoVector = TPCIndirectionUtils::IndirectStubInfoVector;
26
27 static Expected<IndirectStubInfoVector>
getIndirectStubs(TPCIndirectionUtils & TPCIU,unsigned NumStubs)28 getIndirectStubs(TPCIndirectionUtils &TPCIU, unsigned NumStubs) {
29 return TPCIU.getIndirectStubs(NumStubs);
30 };
31 };
32
33 } // end namespace orc
34 } // end namespace llvm
35
36 namespace {
37
38 class TPCTrampolinePool : public TrampolinePool {
39 public:
40 TPCTrampolinePool(TPCIndirectionUtils &TPCIU);
41 Error deallocatePool();
42
43 protected:
44 Error grow() override;
45
46 using Allocation = jitlink::JITLinkMemoryManager::Allocation;
47
48 TPCIndirectionUtils &TPCIU;
49 unsigned TrampolineSize = 0;
50 unsigned TrampolinesPerPage = 0;
51 std::vector<std::unique_ptr<Allocation>> TrampolineBlocks;
52 };
53
54 class TPCIndirectStubsManager : public IndirectStubsManager,
55 private TPCIndirectionUtilsAccess {
56 public:
TPCIndirectStubsManager(TPCIndirectionUtils & TPCIU)57 TPCIndirectStubsManager(TPCIndirectionUtils &TPCIU) : TPCIU(TPCIU) {}
58
59 Error deallocateStubs();
60
61 Error createStub(StringRef StubName, JITTargetAddress StubAddr,
62 JITSymbolFlags StubFlags) override;
63
64 Error createStubs(const StubInitsMap &StubInits) override;
65
66 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override;
67
68 JITEvaluatedSymbol findPointer(StringRef Name) override;
69
70 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override;
71
72 private:
73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
74
75 std::mutex ISMMutex;
76 TPCIndirectionUtils &TPCIU;
77 StringMap<StubInfo> StubInfos;
78 };
79
TPCTrampolinePool(TPCIndirectionUtils & TPCIU)80 TPCTrampolinePool::TPCTrampolinePool(TPCIndirectionUtils &TPCIU)
81 : TPCIU(TPCIU) {
82 auto &TPC = TPCIU.getTargetProcessControl();
83 auto &ABI = TPCIU.getABISupport();
84
85 TrampolineSize = ABI.getTrampolineSize();
86 TrampolinesPerPage =
87 (TPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
88 }
89
deallocatePool()90 Error TPCTrampolinePool::deallocatePool() {
91 Error Err = Error::success();
92 for (auto &Alloc : TrampolineBlocks)
93 Err = joinErrors(std::move(Err), Alloc->deallocate());
94 return Err;
95 }
96
grow()97 Error TPCTrampolinePool::grow() {
98 assert(AvailableTrampolines.empty() &&
99 "Grow called with trampolines still available");
100
101 auto ResolverAddress = TPCIU.getResolverBlockAddress();
102 assert(ResolverAddress && "Resolver address can not be null");
103
104 auto &TPC = TPCIU.getTargetProcessControl();
105 constexpr auto TrampolinePagePermissions =
106 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
107 sys::Memory::MF_EXEC);
108 auto PageSize = TPC.getPageSize();
109 jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
110 Request[TrampolinePagePermissions] = {PageSize, static_cast<size_t>(PageSize),
111 0};
112 auto Alloc = TPC.getMemMgr().allocate(Request);
113
114 if (!Alloc)
115 return Alloc.takeError();
116
117 unsigned NumTrampolines = TrampolinesPerPage;
118
119 auto WorkingMemory = (*Alloc)->getWorkingMemory(TrampolinePagePermissions);
120 auto TargetAddress = (*Alloc)->getTargetMemory(TrampolinePagePermissions);
121
122 TPCIU.getABISupport().writeTrampolines(WorkingMemory.data(), TargetAddress,
123 ResolverAddress, NumTrampolines);
124
125 auto TargetAddr = (*Alloc)->getTargetMemory(TrampolinePagePermissions);
126 for (unsigned I = 0; I < NumTrampolines; ++I)
127 AvailableTrampolines.push_back(TargetAddr + (I * TrampolineSize));
128
129 if (auto Err = (*Alloc)->finalize())
130 return Err;
131
132 TrampolineBlocks.push_back(std::move(*Alloc));
133
134 return Error::success();
135 }
136
createStub(StringRef StubName,JITTargetAddress StubAddr,JITSymbolFlags StubFlags)137 Error TPCIndirectStubsManager::createStub(StringRef StubName,
138 JITTargetAddress StubAddr,
139 JITSymbolFlags StubFlags) {
140 StubInitsMap SIM;
141 SIM[StubName] = std::make_pair(StubAddr, StubFlags);
142 return createStubs(SIM);
143 }
144
createStubs(const StubInitsMap & StubInits)145 Error TPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
146 auto AvailableStubInfos = getIndirectStubs(TPCIU, StubInits.size());
147 if (!AvailableStubInfos)
148 return AvailableStubInfos.takeError();
149
150 {
151 std::lock_guard<std::mutex> Lock(ISMMutex);
152 unsigned ASIdx = 0;
153 for (auto &SI : StubInits) {
154 auto &A = (*AvailableStubInfos)[ASIdx++];
155 StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
156 }
157 }
158
159 auto &MemAccess = TPCIU.getTargetProcessControl().getMemoryAccess();
160 switch (TPCIU.getABISupport().getPointerSize()) {
161 case 4: {
162 unsigned ASIdx = 0;
163 std::vector<tpctypes::UInt32Write> PtrUpdates;
164 for (auto &SI : StubInits)
165 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
166 static_cast<uint32_t>(SI.second.first)});
167 return MemAccess.writeUInt32s(PtrUpdates);
168 }
169 case 8: {
170 unsigned ASIdx = 0;
171 std::vector<tpctypes::UInt64Write> PtrUpdates;
172 for (auto &SI : StubInits)
173 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
174 static_cast<uint64_t>(SI.second.first)});
175 return MemAccess.writeUInt64s(PtrUpdates);
176 }
177 default:
178 return make_error<StringError>("Unsupported pointer size",
179 inconvertibleErrorCode());
180 }
181 }
182
findStub(StringRef Name,bool ExportedStubsOnly)183 JITEvaluatedSymbol TPCIndirectStubsManager::findStub(StringRef Name,
184 bool ExportedStubsOnly) {
185 std::lock_guard<std::mutex> Lock(ISMMutex);
186 auto I = StubInfos.find(Name);
187 if (I == StubInfos.end())
188 return nullptr;
189 return {I->second.first.StubAddress, I->second.second};
190 }
191
findPointer(StringRef Name)192 JITEvaluatedSymbol TPCIndirectStubsManager::findPointer(StringRef Name) {
193 std::lock_guard<std::mutex> Lock(ISMMutex);
194 auto I = StubInfos.find(Name);
195 if (I == StubInfos.end())
196 return nullptr;
197 return {I->second.first.PointerAddress, I->second.second};
198 }
199
updatePointer(StringRef Name,JITTargetAddress NewAddr)200 Error TPCIndirectStubsManager::updatePointer(StringRef Name,
201 JITTargetAddress NewAddr) {
202
203 JITTargetAddress PtrAddr = 0;
204 {
205 std::lock_guard<std::mutex> Lock(ISMMutex);
206 auto I = StubInfos.find(Name);
207 if (I == StubInfos.end())
208 return make_error<StringError>("Unknown stub name",
209 inconvertibleErrorCode());
210 PtrAddr = I->second.first.PointerAddress;
211 }
212
213 auto &MemAccess = TPCIU.getTargetProcessControl().getMemoryAccess();
214 switch (TPCIU.getABISupport().getPointerSize()) {
215 case 4: {
216 tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr);
217 return MemAccess.writeUInt32s(PUpdate);
218 }
219 case 8: {
220 tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr);
221 return MemAccess.writeUInt64s(PUpdate);
222 }
223 default:
224 return make_error<StringError>("Unsupported pointer size",
225 inconvertibleErrorCode());
226 }
227 }
228
229 } // end anonymous namespace.
230
231 namespace llvm {
232 namespace orc {
233
~ABISupport()234 TPCIndirectionUtils::ABISupport::~ABISupport() {}
235
236 Expected<std::unique_ptr<TPCIndirectionUtils>>
Create(TargetProcessControl & TPC)237 TPCIndirectionUtils::Create(TargetProcessControl &TPC) {
238 const auto &TT = TPC.getTargetTriple();
239 switch (TT.getArch()) {
240 default:
241 return make_error<StringError>(
242 std::string("No TPCIndirectionUtils available for ") + TT.str(),
243 inconvertibleErrorCode());
244 case Triple::aarch64:
245 case Triple::aarch64_32:
246 return CreateWithABI<OrcAArch64>(TPC);
247
248 case Triple::x86:
249 return CreateWithABI<OrcI386>(TPC);
250
251 case Triple::mips:
252 return CreateWithABI<OrcMips32Be>(TPC);
253
254 case Triple::mipsel:
255 return CreateWithABI<OrcMips32Le>(TPC);
256
257 case Triple::mips64:
258 case Triple::mips64el:
259 return CreateWithABI<OrcMips64>(TPC);
260
261 case Triple::x86_64:
262 if (TT.getOS() == Triple::OSType::Win32)
263 return CreateWithABI<OrcX86_64_Win32>(TPC);
264 else
265 return CreateWithABI<OrcX86_64_SysV>(TPC);
266 }
267 }
268
cleanup()269 Error TPCIndirectionUtils::cleanup() {
270 Error Err = Error::success();
271
272 for (auto &A : IndirectStubAllocs)
273 Err = joinErrors(std::move(Err), A->deallocate());
274
275 if (TP)
276 Err = joinErrors(std::move(Err),
277 static_cast<TPCTrampolinePool &>(*TP).deallocatePool());
278
279 if (ResolverBlock)
280 Err = joinErrors(std::move(Err), ResolverBlock->deallocate());
281
282 return Err;
283 }
284
285 Expected<JITTargetAddress>
writeResolverBlock(JITTargetAddress ReentryFnAddr,JITTargetAddress ReentryCtxAddr)286 TPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr,
287 JITTargetAddress ReentryCtxAddr) {
288 assert(ABI && "ABI can not be null");
289 constexpr auto ResolverBlockPermissions =
290 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
291 sys::Memory::MF_EXEC);
292 auto ResolverSize = ABI->getResolverCodeSize();
293
294 jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
295 Request[ResolverBlockPermissions] = {TPC.getPageSize(),
296 static_cast<size_t>(ResolverSize), 0};
297 auto Alloc = TPC.getMemMgr().allocate(Request);
298 if (!Alloc)
299 return Alloc.takeError();
300
301 auto WorkingMemory = (*Alloc)->getWorkingMemory(ResolverBlockPermissions);
302 ResolverBlockAddr = (*Alloc)->getTargetMemory(ResolverBlockPermissions);
303 ABI->writeResolverCode(WorkingMemory.data(), ResolverBlockAddr, ReentryFnAddr,
304 ReentryCtxAddr);
305
306 if (auto Err = (*Alloc)->finalize())
307 return std::move(Err);
308
309 ResolverBlock = std::move(*Alloc);
310 return ResolverBlockAddr;
311 }
312
313 std::unique_ptr<IndirectStubsManager>
createIndirectStubsManager()314 TPCIndirectionUtils::createIndirectStubsManager() {
315 return std::make_unique<TPCIndirectStubsManager>(*this);
316 }
317
getTrampolinePool()318 TrampolinePool &TPCIndirectionUtils::getTrampolinePool() {
319 if (!TP)
320 TP = std::make_unique<TPCTrampolinePool>(*this);
321 return *TP;
322 }
323
createLazyCallThroughManager(ExecutionSession & ES,JITTargetAddress ErrorHandlerAddr)324 LazyCallThroughManager &TPCIndirectionUtils::createLazyCallThroughManager(
325 ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) {
326 assert(!LCTM &&
327 "createLazyCallThroughManager can not have been called before");
328 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
329 &getTrampolinePool());
330 return *LCTM;
331 }
332
TPCIndirectionUtils(TargetProcessControl & TPC,std::unique_ptr<ABISupport> ABI)333 TPCIndirectionUtils::TPCIndirectionUtils(TargetProcessControl &TPC,
334 std::unique_ptr<ABISupport> ABI)
335 : TPC(TPC), ABI(std::move(ABI)) {
336 assert(this->ABI && "ABI can not be null");
337
338 assert(TPC.getPageSize() > getABISupport().getStubSize() &&
339 "Stubs larger than one page are not supported");
340 }
341
342 Expected<TPCIndirectionUtils::IndirectStubInfoVector>
getIndirectStubs(unsigned NumStubs)343 TPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
344
345 std::lock_guard<std::mutex> Lock(TPCUIMutex);
346
347 // If there aren't enough stubs available then allocate some more.
348 if (NumStubs > AvailableIndirectStubs.size()) {
349 auto NumStubsToAllocate = NumStubs;
350 auto PageSize = TPC.getPageSize();
351 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
352 NumStubsToAllocate = StubBytes / ABI->getStubSize();
353 auto PointerBytes =
354 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
355
356 constexpr auto StubPagePermissions =
357 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
358 sys::Memory::MF_EXEC);
359 constexpr auto PointerPagePermissions =
360 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
361 sys::Memory::MF_WRITE);
362
363 jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
364 Request[StubPagePermissions] = {PageSize, static_cast<size_t>(StubBytes),
365 0};
366 Request[PointerPagePermissions] = {PageSize, 0, PointerBytes};
367 auto Alloc = TPC.getMemMgr().allocate(Request);
368 if (!Alloc)
369 return Alloc.takeError();
370
371 auto StubTargetAddr = (*Alloc)->getTargetMemory(StubPagePermissions);
372 auto PointerTargetAddr = (*Alloc)->getTargetMemory(PointerPagePermissions);
373
374 ABI->writeIndirectStubsBlock(
375 (*Alloc)->getWorkingMemory(StubPagePermissions).data(), StubTargetAddr,
376 PointerTargetAddr, NumStubsToAllocate);
377
378 if (auto Err = (*Alloc)->finalize())
379 return std::move(Err);
380
381 for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
382 AvailableIndirectStubs.push_back(
383 IndirectStubInfo(StubTargetAddr, PointerTargetAddr));
384 StubTargetAddr += ABI->getStubSize();
385 PointerTargetAddr += ABI->getPointerSize();
386 }
387
388 IndirectStubAllocs.push_back(std::move(*Alloc));
389 }
390
391 assert(NumStubs <= AvailableIndirectStubs.size() &&
392 "Sufficient stubs should have been allocated above");
393
394 IndirectStubInfoVector Result;
395 while (NumStubs--) {
396 Result.push_back(AvailableIndirectStubs.back());
397 AvailableIndirectStubs.pop_back();
398 }
399
400 return std::move(Result);
401 }
402
reentry(JITTargetAddress LCTMAddr,JITTargetAddress TrampolineAddr)403 static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
404 JITTargetAddress TrampolineAddr) {
405 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
406 std::promise<JITTargetAddress> LandingAddrP;
407 auto LandingAddrF = LandingAddrP.get_future();
408 LCTM.resolveTrampolineLandingAddress(
409 TrampolineAddr,
410 [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); });
411 return LandingAddrF.get();
412 }
413
setUpInProcessLCTMReentryViaTPCIU(TPCIndirectionUtils & TPCIU)414 Error setUpInProcessLCTMReentryViaTPCIU(TPCIndirectionUtils &TPCIU) {
415 auto &LCTM = TPCIU.getLazyCallThroughManager();
416 return TPCIU
417 .writeResolverBlock(pointerToJITTargetAddress(&reentry),
418 pointerToJITTargetAddress(&LCTM))
419 .takeError();
420 }
421
422 } // end namespace orc
423 } // end namespace llvm
424