1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
10 #include "llvm/ADT/STLExtras.h"
11 #include "llvm/ADT/Triple.h"
12 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
13 #include "llvm/IR/CallSite.h"
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/Support/Format.h"
16 #include "llvm/Transforms/Utils/Cloning.h"
17 #include <sstream>
18
19 using namespace llvm;
20 using namespace llvm::orc;
21
22 namespace {
23
24 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
25 public:
26 using CompileFunction = JITCompileCallbackManager::CompileFunction;
27
CompileCallbackMaterializationUnit(SymbolStringPtr Name,CompileFunction Compile,VModuleKey K)28 CompileCallbackMaterializationUnit(SymbolStringPtr Name,
29 CompileFunction Compile, VModuleKey K)
30 : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}),
31 std::move(K)),
32 Name(std::move(Name)), Compile(std::move(Compile)) {}
33
getName() const34 StringRef getName() const override { return "<Compile Callbacks>"; }
35
36 private:
materialize(MaterializationResponsibility R)37 void materialize(MaterializationResponsibility R) override {
38 SymbolMap Result;
39 Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported);
40 // No dependencies, so these calls cannot fail.
41 cantFail(R.notifyResolved(Result));
42 cantFail(R.notifyEmitted());
43 }
44
discard(const JITDylib & JD,const SymbolStringPtr & Name)45 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
46 llvm_unreachable("Discard should never occur on a LMU?");
47 }
48
49 SymbolStringPtr Name;
50 CompileFunction Compile;
51 };
52
53 } // namespace
54
55 namespace llvm {
56 namespace orc {
57
anchor()58 void IndirectStubsManager::anchor() {}
anchor()59 void TrampolinePool::anchor() {}
60
61 Expected<JITTargetAddress>
getCompileCallback(CompileFunction Compile)62 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
63 if (auto TrampolineAddr = TP->getTrampoline()) {
64 auto CallbackName =
65 ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
66
67 std::lock_guard<std::mutex> Lock(CCMgrMutex);
68 AddrToSymbol[*TrampolineAddr] = CallbackName;
69 cantFail(CallbacksJD.define(
70 std::make_unique<CompileCallbackMaterializationUnit>(
71 std::move(CallbackName), std::move(Compile),
72 ES.allocateVModule())));
73 return *TrampolineAddr;
74 } else
75 return TrampolineAddr.takeError();
76 }
77
executeCompileCallback(JITTargetAddress TrampolineAddr)78 JITTargetAddress JITCompileCallbackManager::executeCompileCallback(
79 JITTargetAddress TrampolineAddr) {
80 SymbolStringPtr Name;
81
82 {
83 std::unique_lock<std::mutex> Lock(CCMgrMutex);
84 auto I = AddrToSymbol.find(TrampolineAddr);
85
86 // If this address is not associated with a compile callback then report an
87 // error to the execution session and return ErrorHandlerAddress to the
88 // callee.
89 if (I == AddrToSymbol.end()) {
90 Lock.unlock();
91 std::string ErrMsg;
92 {
93 raw_string_ostream ErrMsgStream(ErrMsg);
94 ErrMsgStream << "No compile callback for trampoline at "
95 << format("0x%016" PRIx64, TrampolineAddr);
96 }
97 ES.reportError(
98 make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()));
99 return ErrorHandlerAddress;
100 } else
101 Name = I->second;
102 }
103
104 if (auto Sym =
105 ES.lookup(makeJITDylibSearchOrder(
106 &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),
107 Name))
108 return Sym->getAddress();
109 else {
110 llvm::dbgs() << "Didn't find callback.\n";
111 // If anything goes wrong materializing Sym then report it to the session
112 // and return the ErrorHandlerAddress;
113 ES.reportError(Sym.takeError());
114 return ErrorHandlerAddress;
115 }
116 }
117
118 Expected<std::unique_ptr<JITCompileCallbackManager>>
createLocalCompileCallbackManager(const Triple & T,ExecutionSession & ES,JITTargetAddress ErrorHandlerAddress)119 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
120 JITTargetAddress ErrorHandlerAddress) {
121 switch (T.getArch()) {
122 default:
123 return make_error<StringError>(
124 std::string("No callback manager available for ") + T.str(),
125 inconvertibleErrorCode());
126 case Triple::aarch64:
127 case Triple::aarch64_32: {
128 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
129 return CCMgrT::Create(ES, ErrorHandlerAddress);
130 }
131
132 case Triple::x86: {
133 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
134 return CCMgrT::Create(ES, ErrorHandlerAddress);
135 }
136
137 case Triple::mips: {
138 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
139 return CCMgrT::Create(ES, ErrorHandlerAddress);
140 }
141 case Triple::mipsel: {
142 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
143 return CCMgrT::Create(ES, ErrorHandlerAddress);
144 }
145
146 case Triple::mips64:
147 case Triple::mips64el: {
148 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
149 return CCMgrT::Create(ES, ErrorHandlerAddress);
150 }
151
152 case Triple::x86_64: {
153 if ( T.getOS() == Triple::OSType::Win32 ) {
154 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
155 return CCMgrT::Create(ES, ErrorHandlerAddress);
156 } else {
157 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
158 return CCMgrT::Create(ES, ErrorHandlerAddress);
159 }
160 }
161
162 }
163 }
164
165 std::function<std::unique_ptr<IndirectStubsManager>()>
createLocalIndirectStubsManagerBuilder(const Triple & T)166 createLocalIndirectStubsManagerBuilder(const Triple &T) {
167 switch (T.getArch()) {
168 default:
169 return [](){
170 return std::make_unique<
171 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
172 };
173
174 case Triple::aarch64:
175 case Triple::aarch64_32:
176 return [](){
177 return std::make_unique<
178 orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
179 };
180
181 case Triple::x86:
182 return [](){
183 return std::make_unique<
184 orc::LocalIndirectStubsManager<orc::OrcI386>>();
185 };
186
187 case Triple::mips:
188 return [](){
189 return std::make_unique<
190 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
191 };
192
193 case Triple::mipsel:
194 return [](){
195 return std::make_unique<
196 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
197 };
198
199 case Triple::mips64:
200 case Triple::mips64el:
201 return [](){
202 return std::make_unique<
203 orc::LocalIndirectStubsManager<orc::OrcMips64>>();
204 };
205
206 case Triple::x86_64:
207 if (T.getOS() == Triple::OSType::Win32) {
208 return [](){
209 return std::make_unique<
210 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
211 };
212 } else {
213 return [](){
214 return std::make_unique<
215 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
216 };
217 }
218
219 }
220 }
221
createIRTypedAddress(FunctionType & FT,JITTargetAddress Addr)222 Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) {
223 Constant *AddrIntVal =
224 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr);
225 Constant *AddrPtrVal =
226 ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal,
227 PointerType::get(&FT, 0));
228 return AddrPtrVal;
229 }
230
createImplPointer(PointerType & PT,Module & M,const Twine & Name,Constant * Initializer)231 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
232 const Twine &Name, Constant *Initializer) {
233 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
234 Initializer, Name, nullptr,
235 GlobalValue::NotThreadLocal, 0, true);
236 IP->setVisibility(GlobalValue::HiddenVisibility);
237 return IP;
238 }
239
makeStub(Function & F,Value & ImplPointer)240 void makeStub(Function &F, Value &ImplPointer) {
241 assert(F.isDeclaration() && "Can't turn a definition into a stub.");
242 assert(F.getParent() && "Function isn't in a module.");
243 Module &M = *F.getParent();
244 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
245 IRBuilder<> Builder(EntryBlock);
246 LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
247 std::vector<Value*> CallArgs;
248 for (auto &A : F.args())
249 CallArgs.push_back(&A);
250 CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
251 Call->setTailCall();
252 Call->setAttributes(F.getAttributes());
253 if (F.getReturnType()->isVoidTy())
254 Builder.CreateRetVoid();
255 else
256 Builder.CreateRet(Call);
257 }
258
operator ()(Module & M)259 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
260 std::vector<GlobalValue *> PromotedGlobals;
261
262 for (auto &GV : M.global_values()) {
263 bool Promoted = true;
264
265 // Rename if necessary.
266 if (!GV.hasName())
267 GV.setName("__orc_anon." + Twine(NextId++));
268 else if (GV.getName().startswith("\01L"))
269 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
270 else if (GV.hasLocalLinkage())
271 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
272 else
273 Promoted = false;
274
275 if (GV.hasLocalLinkage()) {
276 GV.setLinkage(GlobalValue::ExternalLinkage);
277 GV.setVisibility(GlobalValue::HiddenVisibility);
278 Promoted = true;
279 }
280 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
281
282 if (Promoted)
283 PromotedGlobals.push_back(&GV);
284 }
285
286 return PromotedGlobals;
287 }
288
cloneFunctionDecl(Module & Dst,const Function & F,ValueToValueMapTy * VMap)289 Function* cloneFunctionDecl(Module &Dst, const Function &F,
290 ValueToValueMapTy *VMap) {
291 Function *NewF =
292 Function::Create(cast<FunctionType>(F.getValueType()),
293 F.getLinkage(), F.getName(), &Dst);
294 NewF->copyAttributesFrom(&F);
295
296 if (VMap) {
297 (*VMap)[&F] = NewF;
298 auto NewArgI = NewF->arg_begin();
299 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
300 ++ArgI, ++NewArgI)
301 (*VMap)[&*ArgI] = &*NewArgI;
302 }
303
304 return NewF;
305 }
306
moveFunctionBody(Function & OrigF,ValueToValueMapTy & VMap,ValueMaterializer * Materializer,Function * NewF)307 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
308 ValueMaterializer *Materializer,
309 Function *NewF) {
310 assert(!OrigF.isDeclaration() && "Nothing to move");
311 if (!NewF)
312 NewF = cast<Function>(VMap[&OrigF]);
313 else
314 assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap.");
315 assert(NewF && "Function mapping missing from VMap.");
316 assert(NewF->getParent() != OrigF.getParent() &&
317 "moveFunctionBody should only be used to move bodies between "
318 "modules.");
319
320 SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
321 CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns,
322 "", nullptr, nullptr, Materializer);
323 OrigF.deleteBody();
324 }
325
cloneGlobalVariableDecl(Module & Dst,const GlobalVariable & GV,ValueToValueMapTy * VMap)326 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
327 ValueToValueMapTy *VMap) {
328 GlobalVariable *NewGV = new GlobalVariable(
329 Dst, GV.getValueType(), GV.isConstant(),
330 GV.getLinkage(), nullptr, GV.getName(), nullptr,
331 GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
332 NewGV->copyAttributesFrom(&GV);
333 if (VMap)
334 (*VMap)[&GV] = NewGV;
335 return NewGV;
336 }
337
moveGlobalVariableInitializer(GlobalVariable & OrigGV,ValueToValueMapTy & VMap,ValueMaterializer * Materializer,GlobalVariable * NewGV)338 void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
339 ValueToValueMapTy &VMap,
340 ValueMaterializer *Materializer,
341 GlobalVariable *NewGV) {
342 assert(OrigGV.hasInitializer() && "Nothing to move");
343 if (!NewGV)
344 NewGV = cast<GlobalVariable>(VMap[&OrigGV]);
345 else
346 assert(VMap[&OrigGV] == NewGV &&
347 "Incorrect global variable mapping in VMap.");
348 assert(NewGV->getParent() != OrigGV.getParent() &&
349 "moveGlobalVariableInitializer should only be used to move "
350 "initializers between modules");
351
352 NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None,
353 nullptr, Materializer));
354 }
355
cloneGlobalAliasDecl(Module & Dst,const GlobalAlias & OrigA,ValueToValueMapTy & VMap)356 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
357 ValueToValueMapTy &VMap) {
358 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
359 auto *NewA = GlobalAlias::create(OrigA.getValueType(),
360 OrigA.getType()->getPointerAddressSpace(),
361 OrigA.getLinkage(), OrigA.getName(), &Dst);
362 NewA->copyAttributesFrom(&OrigA);
363 VMap[&OrigA] = NewA;
364 return NewA;
365 }
366
cloneModuleFlagsMetadata(Module & Dst,const Module & Src,ValueToValueMapTy & VMap)367 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
368 ValueToValueMapTy &VMap) {
369 auto *MFs = Src.getModuleFlagsMetadata();
370 if (!MFs)
371 return;
372 for (auto *MF : MFs->operands())
373 Dst.addModuleFlag(MapMetadata(MF, VMap));
374 }
375
376 } // End namespace orc.
377 } // End namespace llvm.
378