1 //===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
11 #define LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
12
13 #include "llvm-c/OrcBindings.h"
14 #include "llvm-c/TargetMachine.h"
15 #include "llvm/ADT/STLExtras.h"
16 #include "llvm/ADT/StringRef.h"
17 #include "llvm/ExecutionEngine/JITSymbol.h"
18 #include "llvm/ExecutionEngine/JITEventListener.h"
19 #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
20 #include "llvm/ExecutionEngine/Orc/CompileUtils.h"
21 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
22 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
23 #include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
24 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
25 #include "llvm/ExecutionEngine/RuntimeDyld.h"
26 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
27 #include "llvm/IR/DataLayout.h"
28 #include "llvm/IR/Mangler.h"
29 #include "llvm/IR/Module.h"
30 #include "llvm/Support/CBindingWrapping.h"
31 #include "llvm/Support/Error.h"
32 #include "llvm/Support/raw_ostream.h"
33 #include "llvm/Target/TargetMachine.h"
34 #include <algorithm>
35 #include <cstdint>
36 #include <functional>
37 #include <map>
38 #include <memory>
39 #include <set>
40 #include <string>
41 #include <vector>
42
43 namespace llvm {
44
45 class OrcCBindingsStack;
46
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack,LLVMOrcJITStackRef)47 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef)
48 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
49
50 namespace detail {
51
52 // FIXME: Kill this off once the Layer concept becomes an interface.
53 class GenericLayer {
54 public:
55 virtual ~GenericLayer() = default;
56
57 virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
58 bool ExportedSymbolsOnly) = 0;
59 virtual Error removeModule(orc::VModuleKey K) = 0;
60 };
61
62 template <typename LayerT> class GenericLayerImpl : public GenericLayer {
63 public:
64 GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
65
66 JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
67 bool ExportedSymbolsOnly) override {
68 return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
69 }
70
71 Error removeModule(orc::VModuleKey K) override {
72 return Layer.removeModule(K);
73 }
74
75 private:
76 LayerT &Layer;
77 };
78
79 template <>
80 class GenericLayerImpl<orc::RTDyldObjectLinkingLayer> : public GenericLayer {
81 private:
82 using LayerT = orc::RTDyldObjectLinkingLayer;
83 public:
84 GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
85
86 JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
87 bool ExportedSymbolsOnly) override {
88 return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
89 }
90
91 Error removeModule(orc::VModuleKey K) override {
92 return Layer.removeObject(K);
93 }
94
95 private:
96 LayerT &Layer;
97 };
98
99 template <typename LayerT>
100 std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) {
101 return llvm::make_unique<GenericLayerImpl<LayerT>>(Layer);
102 }
103
104 } // end namespace detail
105
106 class OrcCBindingsStack {
107 public:
108
109 using CompileCallbackMgr = orc::JITCompileCallbackManager;
110 using ObjLayerT = orc::RTDyldObjectLinkingLayer;
111 using CompileLayerT = orc::IRCompileLayer<ObjLayerT, orc::SimpleCompiler>;
112 using CODLayerT =
113 orc::CompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>;
114
115 using CallbackManagerBuilder =
116 std::function<std::unique_ptr<CompileCallbackMgr>()>;
117
118 using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT;
119
120 private:
121
122 using OwningObject = object::OwningBinary<object::ObjectFile>;
123
124 class CBindingsResolver : public orc::SymbolResolver {
125 public:
CBindingsResolver(OrcCBindingsStack & Stack,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)126 CBindingsResolver(OrcCBindingsStack &Stack,
127 LLVMOrcSymbolResolverFn ExternalResolver,
128 void *ExternalResolverCtx)
129 : Stack(Stack), ExternalResolver(std::move(ExternalResolver)),
130 ExternalResolverCtx(std::move(ExternalResolverCtx)) {}
131
132 orc::SymbolFlagsMap
lookupFlags(const orc::SymbolNameSet & Symbols)133 lookupFlags(const orc::SymbolNameSet &Symbols) override {
134 orc::SymbolFlagsMap SymbolFlags;
135
136 for (auto &S : Symbols) {
137 if (auto Sym = findSymbol(*S))
138 SymbolFlags[S] = Sym.getFlags();
139 else if (auto Err = Sym.takeError()) {
140 Stack.reportError(std::move(Err));
141 return orc::SymbolFlagsMap();
142 }
143 }
144
145 return SymbolFlags;
146 }
147
148 orc::SymbolNameSet
lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query,orc::SymbolNameSet Symbols)149 lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query,
150 orc::SymbolNameSet Symbols) override {
151 orc::SymbolNameSet UnresolvedSymbols;
152
153 for (auto &S : Symbols) {
154 if (auto Sym = findSymbol(*S)) {
155 if (auto Addr = Sym.getAddress()) {
156 Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags()));
157 Query->notifySymbolReady();
158 } else {
159 Stack.ES.legacyFailQuery(*Query, Addr.takeError());
160 return orc::SymbolNameSet();
161 }
162 } else if (auto Err = Sym.takeError()) {
163 Stack.ES.legacyFailQuery(*Query, std::move(Err));
164 return orc::SymbolNameSet();
165 } else
166 UnresolvedSymbols.insert(S);
167 }
168
169 if (Query->isFullyResolved())
170 Query->handleFullyResolved();
171
172 if (Query->isFullyReady())
173 Query->handleFullyReady();
174
175 return UnresolvedSymbols;
176 }
177
178 private:
findSymbol(const std::string & Name)179 JITSymbol findSymbol(const std::string &Name) {
180 // Search order:
181 // 1. JIT'd symbols.
182 // 2. Runtime overrides.
183 // 3. External resolver (if present).
184
185 if (auto Sym = Stack.CODLayer.findSymbol(Name, true))
186 return Sym;
187 else if (auto Err = Sym.takeError())
188 return Sym.takeError();
189
190 if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name))
191 return Sym;
192
193 if (ExternalResolver)
194 return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx),
195 JITSymbolFlags::Exported);
196
197 return JITSymbol(nullptr);
198 }
199
200 OrcCBindingsStack &Stack;
201 LLVMOrcSymbolResolverFn ExternalResolver;
202 void *ExternalResolverCtx = nullptr;
203 };
204
205 public:
OrcCBindingsStack(TargetMachine & TM,IndirectStubsManagerBuilder IndirectStubsMgrBuilder)206 OrcCBindingsStack(TargetMachine &TM,
207 IndirectStubsManagerBuilder IndirectStubsMgrBuilder)
208 : CCMgr(createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0)),
209 DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()),
210 ObjectLayer(ES,
211 [this](orc::VModuleKey K) {
212 auto ResolverI = Resolvers.find(K);
213 assert(ResolverI != Resolvers.end() &&
214 "No resolver for module K");
215 auto Resolver = std::move(ResolverI->second);
216 Resolvers.erase(ResolverI);
217 return ObjLayerT::Resources{
218 std::make_shared<SectionMemoryManager>(), Resolver};
219 },
220 nullptr,
221 [this](orc::VModuleKey K, const object::ObjectFile &Obj,
222 const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
223 this->notifyFinalized(K, Obj, LoadedObjInfo);
224 },
225 [this](orc::VModuleKey K, const object::ObjectFile &Obj) {
226 this->notifyFreed(K, Obj);
227 }),
228 CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)),
229 CODLayer(ES, CompileLayer,
230 [this](orc::VModuleKey K) {
231 auto ResolverI = Resolvers.find(K);
232 assert(ResolverI != Resolvers.end() &&
233 "No resolver for module K");
234 return ResolverI->second;
235 },
236 [this](orc::VModuleKey K,
237 std::shared_ptr<orc::SymbolResolver> Resolver) {
238 assert(!Resolvers.count(K) && "Resolver already present");
239 Resolvers[K] = std::move(Resolver);
240 },
241 [](Function &F) { return std::set<Function *>({&F}); },
242 *this->CCMgr, std::move(IndirectStubsMgrBuilder), false),
243 CXXRuntimeOverrides(
244 [this](const std::string &S) { return mangle(S); }) {}
245
shutdown()246 LLVMOrcErrorCode shutdown() {
247 // Run any destructors registered with __cxa_atexit.
248 CXXRuntimeOverrides.runDestructors();
249 // Run any IR destructors.
250 for (auto &DtorRunner : IRStaticDestructorRunners)
251 if (auto Err = DtorRunner.runViaLayer(*this))
252 return mapError(std::move(Err));
253 return LLVMOrcErrSuccess;
254 }
255
mangle(StringRef Name)256 std::string mangle(StringRef Name) {
257 std::string MangledName;
258 {
259 raw_string_ostream MangledNameStream(MangledName);
260 Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
261 }
262 return MangledName;
263 }
264
265 template <typename PtrTy>
fromTargetAddress(JITTargetAddress Addr)266 static PtrTy fromTargetAddress(JITTargetAddress Addr) {
267 return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr));
268 }
269
270
271 LLVMOrcErrorCode
createLazyCompileCallback(JITTargetAddress & RetAddr,LLVMOrcLazyCompileCallbackFn Callback,void * CallbackCtx)272 createLazyCompileCallback(JITTargetAddress &RetAddr,
273 LLVMOrcLazyCompileCallbackFn Callback,
274 void *CallbackCtx) {
275 auto WrappedCallback = [=]() -> JITTargetAddress {
276 return Callback(wrap(this), CallbackCtx);
277 };
278
279 if (auto CCAddr = CCMgr->getCompileCallback(std::move(WrappedCallback))) {
280 RetAddr = *CCAddr;
281 return LLVMOrcErrSuccess;
282 } else
283 return mapError(CCAddr.takeError());
284 }
285
createIndirectStub(StringRef StubName,JITTargetAddress Addr)286 LLVMOrcErrorCode createIndirectStub(StringRef StubName,
287 JITTargetAddress Addr) {
288 return mapError(
289 IndirectStubsMgr->createStub(StubName, Addr, JITSymbolFlags::Exported));
290 }
291
setIndirectStubPointer(StringRef Name,JITTargetAddress Addr)292 LLVMOrcErrorCode setIndirectStubPointer(StringRef Name,
293 JITTargetAddress Addr) {
294 return mapError(IndirectStubsMgr->updatePointer(Name, Addr));
295 }
296 template <typename LayerT>
297 LLVMOrcErrorCode
addIRModule(orc::VModuleKey & RetKey,LayerT & Layer,std::unique_ptr<Module> M,std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)298 addIRModule(orc::VModuleKey &RetKey, LayerT &Layer, std::unique_ptr<Module> M,
299 std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr,
300 LLVMOrcSymbolResolverFn ExternalResolver,
301 void *ExternalResolverCtx) {
302
303 // Attach a data-layout if one isn't already present.
304 if (M->getDataLayout().isDefault())
305 M->setDataLayout(DL);
306
307 // Record the static constructors and destructors. We have to do this before
308 // we hand over ownership of the module to the JIT.
309 std::vector<std::string> CtorNames, DtorNames;
310 for (auto Ctor : orc::getConstructors(*M))
311 CtorNames.push_back(mangle(Ctor.Func->getName()));
312 for (auto Dtor : orc::getDestructors(*M))
313 DtorNames.push_back(mangle(Dtor.Func->getName()));
314
315 // Add the module to the JIT.
316 RetKey = ES.allocateVModule();
317 Resolvers[RetKey] = std::make_shared<CBindingsResolver>(
318 *this, ExternalResolver, ExternalResolverCtx);
319 if (auto Err = Layer.addModule(RetKey, std::move(M)))
320 return mapError(std::move(Err));
321
322 KeyLayers[RetKey] = detail::createGenericLayer(Layer);
323
324 // Run the static constructors, and save the static destructor runner for
325 // execution when the JIT is torn down.
326 orc::CtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames),
327 RetKey);
328 if (auto Err = CtorRunner.runViaLayer(*this))
329 return mapError(std::move(Err));
330
331 IRStaticDestructorRunners.emplace_back(std::move(DtorNames), RetKey);
332
333 return LLVMOrcErrSuccess;
334 }
335
addIRModuleEager(orc::VModuleKey & RetKey,std::unique_ptr<Module> M,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)336 LLVMOrcErrorCode addIRModuleEager(orc::VModuleKey &RetKey,
337 std::unique_ptr<Module> M,
338 LLVMOrcSymbolResolverFn ExternalResolver,
339 void *ExternalResolverCtx) {
340 return addIRModule(RetKey, CompileLayer, std::move(M),
341 llvm::make_unique<SectionMemoryManager>(),
342 std::move(ExternalResolver), ExternalResolverCtx);
343 }
344
addIRModuleLazy(orc::VModuleKey & RetKey,std::unique_ptr<Module> M,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)345 LLVMOrcErrorCode addIRModuleLazy(orc::VModuleKey &RetKey,
346 std::unique_ptr<Module> M,
347 LLVMOrcSymbolResolverFn ExternalResolver,
348 void *ExternalResolverCtx) {
349 return addIRModule(RetKey, CODLayer, std::move(M),
350 llvm::make_unique<SectionMemoryManager>(),
351 std::move(ExternalResolver), ExternalResolverCtx);
352 }
353
removeModule(orc::VModuleKey K)354 LLVMOrcErrorCode removeModule(orc::VModuleKey K) {
355 // FIXME: Should error release the module key?
356 if (auto Err = KeyLayers[K]->removeModule(K))
357 return mapError(std::move(Err));
358 ES.releaseVModule(K);
359 KeyLayers.erase(K);
360 return LLVMOrcErrSuccess;
361 }
362
addObject(orc::VModuleKey & RetKey,std::unique_ptr<MemoryBuffer> ObjBuffer,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)363 LLVMOrcErrorCode addObject(orc::VModuleKey &RetKey,
364 std::unique_ptr<MemoryBuffer> ObjBuffer,
365 LLVMOrcSymbolResolverFn ExternalResolver,
366 void *ExternalResolverCtx) {
367 if (auto Obj = object::ObjectFile::createObjectFile(
368 ObjBuffer->getMemBufferRef())) {
369
370 RetKey = ES.allocateVModule();
371 Resolvers[RetKey] = std::make_shared<CBindingsResolver>(
372 *this, ExternalResolver, ExternalResolverCtx);
373
374 if (auto Err = ObjectLayer.addObject(RetKey, std::move(ObjBuffer)))
375 return mapError(std::move(Err));
376
377 KeyLayers[RetKey] = detail::createGenericLayer(ObjectLayer);
378
379 return LLVMOrcErrSuccess;
380 } else
381 return mapError(Obj.takeError());
382 }
383
findSymbol(const std::string & Name,bool ExportedSymbolsOnly)384 JITSymbol findSymbol(const std::string &Name,
385 bool ExportedSymbolsOnly) {
386 if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly))
387 return Sym;
388 return CODLayer.findSymbol(mangle(Name), ExportedSymbolsOnly);
389 }
390
findSymbolIn(orc::VModuleKey K,const std::string & Name,bool ExportedSymbolsOnly)391 JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
392 bool ExportedSymbolsOnly) {
393 assert(KeyLayers.count(K) && "looking up symbol in unknown module");
394 return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly);
395 }
396
findSymbolAddress(JITTargetAddress & RetAddr,const std::string & Name,bool ExportedSymbolsOnly)397 LLVMOrcErrorCode findSymbolAddress(JITTargetAddress &RetAddr,
398 const std::string &Name,
399 bool ExportedSymbolsOnly) {
400 RetAddr = 0;
401 if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) {
402 // Successful lookup, non-null symbol:
403 if (auto AddrOrErr = Sym.getAddress()) {
404 RetAddr = *AddrOrErr;
405 return LLVMOrcErrSuccess;
406 } else
407 return mapError(AddrOrErr.takeError());
408 } else if (auto Err = Sym.takeError()) {
409 // Lookup failure - report error.
410 return mapError(std::move(Err));
411 }
412 // Otherwise we had a successful lookup but got a null result. We already
413 // set RetAddr to '0' above, so just return success.
414 return LLVMOrcErrSuccess;
415 }
416
findSymbolAddressIn(JITTargetAddress & RetAddr,orc::VModuleKey K,const std::string & Name,bool ExportedSymbolsOnly)417 LLVMOrcErrorCode findSymbolAddressIn(JITTargetAddress &RetAddr,
418 orc::VModuleKey K,
419 const std::string &Name,
420 bool ExportedSymbolsOnly) {
421 RetAddr = 0;
422 if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) {
423 // Successful lookup, non-null symbol:
424 if (auto AddrOrErr = Sym.getAddress()) {
425 RetAddr = *AddrOrErr;
426 return LLVMOrcErrSuccess;
427 } else
428 return mapError(AddrOrErr.takeError());
429 } else if (auto Err = Sym.takeError()) {
430 // Lookup failure - report error.
431 return mapError(std::move(Err));
432 }
433 // Otherwise we had a successful lookup but got a null result. We already
434 // set RetAddr to '0' above, so just return success.
435 return LLVMOrcErrSuccess;
436 }
437
getErrorMessage()438 const std::string &getErrorMessage() const { return ErrMsg; }
439
RegisterJITEventListener(JITEventListener * L)440 void RegisterJITEventListener(JITEventListener *L) {
441 if (!L)
442 return;
443 EventListeners.push_back(L);
444 }
445
UnregisterJITEventListener(JITEventListener * L)446 void UnregisterJITEventListener(JITEventListener *L) {
447 if (!L)
448 return;
449
450 auto I = find(reverse(EventListeners), L);
451 if (I != EventListeners.rend()) {
452 std::swap(*I, EventListeners.back());
453 EventListeners.pop_back();
454 }
455 }
456
457 private:
458
mapError(Error Err)459 LLVMOrcErrorCode mapError(Error Err) {
460 LLVMOrcErrorCode Result = LLVMOrcErrSuccess;
461 handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) {
462 // Handler of last resort.
463 Result = LLVMOrcErrGeneric;
464 ErrMsg = "";
465 raw_string_ostream ErrStream(ErrMsg);
466 EIB.log(ErrStream);
467 });
468 return Result;
469 }
470
reportError(Error Err)471 void reportError(Error Err) {
472 // FIXME: Report errors on the execution session.
473 logAllUnhandledErrors(std::move(Err), errs(), "ORC error: ");
474 };
475
notifyFinalized(orc::VModuleKey K,const object::ObjectFile & Obj,const RuntimeDyld::LoadedObjectInfo & LoadedObjInfo)476 void notifyFinalized(orc::VModuleKey K,
477 const object::ObjectFile &Obj,
478 const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
479 for (auto &Listener : EventListeners)
480 Listener->NotifyObjectEmitted(Obj, LoadedObjInfo);
481 }
482
notifyFreed(orc::VModuleKey K,const object::ObjectFile & Obj)483 void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) {
484 for (auto &Listener : EventListeners)
485 Listener->NotifyFreeingObject(Obj);
486 }
487
488 orc::ExecutionSession ES;
489 std::unique_ptr<CompileCallbackMgr> CCMgr;
490
491 std::vector<JITEventListener *> EventListeners;
492
493 DataLayout DL;
494 SectionMemoryManager CCMgrMemMgr;
495
496 std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr;
497
498 ObjLayerT ObjectLayer;
499 CompileLayerT CompileLayer;
500 CODLayerT CODLayer;
501
502 std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers;
503
504 orc::LocalCXXRuntimeOverrides CXXRuntimeOverrides;
505 std::vector<orc::CtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners;
506 std::string ErrMsg;
507
508 std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>> Resolvers;
509 };
510
511 } // end namespace llvm
512
513 #endif // LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
514