1 // Copyright 2020 The SwiftShader Authors. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "LLVMReactor.hpp"
16
17 #include "Debug.hpp"
18 #include "ExecutableMemory.hpp"
19 #include "LLVMAsm.hpp"
20 #include "PragmaInternals.hpp"
21 #include "Routine.hpp"
22
23 // TODO(b/143539525): Eliminate when warning has been fixed.
24 #ifdef _MSC_VER
25 __pragma(warning(push))
26 __pragma(warning(disable : 4146)) // unary minus operator applied to unsigned type, result still unsigned
27 #endif
28
29 #include "llvm/ExecutionEngine/Orc/CompileUtils.h"
30 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
31 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
32 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
33 #include "llvm/IR/DiagnosticInfo.h"
34 #include "llvm/IR/Verifier.h"
35 #include "llvm/Support/CommandLine.h"
36 #include "llvm/Support/Host.h"
37 #include "llvm/Support/TargetSelect.h"
38 #include "llvm/Transforms/InstCombine/InstCombine.h"
39 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
40 #include "llvm/Transforms/Scalar.h"
41 #include "llvm/Transforms/Scalar/GVN.h"
42
43 #if LLVM_VERSION_MAJOR >= 13 // New pass manager
44 # include "llvm/IR/PassManager.h"
45 # include "llvm/Passes/PassBuilder.h"
46 # include "llvm/Transforms/Scalar/ADCE.h"
47 # include "llvm/Transforms/Scalar/DeadStoreElimination.h"
48 # include "llvm/Transforms/Scalar/EarlyCSE.h"
49 # include "llvm/Transforms/Scalar/LICM.h"
50 # include "llvm/Transforms/Scalar/Reassociate.h"
51 # include "llvm/Transforms/Scalar/SCCP.h"
52 # include "llvm/Transforms/Scalar/SROA.h"
53 # include "llvm/Transforms/Scalar/SimplifyCFG.h"
54 #else // Legacy pass manager
55 # include "llvm/IR/LegacyPassManager.h"
56 # include "llvm/Pass.h"
57 # include "llvm/Transforms/Coroutines.h"
58 # include "llvm/Transforms/IPO.h"
59 #endif
60
61 #ifdef _MSC_VER
62 __pragma(warning(pop))
63 #endif
64
65 #if defined(_WIN64)
66 extern "C" void __chkstk();
67 #elif defined(_WIN32)
68 extern "C" void _chkstk();
69 #endif
70
71 #ifdef __ARM_EABI__
72 extern "C" signed __aeabi_idivmod();
73 #endif
74
75 #if __has_feature(memory_sanitizer)
76
77 // TODO(b/155148722): Remove when we no longer unpoison any writes.
78 # include "sanitizer/msan_interface.h"
79
80 # include <dlfcn.h> // dlsym()
81
82 // MemorySanitizer uses thread-local storage (TLS) data arrays for passing around
83 // the 'shadow' values of function arguments and return values. The LLVM JIT can't
84 // access TLS directly, but it calls __emutls_get_address() to obtain the address.
85 // Typically, it would be passed a pointer to an __emutls_control structure with a
86 // name starting with "__emutls_v." that represents the TLS. Both the address of
87 // __emutls_get_address and the __emutls_v. structures are provided to the JIT by
88 // the symbol resolver, which can be overridden.
89 // We take advantage of this by substituting __emutls_get_address() with our own
90 // implementation, namely rr::getTLSAddress(), and substituting the __emutls_v
91 // variables with rr::MSanTLS enums. getTLSAddress() can then provide the address
92 // of the real TLS variable corresponding to the enum, in statically compiled C++.
93
94 // Forward declare the real TLS variables used by MemorySanitizer. These are
95 // defined in llvm-project/compiler-rt/lib/msan/msan.cpp.
96 extern __thread unsigned long long __msan_param_tls[];
97 extern __thread unsigned long long __msan_retval_tls[];
98 extern __thread unsigned long long __msan_va_arg_tls[];
99 extern __thread unsigned long long __msan_va_arg_overflow_size_tls;
100
101 namespace rr {
102
103 enum class MSanTLS
104 {
105 param = 1, // __msan_param_tls
106 retval, // __msan_retval_tls
107 va_arg, // __msan_va_arg_tls
108 va_arg_overflow_size // __msan_va_arg_overflow_size_tls
109 };
110
getTLSAddress(void * control)111 static void *getTLSAddress(void *control)
112 {
113 auto tlsIndex = static_cast<MSanTLS>(reinterpret_cast<uintptr_t>(control));
114 switch(tlsIndex)
115 {
116
117 case MSanTLS::param: return reinterpret_cast<void *>(&__msan_param_tls);
118 case MSanTLS::retval: return reinterpret_cast<void *>(&__msan_retval_tls);
119 case MSanTLS::va_arg: return reinterpret_cast<void *>(&__msan_va_arg_tls);
120 case MSanTLS::va_arg_overflow_size: return reinterpret_cast<void *>(&__msan_va_arg_overflow_size_tls);
121 default:
122 UNSUPPORTED("MemorySanitizer used an unrecognized TLS variable: %d", tlsIndex);
123 return nullptr;
124 }
125 }
126
127 } // namespace rr
128 #endif
129
130 namespace {
131
132 // TODO(b/174587935): Eliminate command-line parsing.
parseCommandLineOptionsOnce(int argc,const char * const * argv)133 bool parseCommandLineOptionsOnce(int argc, const char *const *argv)
134 {
135 // Use a static immediately invoked lambda to make this thread safe
136 static auto initialized = [=]() {
137 return llvm::cl::ParseCommandLineOptions(argc, argv);
138 }();
139
140 return initialized;
141 }
142
143 // JITGlobals is a singleton that holds all the immutable machine specific
144 // information for the host device.
145 class JITGlobals
146 {
147 public:
148 static JITGlobals *get();
149
150 llvm::orc::JITTargetMachineBuilder getTargetMachineBuilder(rr::Optimization::Level optLevel) const;
151 const llvm::DataLayout &getDataLayout() const;
152 const llvm::Triple &getTargetTriple() const;
153
154 private:
155 JITGlobals(llvm::orc::JITTargetMachineBuilder &&jitTargetMachineBuilder, llvm::DataLayout &&dataLayout);
156
157 static llvm::CodeGenOpt::Level toLLVM(rr::Optimization::Level level);
158
159 const llvm::orc::JITTargetMachineBuilder jitTargetMachineBuilder;
160 const llvm::DataLayout dataLayout;
161 };
162
get()163 JITGlobals *JITGlobals::get()
164 {
165 static JITGlobals instance = [] {
166 const char *argv[] = {
167 "Reactor",
168 #if defined(__i386__) || defined(__x86_64__)
169 "-x86-asm-syntax=intel", // Use Intel syntax rather than the default AT&T
170 #endif
171 #if LLVM_VERSION_MAJOR <= 12
172 "-warn-stack-size=524288" // Warn when a function uses more than 512 KiB of stack memory
173 #endif
174 };
175
176 parseCommandLineOptionsOnce(sizeof(argv) / sizeof(argv[0]), argv);
177
178 llvm::InitializeNativeTarget();
179 llvm::InitializeNativeTargetAsmPrinter();
180 llvm::InitializeNativeTargetAsmParser();
181
182 // TODO(b/171236524): JITTargetMachineBuilder::detectHost() currently uses the target triple of the host,
183 // rather than a valid triple for the current process. Once fixed, we can use that function instead.
184 llvm::orc::JITTargetMachineBuilder jitTargetMachineBuilder(llvm::Triple(LLVM_DEFAULT_TARGET_TRIPLE));
185
186 // Retrieve host CPU name and sub-target features and add them to builder.
187 // Relocation model, code model and codegen opt level are kept to default values.
188 llvm::StringMap<bool> cpuFeatures;
189 bool ok = llvm::sys::getHostCPUFeatures(cpuFeatures);
190
191 #if defined(__i386__) || defined(__x86_64__) || \
192 (defined(__linux__) && (defined(__arm__) || defined(__aarch64__)))
193 ASSERT_MSG(ok, "llvm::sys::getHostCPUFeatures returned false");
194 #else
195 (void)ok; // getHostCPUFeatures always returns false on other platforms
196 #endif
197
198 for(auto &feature : cpuFeatures)
199 {
200 jitTargetMachineBuilder.getFeatures().AddFeature(feature.first(), feature.second);
201 }
202
203 #if LLVM_VERSION_MAJOR >= 11 /* TODO(b/165000222): Unconditional after LLVM 11 upgrade */
204 jitTargetMachineBuilder.setCPU(std::string(llvm::sys::getHostCPUName()));
205 #else
206 jitTargetMachineBuilder.setCPU(llvm::sys::getHostCPUName());
207 #endif
208
209 // Reactor's MemorySanitizer support depends on intercepting __emutls_get_address calls.
210 ASSERT(!__has_feature(memory_sanitizer) || (jitTargetMachineBuilder.getOptions().ExplicitEmulatedTLS &&
211 jitTargetMachineBuilder.getOptions().EmulatedTLS));
212
213 auto dataLayout = jitTargetMachineBuilder.getDefaultDataLayoutForTarget();
214 ASSERT_MSG(dataLayout, "JITTargetMachineBuilder::getDefaultDataLayoutForTarget() failed");
215
216 return JITGlobals(std::move(jitTargetMachineBuilder), std::move(dataLayout.get()));
217 }();
218
219 return &instance;
220 }
221
getTargetMachineBuilder(rr::Optimization::Level optLevel) const222 llvm::orc::JITTargetMachineBuilder JITGlobals::getTargetMachineBuilder(rr::Optimization::Level optLevel) const
223 {
224 llvm::orc::JITTargetMachineBuilder out = jitTargetMachineBuilder;
225 out.setCodeGenOptLevel(toLLVM(optLevel));
226
227 return out;
228 }
229
getDataLayout() const230 const llvm::DataLayout &JITGlobals::getDataLayout() const
231 {
232 return dataLayout;
233 }
234
getTargetTriple() const235 const llvm::Triple &JITGlobals::getTargetTriple() const
236 {
237 return jitTargetMachineBuilder.getTargetTriple();
238 }
239
JITGlobals(llvm::orc::JITTargetMachineBuilder && jitTargetMachineBuilder,llvm::DataLayout && dataLayout)240 JITGlobals::JITGlobals(llvm::orc::JITTargetMachineBuilder &&jitTargetMachineBuilder, llvm::DataLayout &&dataLayout)
241 : jitTargetMachineBuilder(jitTargetMachineBuilder)
242 , dataLayout(dataLayout)
243 {
244 }
245
toLLVM(rr::Optimization::Level level)246 llvm::CodeGenOpt::Level JITGlobals::toLLVM(rr::Optimization::Level level)
247 {
248 // TODO(b/173257647): MemorySanitizer instrumentation produces IR which takes
249 // a lot longer to process by the machine code optimization passes. Disabling
250 // them has a negligible effect on code quality but compiles much faster.
251 if(__has_feature(memory_sanitizer))
252 {
253 return llvm::CodeGenOpt::None;
254 }
255
256 switch(level)
257 {
258 case rr::Optimization::Level::None: return llvm::CodeGenOpt::None;
259 case rr::Optimization::Level::Less: return llvm::CodeGenOpt::Less;
260 case rr::Optimization::Level::Default: return llvm::CodeGenOpt::Default;
261 case rr::Optimization::Level::Aggressive: return llvm::CodeGenOpt::Aggressive;
262 default: UNREACHABLE("Unknown Optimization Level %d", int(level));
263 }
264
265 return llvm::CodeGenOpt::Default;
266 }
267
268 class MemoryMapper final : public llvm::SectionMemoryManager::MemoryMapper
269 {
270 public:
MemoryMapper()271 MemoryMapper() {}
~MemoryMapper()272 ~MemoryMapper() final {}
273
allocateMappedMemory(llvm::SectionMemoryManager::AllocationPurpose purpose,size_t numBytes,const llvm::sys::MemoryBlock * const nearBlock,unsigned flags,std::error_code & errorCode)274 llvm::sys::MemoryBlock allocateMappedMemory(
275 llvm::SectionMemoryManager::AllocationPurpose purpose,
276 size_t numBytes, const llvm::sys::MemoryBlock *const nearBlock,
277 unsigned flags, std::error_code &errorCode) final
278 {
279 errorCode = std::error_code();
280
281 // Round up numBytes to page size.
282 size_t pageSize = rr::memoryPageSize();
283 numBytes = (numBytes + pageSize - 1) & ~(pageSize - 1);
284
285 bool need_exec =
286 purpose == llvm::SectionMemoryManager::AllocationPurpose::Code;
287 void *addr = rr::allocateMemoryPages(
288 numBytes, flagsToPermissions(flags), need_exec);
289 if(!addr)
290 return llvm::sys::MemoryBlock();
291 return llvm::sys::MemoryBlock(addr, numBytes);
292 }
293
protectMappedMemory(const llvm::sys::MemoryBlock & block,unsigned flags)294 std::error_code protectMappedMemory(const llvm::sys::MemoryBlock &block,
295 unsigned flags)
296 {
297 // Round down base address to align with a page boundary. This matches
298 // DefaultMMapper behavior.
299 void *addr = block.base();
300 size_t size = block.allocatedSize();
301 size_t pageSize = rr::memoryPageSize();
302 addr = reinterpret_cast<void *>(
303 reinterpret_cast<uintptr_t>(addr) & ~(pageSize - 1));
304 size += reinterpret_cast<uintptr_t>(block.base()) -
305 reinterpret_cast<uintptr_t>(addr);
306
307 rr::protectMemoryPages(addr, size, flagsToPermissions(flags));
308 return std::error_code();
309 }
310
releaseMappedMemory(llvm::sys::MemoryBlock & block)311 std::error_code releaseMappedMemory(llvm::sys::MemoryBlock &block)
312 {
313 size_t size = block.allocatedSize();
314
315 rr::deallocateMemoryPages(block.base(), size);
316 return std::error_code();
317 }
318
319 private:
flagsToPermissions(unsigned flags)320 int flagsToPermissions(unsigned flags)
321 {
322 int result = 0;
323 if(flags & llvm::sys::Memory::MF_READ)
324 {
325 result |= rr::PERMISSION_READ;
326 }
327 if(flags & llvm::sys::Memory::MF_WRITE)
328 {
329 result |= rr::PERMISSION_WRITE;
330 }
331 if(flags & llvm::sys::Memory::MF_EXEC)
332 {
333 result |= rr::PERMISSION_EXECUTE;
334 }
335 return result;
336 }
337 };
338
339 template<typename T>
alignUp(T val,T alignment)340 T alignUp(T val, T alignment)
341 {
342 return alignment * ((val + alignment - 1) / alignment);
343 }
344
alignedAlloc(size_t size,size_t alignment)345 void *alignedAlloc(size_t size, size_t alignment)
346 {
347 ASSERT(alignment < 256);
348 auto allocation = new uint8_t[size + sizeof(uint8_t) + alignment];
349 auto aligned = allocation;
350 aligned += sizeof(uint8_t); // Make space for the base-address offset.
351 aligned = reinterpret_cast<uint8_t *>(alignUp(reinterpret_cast<uintptr_t>(aligned), alignment)); // align
352 auto offset = static_cast<uint8_t>(aligned - allocation);
353 aligned[-1] = offset;
354 return aligned;
355 }
356
alignedFree(void * ptr)357 void alignedFree(void *ptr)
358 {
359 auto aligned = reinterpret_cast<uint8_t *>(ptr);
360 auto offset = aligned[-1];
361 auto allocation = aligned - offset;
362 delete[] allocation;
363 }
364
365 template<typename T>
atomicLoad(void * ptr,void * ret,llvm::AtomicOrdering ordering)366 static void atomicLoad(void *ptr, void *ret, llvm::AtomicOrdering ordering)
367 {
368 *reinterpret_cast<T *>(ret) = std::atomic_load_explicit<T>(reinterpret_cast<std::atomic<T> *>(ptr), rr::atomicOrdering(ordering));
369 }
370
371 template<typename T>
atomicStore(void * ptr,void * val,llvm::AtomicOrdering ordering)372 static void atomicStore(void *ptr, void *val, llvm::AtomicOrdering ordering)
373 {
374 std::atomic_store_explicit<T>(reinterpret_cast<std::atomic<T> *>(ptr), *reinterpret_cast<T *>(val), rr::atomicOrdering(ordering));
375 }
376
377 #ifdef __ANDROID__
378 template<typename F>
sync_fetch_and_op(uint32_t volatile * ptr,uint32_t val,F f)379 static uint32_t sync_fetch_and_op(uint32_t volatile *ptr, uint32_t val, F f)
380 {
381 // Build an arbitrary op out of looped CAS
382 for(;;)
383 {
384 uint32_t expected = *ptr;
385 uint32_t desired = f(expected, val);
386
387 if(expected == __sync_val_compare_and_swap_4(ptr, expected, desired))
388 {
389 return expected;
390 }
391 }
392 }
393 #endif
394
395 #if LLVM_VERSION_MAJOR >= 11 /* TODO(b/165000222): Unconditional after LLVM 11 upgrade */
396 class ExternalSymbolGenerator : public llvm::orc::DefinitionGenerator
397 #else
398 class ExternalSymbolGenerator : public llvm::orc::JITDylib::DefinitionGenerator
399 #endif
400 {
401 struct Atomic
402 {
load__anon61150e790111::ExternalSymbolGenerator::Atomic403 static void load(size_t size, void *ptr, void *ret, llvm::AtomicOrdering ordering)
404 {
405 switch(size)
406 {
407 case 1: atomicLoad<uint8_t>(ptr, ret, ordering); break;
408 case 2: atomicLoad<uint16_t>(ptr, ret, ordering); break;
409 case 4: atomicLoad<uint32_t>(ptr, ret, ordering); break;
410 case 8: atomicLoad<uint64_t>(ptr, ret, ordering); break;
411 default:
412 UNIMPLEMENTED_NO_BUG("Atomic::load(size: %d)", int(size));
413 }
414 }
store__anon61150e790111::ExternalSymbolGenerator::Atomic415 static void store(size_t size, void *ptr, void *ret, llvm::AtomicOrdering ordering)
416 {
417 switch(size)
418 {
419 case 1: atomicStore<uint8_t>(ptr, ret, ordering); break;
420 case 2: atomicStore<uint16_t>(ptr, ret, ordering); break;
421 case 4: atomicStore<uint32_t>(ptr, ret, ordering); break;
422 case 8: atomicStore<uint64_t>(ptr, ret, ordering); break;
423 default:
424 UNIMPLEMENTED_NO_BUG("Atomic::store(size: %d)", int(size));
425 }
426 }
427 };
428
nop()429 static void nop() {}
neverCalled()430 static void neverCalled() { UNREACHABLE("Should never be called"); }
431
coroutine_alloc_frame(size_t size)432 static void *coroutine_alloc_frame(size_t size) { return alignedAlloc(size, 16); }
coroutine_free_frame(void * ptr)433 static void coroutine_free_frame(void *ptr) { alignedFree(ptr); }
434
435 #ifdef __ANDROID__
436 // forwarders since we can't take address of builtins
sync_synchronize()437 static void sync_synchronize() { __sync_synchronize(); }
sync_fetch_and_add_4(uint32_t * ptr,uint32_t val)438 static uint32_t sync_fetch_and_add_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_add_4(ptr, val); }
sync_fetch_and_and_4(uint32_t * ptr,uint32_t val)439 static uint32_t sync_fetch_and_and_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_and_4(ptr, val); }
sync_fetch_and_or_4(uint32_t * ptr,uint32_t val)440 static uint32_t sync_fetch_and_or_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_or_4(ptr, val); }
sync_fetch_and_xor_4(uint32_t * ptr,uint32_t val)441 static uint32_t sync_fetch_and_xor_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_xor_4(ptr, val); }
sync_fetch_and_sub_4(uint32_t * ptr,uint32_t val)442 static uint32_t sync_fetch_and_sub_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_sub_4(ptr, val); }
sync_lock_test_and_set_4(uint32_t * ptr,uint32_t val)443 static uint32_t sync_lock_test_and_set_4(uint32_t *ptr, uint32_t val) { return __sync_lock_test_and_set_4(ptr, val); }
sync_val_compare_and_swap_4(uint32_t * ptr,uint32_t expected,uint32_t desired)444 static uint32_t sync_val_compare_and_swap_4(uint32_t *ptr, uint32_t expected, uint32_t desired) { return __sync_val_compare_and_swap_4(ptr, expected, desired); }
445
sync_fetch_and_max_4(uint32_t * ptr,uint32_t val)446 static uint32_t sync_fetch_and_max_4(uint32_t *ptr, uint32_t val)
447 {
448 return sync_fetch_and_op(ptr, val, [](int32_t a, int32_t b) { return std::max(a, b); });
449 }
sync_fetch_and_min_4(uint32_t * ptr,uint32_t val)450 static uint32_t sync_fetch_and_min_4(uint32_t *ptr, uint32_t val)
451 {
452 return sync_fetch_and_op(ptr, val, [](int32_t a, int32_t b) { return std::min(a, b); });
453 }
sync_fetch_and_umax_4(uint32_t * ptr,uint32_t val)454 static uint32_t sync_fetch_and_umax_4(uint32_t *ptr, uint32_t val)
455 {
456 return sync_fetch_and_op(ptr, val, [](uint32_t a, uint32_t b) { return std::max(a, b); });
457 }
sync_fetch_and_umin_4(uint32_t * ptr,uint32_t val)458 static uint32_t sync_fetch_and_umin_4(uint32_t *ptr, uint32_t val)
459 {
460 return sync_fetch_and_op(ptr, val, [](uint32_t a, uint32_t b) { return std::min(a, b); });
461 }
462 #endif
463
464 class Resolver
465 {
466 public:
467 using FunctionMap = llvm::StringMap<void *>;
468
469 FunctionMap functions;
470
Resolver()471 Resolver()
472 {
473 #ifdef ENABLE_RR_PRINT
474 functions.try_emplace("rr::DebugPrintf", reinterpret_cast<void *>(rr::DebugPrintf));
475 #endif
476 functions.try_emplace("nop", reinterpret_cast<void *>(nop));
477 functions.try_emplace("floorf", reinterpret_cast<void *>(floorf));
478 functions.try_emplace("nearbyintf", reinterpret_cast<void *>(nearbyintf));
479 functions.try_emplace("truncf", reinterpret_cast<void *>(truncf));
480 functions.try_emplace("printf", reinterpret_cast<void *>(printf));
481 functions.try_emplace("puts", reinterpret_cast<void *>(puts));
482 functions.try_emplace("fmodf", reinterpret_cast<void *>(fmodf));
483
484 functions.try_emplace("sinf", reinterpret_cast<void *>(sinf));
485 functions.try_emplace("cosf", reinterpret_cast<void *>(cosf));
486 functions.try_emplace("asinf", reinterpret_cast<void *>(asinf));
487 functions.try_emplace("acosf", reinterpret_cast<void *>(acosf));
488 functions.try_emplace("atanf", reinterpret_cast<void *>(atanf));
489 functions.try_emplace("sinhf", reinterpret_cast<void *>(sinhf));
490 functions.try_emplace("coshf", reinterpret_cast<void *>(coshf));
491 functions.try_emplace("tanhf", reinterpret_cast<void *>(tanhf));
492 functions.try_emplace("asinhf", reinterpret_cast<void *>(asinhf));
493 functions.try_emplace("acoshf", reinterpret_cast<void *>(acoshf));
494 functions.try_emplace("atanhf", reinterpret_cast<void *>(atanhf));
495 functions.try_emplace("atan2f", reinterpret_cast<void *>(atan2f));
496 functions.try_emplace("powf", reinterpret_cast<void *>(powf));
497 functions.try_emplace("expf", reinterpret_cast<void *>(expf));
498 functions.try_emplace("logf", reinterpret_cast<void *>(logf));
499 functions.try_emplace("exp2f", reinterpret_cast<void *>(exp2f));
500 functions.try_emplace("log2f", reinterpret_cast<void *>(log2f));
501 functions.try_emplace("fmaf", reinterpret_cast<void *>(fmaf));
502
503 functions.try_emplace("fmod", reinterpret_cast<void *>(static_cast<double (*)(double, double)>(fmod)));
504 functions.try_emplace("sin", reinterpret_cast<void *>(static_cast<double (*)(double)>(sin)));
505 functions.try_emplace("cos", reinterpret_cast<void *>(static_cast<double (*)(double)>(cos)));
506 functions.try_emplace("asin", reinterpret_cast<void *>(static_cast<double (*)(double)>(asin)));
507 functions.try_emplace("acos", reinterpret_cast<void *>(static_cast<double (*)(double)>(acos)));
508 functions.try_emplace("atan", reinterpret_cast<void *>(static_cast<double (*)(double)>(atan)));
509 functions.try_emplace("sinh", reinterpret_cast<void *>(static_cast<double (*)(double)>(sinh)));
510 functions.try_emplace("cosh", reinterpret_cast<void *>(static_cast<double (*)(double)>(cosh)));
511 functions.try_emplace("tanh", reinterpret_cast<void *>(static_cast<double (*)(double)>(tanh)));
512 functions.try_emplace("asinh", reinterpret_cast<void *>(static_cast<double (*)(double)>(asinh)));
513 functions.try_emplace("acosh", reinterpret_cast<void *>(static_cast<double (*)(double)>(acosh)));
514 functions.try_emplace("atanh", reinterpret_cast<void *>(static_cast<double (*)(double)>(atanh)));
515 functions.try_emplace("atan2", reinterpret_cast<void *>(static_cast<double (*)(double, double)>(atan2)));
516 functions.try_emplace("pow", reinterpret_cast<void *>(static_cast<double (*)(double, double)>(pow)));
517 functions.try_emplace("exp", reinterpret_cast<void *>(static_cast<double (*)(double)>(exp)));
518 functions.try_emplace("log", reinterpret_cast<void *>(static_cast<double (*)(double)>(log)));
519 functions.try_emplace("exp2", reinterpret_cast<void *>(static_cast<double (*)(double)>(exp2)));
520 functions.try_emplace("log2", reinterpret_cast<void *>(static_cast<double (*)(double)>(log2)));
521
522 functions.try_emplace("atomic_load", reinterpret_cast<void *>(Atomic::load));
523 functions.try_emplace("atomic_store", reinterpret_cast<void *>(Atomic::store));
524
525 // FIXME(b/119409619): use an allocator here so we can control all memory allocations
526 functions.try_emplace("coroutine_alloc_frame", reinterpret_cast<void *>(coroutine_alloc_frame));
527 functions.try_emplace("coroutine_free_frame", reinterpret_cast<void *>(coroutine_free_frame));
528
529 functions.try_emplace("memset", reinterpret_cast<void *>(memset));
530
531 #ifdef __APPLE__
532 functions.try_emplace("sincosf_stret", reinterpret_cast<void *>(__sincosf_stret));
533 #elif defined(__linux__)
534 functions.try_emplace("sincosf", reinterpret_cast<void *>(sincosf));
535 #elif defined(_WIN64)
536 functions.try_emplace("chkstk", reinterpret_cast<void *>(__chkstk));
537 #elif defined(_WIN32)
538 functions.try_emplace("chkstk", reinterpret_cast<void *>(_chkstk));
539 #endif
540
541 #ifdef __ARM_EABI__
542 functions.try_emplace("aeabi_idivmod", reinterpret_cast<void *>(__aeabi_idivmod));
543 #endif
544 #ifdef __ANDROID__
545 functions.try_emplace("aeabi_unwind_cpp_pr0", reinterpret_cast<void *>(neverCalled));
546 functions.try_emplace("sync_synchronize", reinterpret_cast<void *>(sync_synchronize));
547 functions.try_emplace("sync_fetch_and_add_4", reinterpret_cast<void *>(sync_fetch_and_add_4));
548 functions.try_emplace("sync_fetch_and_and_4", reinterpret_cast<void *>(sync_fetch_and_and_4));
549 functions.try_emplace("sync_fetch_and_or_4", reinterpret_cast<void *>(sync_fetch_and_or_4));
550 functions.try_emplace("sync_fetch_and_xor_4", reinterpret_cast<void *>(sync_fetch_and_xor_4));
551 functions.try_emplace("sync_fetch_and_sub_4", reinterpret_cast<void *>(sync_fetch_and_sub_4));
552 functions.try_emplace("sync_lock_test_and_set_4", reinterpret_cast<void *>(sync_lock_test_and_set_4));
553 functions.try_emplace("sync_val_compare_and_swap_4", reinterpret_cast<void *>(sync_val_compare_and_swap_4));
554 functions.try_emplace("sync_fetch_and_max_4", reinterpret_cast<void *>(sync_fetch_and_max_4));
555 functions.try_emplace("sync_fetch_and_min_4", reinterpret_cast<void *>(sync_fetch_and_min_4));
556 functions.try_emplace("sync_fetch_and_umax_4", reinterpret_cast<void *>(sync_fetch_and_umax_4));
557 functions.try_emplace("sync_fetch_and_umin_4", reinterpret_cast<void *>(sync_fetch_and_umin_4));
558
559 # if defined(__i386__)
560 // TODO(b/172974501): Workaround for an x86-32 issue where an R_386_PC32 relocation is used
561 // When calling a C function from Reactor code, who's address is not associated with any symbol
562 // (since it's an absolute constant), but it still invokes the symbol resolver for "".
563 functions.try_emplace("", nullptr);
564 # endif
565 #endif
566 #if __has_feature(memory_sanitizer)
567 functions.try_emplace("emutls_get_address", reinterpret_cast<void *>(rr::getTLSAddress));
568 functions.try_emplace("emutls_v.__msan_retval_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::retval)));
569 functions.try_emplace("emutls_v.__msan_param_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::param)));
570 functions.try_emplace("emutls_v.__msan_va_arg_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::va_arg)));
571 functions.try_emplace("emutls_v.__msan_va_arg_overflow_size_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::va_arg_overflow_size)));
572
573 // TODO(b/155148722): Remove when we no longer unpoison any writes.
574 functions.try_emplace("msan_unpoison", reinterpret_cast<void *>(__msan_unpoison));
575 functions.try_emplace("msan_unpoison_param", reinterpret_cast<void *>(__msan_unpoison_param));
576 #endif
577 }
578 };
579
tryToGenerate(llvm::orc::LookupState & state,llvm::orc::LookupKind kind,llvm::orc::JITDylib & dylib,llvm::orc::JITDylibLookupFlags flags,const llvm::orc::SymbolLookupSet & set)580 llvm::Error tryToGenerate(
581 #if LLVM_VERSION_MAJOR >= 11 /* TODO(b/165000222): Unconditional after LLVM 11 upgrade */
582 llvm::orc::LookupState &state,
583 #endif
584 llvm::orc::LookupKind kind,
585 llvm::orc::JITDylib &dylib,
586 llvm::orc::JITDylibLookupFlags flags,
587 const llvm::orc::SymbolLookupSet &set) override
588 {
589 static Resolver resolver;
590
591 llvm::orc::SymbolMap symbols;
592
593 #if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
594 std::string missing;
595 #endif // !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
596
597 for(auto symbol : set)
598 {
599 auto name = symbol.first;
600
601 // Trim off any underscores from the start of the symbol. LLVM likes
602 // to append these on macOS.
603 auto trimmed = (*name).drop_while([](char c) { return c == '_'; });
604
605 auto it = resolver.functions.find(trimmed.str());
606 if(it != resolver.functions.end())
607 {
608 symbols[name] = llvm::JITEvaluatedSymbol(
609 static_cast<llvm::JITTargetAddress>(reinterpret_cast<uintptr_t>(it->second)),
610 llvm::JITSymbolFlags::Exported);
611
612 continue;
613 }
614
615 #if __has_feature(memory_sanitizer)
616 // MemorySanitizer uses a dynamically linked runtime. Instrumented routines reference
617 // some symbols from this library. Look them up dynamically in the default namespace.
618 // Note this approach should not be used for other symbols, since they might not be
619 // visible (e.g. due to static linking), we may wish to provide an alternate
620 // implementation, and/or it would be a security vulnerability.
621
622 void *address = dlsym(RTLD_DEFAULT, (*symbol.first).data());
623
624 if(address)
625 {
626 symbols[name] = llvm::JITEvaluatedSymbol(
627 static_cast<llvm::JITTargetAddress>(reinterpret_cast<uintptr_t>(address)),
628 llvm::JITSymbolFlags::Exported);
629
630 continue;
631 }
632 #endif
633
634 #if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
635 missing += (missing.empty() ? "'" : ", '") + (*name).str() + "'";
636 #endif
637 }
638
639 #if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
640 // Missing functions will likely make the module fail in non-obvious ways.
641 if(!missing.empty())
642 {
643 WARN("Missing external functions: %s", missing.c_str());
644 }
645 #endif
646
647 if(symbols.empty())
648 {
649 return llvm::Error::success();
650 }
651
652 return dylib.define(llvm::orc::absoluteSymbols(std::move(symbols)));
653 }
654 };
655
656 // As we must support different LLVM versions, add a generic Unwrap for functions that return Expected<T> or the actual T.
657 // TODO(b/165000222): Remove after LLVM 11 upgrade
658 template<typename T>
Unwrap(llvm::Expected<T> && v)659 T &Unwrap(llvm::Expected<T> &&v)
660 {
661 assert(v);
662 return v.get();
663 }
664 template<typename T>
Unwrap(T && v)665 T &Unwrap(T &&v)
666 {
667 return v;
668 }
669
670 // Sets *fatal to true if a diagnostic is received which makes a routine invalid or unusable.
671 struct FatalDiagnosticsHandler : public llvm::DiagnosticHandler
672 {
FatalDiagnosticsHandler__anon61150e790111::FatalDiagnosticsHandler673 FatalDiagnosticsHandler(bool *fatal)
674 : fatal(fatal)
675 {}
676
handleDiagnostics__anon61150e790111::FatalDiagnosticsHandler677 bool handleDiagnostics(const llvm::DiagnosticInfo &info) override
678 {
679 switch(info.getSeverity())
680 {
681 case llvm::DS_Error:
682 ASSERT_MSG(false, "LLVM JIT compilation failure");
683 *fatal = true;
684 break;
685 case llvm::DS_Warning:
686 if(info.getKind() == llvm::DK_StackSize)
687 {
688 // Stack size limit exceeded
689 *fatal = true;
690 }
691 break;
692 case llvm::DS_Remark:
693 break;
694 case llvm::DS_Note:
695 break;
696 }
697
698 return true; // Diagnostic handled, don't let LLVM print it.
699 }
700
701 bool *fatal;
702 };
703
704 // JITRoutine is a rr::Routine that holds a LLVM JIT session, compiler and
705 // object layer as each routine may require different target machine
706 // settings and no Reactor routine directly links against another.
707 class JITRoutine : public rr::Routine
708 {
709 public:
JITRoutine(std::unique_ptr<llvm::Module> module,std::unique_ptr<llvm::LLVMContext> context,const char * name,llvm::Function ** funcs,size_t count,const rr::Config & config)710 JITRoutine(
711 std::unique_ptr<llvm::Module> module,
712 std::unique_ptr<llvm::LLVMContext> context,
713 const char *name,
714 llvm::Function **funcs,
715 size_t count,
716 const rr::Config &config)
717 : name(name)
718 #if LLVM_VERSION_MAJOR >= 13
719 , session(std::move(Unwrap(llvm::orc::SelfExecutorProcessControl::Create())))
720 #endif
721 , objectLayer(session, [this]() {
722 return std::make_unique<llvm::SectionMemoryManager>(&memoryMapper);
723 })
724 , addresses(count)
725 {
726 bool fatalCompileIssue = false;
727 context->setDiagnosticHandler(std::make_unique<FatalDiagnosticsHandler>(&fatalCompileIssue), true);
728
729 #ifdef ENABLE_RR_DEBUG_INFO
730 // TODO(b/165000222): Update this on next LLVM roll.
731 // https://github.com/llvm/llvm-project/commit/98f2bb4461072347dcca7d2b1b9571b3a6525801
732 // introduces RTDyldObjectLinkingLayer::registerJITEventListener().
733 // The current API does not appear to have any way to bind the
734 // rr::DebugInfo::NotifyFreeingObject event.
735 # if LLVM_VERSION_MAJOR >= 12
736 objectLayer.setNotifyLoaded([](llvm::orc::MaterializationResponsibility &R,
737 const llvm::object::ObjectFile &obj,
__anon61150e790a02(llvm::orc::MaterializationResponsibility &R, const llvm::object::ObjectFile &obj, const llvm::RuntimeDyld::LoadedObjectInfo &l) 738 const llvm::RuntimeDyld::LoadedObjectInfo &l) {
739 static std::atomic<uint64_t> unique_key{ 0 };
740 rr::DebugInfo::NotifyObjectEmitted(unique_key++, obj, l);
741 });
742 # else
743 objectLayer.setNotifyLoaded([](llvm::orc::VModuleKey,
744 const llvm::object::ObjectFile &obj,
__anon61150e790b02(llvm::orc::VModuleKey, const llvm::object::ObjectFile &obj, const llvm::RuntimeDyld::LoadedObjectInfo &l) 745 const llvm::RuntimeDyld::LoadedObjectInfo &l) {
746 static std::atomic<uint64_t> unique_key{ 0 };
747 rr::DebugInfo::NotifyObjectEmitted(unique_key++, obj, l);
748 });
749 # endif
750 #endif // ENABLE_RR_DEBUG_INFO
751
752 if(JITGlobals::get()->getTargetTriple().isOSBinFormatCOFF())
753 {
754 // Hack to support symbol visibility in COFF.
755 // Matches hack in llvm::orc::LLJIT::createObjectLinkingLayer().
756 // See documentation on these functions for more detail.
757 objectLayer.setOverrideObjectFlagsWithResponsibilityFlags(true);
758 objectLayer.setAutoClaimResponsibilityForObjectSymbols(true);
759 }
760
761 llvm::SmallVector<llvm::orc::SymbolStringPtr, 8> functionNames(count);
762 llvm::orc::MangleAndInterner mangle(session, JITGlobals::get()->getDataLayout());
763
764 for(size_t i = 0; i < count; i++)
765 {
766 llvm::Function *func = funcs[i];
767
768 if(!func->hasName())
769 {
770 func->setName("f" + llvm::Twine(i).str());
771 }
772
773 functionNames[i] = mangle(func->getName());
774 }
775
776 #ifdef ENABLE_RR_EMIT_ASM_FILE
777 const auto asmFilename = rr::AsmFile::generateFilename(config.getDebugConfig().asmEmitDir, name);
778 rr::AsmFile::emitAsmFile(asmFilename, JITGlobals::get()->getTargetMachineBuilder(config.getOptimization().getLevel()), *module);
779 #endif
780
781 // Once the module is passed to the compileLayer, the llvm::Functions are freed.
782 // Make sure funcs are not referenced after this point.
783 funcs = nullptr;
784
785 llvm::orc::IRCompileLayer compileLayer(session, objectLayer, std::make_unique<llvm::orc::ConcurrentIRCompiler>(JITGlobals::get()->getTargetMachineBuilder(config.getOptimization().getLevel())));
786 llvm::orc::JITDylib &dylib(Unwrap(session.createJITDylib("<routine>")));
787 dylib.addGenerator(std::make_unique<ExternalSymbolGenerator>());
788
789 llvm::cantFail(compileLayer.add(dylib, llvm::orc::ThreadSafeModule(std::move(module), std::move(context))));
790
791 // Resolve the function addresses.
792 for(size_t i = 0; i < count; i++)
793 {
794 fatalCompileIssue = false; // May be set to true by session.lookup()
795
796 // This is where the actual compilation happens.
797 auto symbol = session.lookup({ &dylib }, functionNames[i]);
798
799 ASSERT_MSG(symbol, "Failed to lookup address of routine function %d: %s",
800 (int)i, llvm::toString(symbol.takeError()).c_str());
801
802 if(fatalCompileIssue)
803 {
804 addresses[i] = nullptr;
805 }
806 else // Successful compilation
807 {
808 addresses[i] = reinterpret_cast<void *>(static_cast<intptr_t>(symbol->getAddress()));
809 }
810 }
811
812 #ifdef ENABLE_RR_EMIT_ASM_FILE
813 rr::AsmFile::fixupAsmFile(asmFilename, addresses);
814 #endif
815 }
816
~JITRoutine()817 ~JITRoutine()
818 {
819 #if LLVM_VERSION_MAJOR >= 11 /* TODO(b/165000222): Unconditional after LLVM 11 upgrade */
820 if(auto err = session.endSession())
821 {
822 session.reportError(std::move(err));
823 }
824 #endif
825 }
826
getEntry(int index) const827 const void *getEntry(int index) const override
828 {
829 return addresses[index];
830 }
831
832 private:
833 std::string name;
834 llvm::orc::ExecutionSession session;
835 MemoryMapper memoryMapper;
836 llvm::orc::RTDyldObjectLinkingLayer objectLayer;
837 std::vector<const void *> addresses;
838 };
839
840 } // anonymous namespace
841
842 namespace rr {
843
JITBuilder(const rr::Config & config)844 JITBuilder::JITBuilder(const rr::Config &config)
845 : config(config)
846 , context(new llvm::LLVMContext())
847 , module(new llvm::Module("", *context))
848 , builder(new llvm::IRBuilder<>(*context))
849 {
850 module->setTargetTriple(LLVM_DEFAULT_TARGET_TRIPLE);
851 module->setDataLayout(JITGlobals::get()->getDataLayout());
852
853 if(REACTOR_ENABLE_MEMORY_SANITIZER_INSTRUMENTATION ||
854 getPragmaState(MemorySanitizerInstrumentation))
855 {
856 msanInstrumentation = true;
857 }
858 }
859
runPasses(const rr::Config & cfg)860 void JITBuilder::runPasses(const rr::Config &cfg)
861 {
862 if(coroutine.id) // Run manadory coroutine transforms.
863 {
864 #if LLVM_VERSION_MAJOR >= 13 // New pass manager
865 llvm::PassBuilder pb;
866 llvm::LoopAnalysisManager lam;
867 llvm::FunctionAnalysisManager fam;
868 llvm::CGSCCAnalysisManager cgam;
869 llvm::ModuleAnalysisManager mam;
870
871 pb.registerModuleAnalyses(mam);
872 pb.registerCGSCCAnalyses(cgam);
873 pb.registerFunctionAnalyses(fam);
874 pb.registerLoopAnalyses(lam);
875 pb.crossRegisterProxies(lam, fam, cgam, mam);
876
877 llvm::ModulePassManager mpm =
878 pb.buildO0DefaultPipeline(llvm::OptimizationLevel::O0);
879 mpm.run(*module, mam);
880 #else // Legacy pass manager
881 llvm::legacy::PassManager pm;
882
883 pm.add(llvm::createCoroEarlyLegacyPass());
884 pm.add(llvm::createCoroSplitLegacyPass());
885 pm.add(llvm::createCoroElideLegacyPass());
886 pm.add(llvm::createBarrierNoopPass());
887 pm.add(llvm::createCoroCleanupLegacyPass());
888
889 pm.run(*module);
890 #endif
891 }
892
893 #if defined(ENABLE_RR_LLVM_IR_VERIFICATION) || !defined(NDEBUG)
894 if(llvm::verifyModule(*module, &llvm::errs()))
895 {
896 llvm::report_fatal_error("Invalid LLVM module");
897 }
898 #endif
899
900 #ifdef ENABLE_RR_DEBUG_INFO
901 if(debugInfo != nullptr)
902 {
903 return; // Don't optimize if we're generating debug info.
904 }
905 #endif // ENABLE_RR_DEBUG_INFO
906
907 #if LLVM_VERSION_MAJOR >= 13 // New pass manager
908 llvm::LoopAnalysisManager lam;
909 llvm::FunctionAnalysisManager fam;
910 llvm::CGSCCAnalysisManager cgam;
911 llvm::ModuleAnalysisManager mam;
912 llvm::PassBuilder pb;
913
914 pb.registerModuleAnalyses(mam);
915 pb.registerCGSCCAnalyses(cgam);
916 pb.registerFunctionAnalyses(fam);
917 pb.registerLoopAnalyses(lam);
918 pb.crossRegisterProxies(lam, fam, cgam, mam);
919
920 llvm::ModulePassManager pm;
921 llvm::FunctionPassManager fpm;
922
923 if(__has_feature(memory_sanitizer) && msanInstrumentation)
924 {
925 llvm::MemorySanitizerOptions msanOpts;
926 pm.addPass(llvm::ModuleMemorySanitizerPass(msanOpts));
927 pm.addPass(llvm::createModuleToFunctionPassAdaptor(llvm::MemorySanitizerPass(msanOpts)));
928 }
929
930 for(auto pass : cfg.getOptimization().getPasses())
931 {
932 switch(pass)
933 {
934 case rr::Optimization::Pass::Disabled: break;
935 case rr::Optimization::Pass::CFGSimplification: fpm.addPass(llvm::SimplifyCFGPass()); break;
936 case rr::Optimization::Pass::LICM:
937 fpm.addPass(llvm::createFunctionToLoopPassAdaptor(
938 llvm::LICMPass(llvm::SetLicmMssaOptCap, llvm::SetLicmMssaNoAccForPromotionCap, true)));
939 break;
940 case rr::Optimization::Pass::AggressiveDCE: fpm.addPass(llvm::ADCEPass()); break;
941 case rr::Optimization::Pass::GVN: fpm.addPass(llvm::GVNPass()); break;
942 case rr::Optimization::Pass::InstructionCombining: fpm.addPass(llvm::InstCombinePass()); break;
943 case rr::Optimization::Pass::Reassociate: fpm.addPass(llvm::ReassociatePass()); break;
944 case rr::Optimization::Pass::DeadStoreElimination: fpm.addPass(llvm::DSEPass()); break;
945 case rr::Optimization::Pass::SCCP: fpm.addPass(llvm::SCCPPass()); break;
946 case rr::Optimization::Pass::ScalarReplAggregates: fpm.addPass(llvm::SROAPass()); break;
947 case rr::Optimization::Pass::EarlyCSEPass: fpm.addPass(llvm::EarlyCSEPass()); break;
948 default:
949 UNREACHABLE("pass: %d", int(pass));
950 }
951 }
952
953 if(!fpm.isEmpty())
954 {
955 pm.addPass(llvm::createModuleToFunctionPassAdaptor(std::move(fpm)));
956 }
957
958 pm.run(*module, mam);
959 #else // Legacy pass manager
960 llvm::legacy::PassManager passManager;
961
962 if(__has_feature(memory_sanitizer) && msanInstrumentation)
963 {
964 passManager.add(llvm::createMemorySanitizerLegacyPassPass());
965 }
966
967 for(auto pass : cfg.getOptimization().getPasses())
968 {
969 switch(pass)
970 {
971 case rr::Optimization::Pass::Disabled: break;
972 case rr::Optimization::Pass::CFGSimplification: passManager.add(llvm::createCFGSimplificationPass()); break;
973 case rr::Optimization::Pass::LICM: passManager.add(llvm::createLICMPass()); break;
974 case rr::Optimization::Pass::AggressiveDCE: passManager.add(llvm::createAggressiveDCEPass()); break;
975 case rr::Optimization::Pass::GVN: passManager.add(llvm::createGVNPass()); break;
976 case rr::Optimization::Pass::InstructionCombining: passManager.add(llvm::createInstructionCombiningPass()); break;
977 case rr::Optimization::Pass::Reassociate: passManager.add(llvm::createReassociatePass()); break;
978 case rr::Optimization::Pass::DeadStoreElimination: passManager.add(llvm::createDeadStoreEliminationPass()); break;
979 case rr::Optimization::Pass::SCCP: passManager.add(llvm::createSCCPPass()); break;
980 case rr::Optimization::Pass::ScalarReplAggregates: passManager.add(llvm::createSROAPass()); break;
981 case rr::Optimization::Pass::EarlyCSEPass: passManager.add(llvm::createEarlyCSEPass()); break;
982 default:
983 UNREACHABLE("pass: %d", int(pass));
984 }
985 }
986
987 passManager.run(*module);
988 #endif
989 }
990
acquireRoutine(const char * name,llvm::Function ** funcs,size_t count,const rr::Config & cfg)991 std::shared_ptr<rr::Routine> JITBuilder::acquireRoutine(const char *name, llvm::Function **funcs, size_t count, const rr::Config &cfg)
992 {
993 ASSERT(module);
994 return std::make_shared<JITRoutine>(std::move(module), std::move(context), name, funcs, count, cfg);
995 }
996
997 } // namespace rr
998