1 // Copyright 2020 The SwiftShader Authors. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "LLVMReactor.hpp"
16
17 #include "Debug.hpp"
18 #include "ExecutableMemory.hpp"
19 #include "LLVMAsm.hpp"
20 #include "PragmaInternals.hpp"
21 #include "Routine.hpp"
22
23 // TODO(b/143539525): Eliminate when warning has been fixed.
24 #ifdef _MSC_VER
25 __pragma(warning(push))
26 __pragma(warning(disable : 4146)) // unary minus operator applied to unsigned type, result still unsigned
27 #endif
28
29 #include "llvm/ExecutionEngine/Orc/CompileUtils.h"
30 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
31 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
32 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
33 #include "llvm/IR/DiagnosticInfo.h"
34 #include "llvm/IR/Verifier.h"
35 #include "llvm/Support/CommandLine.h"
36 #include "llvm/Support/Host.h"
37 #include "llvm/Support/TargetSelect.h"
38 #include "llvm/Transforms/InstCombine/InstCombine.h"
39 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
40 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
41 #include "llvm/Transforms/Scalar.h"
42 #include "llvm/Transforms/Scalar/GVN.h"
43
44 #if LLVM_VERSION_MAJOR >= 13 // New pass manager
45 # include "llvm/IR/PassManager.h"
46 # include "llvm/Passes/PassBuilder.h"
47 # include "llvm/Transforms/Scalar/ADCE.h"
48 # include "llvm/Transforms/Scalar/DeadStoreElimination.h"
49 # include "llvm/Transforms/Scalar/EarlyCSE.h"
50 # include "llvm/Transforms/Scalar/LICM.h"
51 # include "llvm/Transforms/Scalar/Reassociate.h"
52 # include "llvm/Transforms/Scalar/SCCP.h"
53 # include "llvm/Transforms/Scalar/SROA.h"
54 # include "llvm/Transforms/Scalar/SimplifyCFG.h"
55 #else // Legacy pass manager
56 # include "llvm/IR/LegacyPassManager.h"
57 # include "llvm/Pass.h"
58 # include "llvm/Transforms/Coroutines.h"
59 # include "llvm/Transforms/IPO.h"
60 #endif
61
62 #ifdef _MSC_VER
63 __pragma(warning(pop))
64 #endif
65
66 #if defined(__unix__) || defined(__APPLE__) || defined(__Fuchsia__)
67 # define ADDRESS_SANITIZER_INSTRUMENTATION_SUPPORTED true
68 # if __has_feature(memory_sanitizer) || __has_feature(address_sanitizer)
69 # include <dlfcn.h> // dlsym()
70 # endif
71 #else
72 # define ADDRESS_SANITIZER_INSTRUMENTATION_SUPPORTED false
73 #endif
74
75 #ifndef REACTOR_ASM_EMIT_DIR
76 # define REACTOR_ASM_EMIT_DIR "./"
77 #endif
78
79 #if defined(_WIN64)
80 extern "C" void __chkstk();
81 #elif defined(_WIN32)
82 extern "C" void _chkstk();
83 #endif
84
85 #ifdef __ARM_EABI__
86 extern "C" signed __aeabi_idivmod();
87 #endif
88
89 #if __has_feature(memory_sanitizer)
90 # include "sanitizer/msan_interface.h"
91
92 // MemorySanitizer uses thread-local storage (TLS) data arrays for passing around
93 // the 'shadow' values of function arguments and return values. The LLVM JIT can't
94 // access TLS directly, but it calls __emutls_get_address() to obtain the address.
95 // Typically, it would be passed a pointer to an __emutls_control structure with a
96 // name starting with "__emutls_v." that represents the TLS. Both the address of
97 // __emutls_get_address and the __emutls_v. structures are provided to the JIT by
98 // the symbol resolver, which can be overridden.
99 // We take advantage of this by substituting __emutls_get_address() with our own
100 // implementation, namely rr::getTLSAddress(), and substituting the __emutls_v
101 // variables with rr::MSanTLS enums. getTLSAddress() can then provide the address
102 // of the real TLS variable corresponding to the enum, in statically compiled C++.
103
104 // Forward declare the real TLS variables used by MemorySanitizer. These are
105 // defined in llvm-project/compiler-rt/lib/msan/msan.cpp.
106 extern __thread unsigned long long __msan_param_tls[];
107 extern __thread unsigned int __msan_param_origin_tls[];
108 extern __thread unsigned long long __msan_retval_tls[];
109 extern __thread unsigned int __msan_retval_origin_tls;
110 extern __thread unsigned long long __msan_va_arg_tls[];
111 extern __thread unsigned int __msan_va_arg_origin_tls[];
112 extern __thread unsigned long long __msan_va_arg_overflow_size_tls;
113 extern __thread unsigned int __msan_origin_tls;
114
115 namespace rr {
116
117 enum class MSanTLS
118 {
119 param = 1, // __msan_param_tls
120 param_origin, //__msan_param_origin_tls
121 retval, // __msan_retval_tls
122 retval_origin, //__msan_retval_origin_tls
123 va_arg, // __msan_va_arg_tls
124 va_arg_origin, // __msan_va_arg_origin_tls
125 va_arg_overflow_size, // __msan_va_arg_overflow_size_tls
126 origin, //__msan_origin_tls
127 };
128
getTLSAddress(void * control)129 static void *getTLSAddress(void *control)
130 {
131 auto tlsIndex = static_cast<MSanTLS>(reinterpret_cast<uintptr_t>(control));
132 switch(tlsIndex)
133 {
134 case MSanTLS::param: return reinterpret_cast<void *>(&__msan_param_tls);
135 case MSanTLS::param_origin: return reinterpret_cast<void *>(&__msan_param_origin_tls);
136 case MSanTLS::retval: return reinterpret_cast<void *>(&__msan_retval_tls);
137 case MSanTLS::retval_origin: return reinterpret_cast<void *>(&__msan_retval_origin_tls);
138 case MSanTLS::va_arg: return reinterpret_cast<void *>(&__msan_va_arg_tls);
139 case MSanTLS::va_arg_origin: return reinterpret_cast<void *>(&__msan_va_arg_origin_tls);
140 case MSanTLS::va_arg_overflow_size: return reinterpret_cast<void *>(&__msan_va_arg_overflow_size_tls);
141 case MSanTLS::origin: return reinterpret_cast<void *>(&__msan_origin_tls);
142
143 default:
144 UNSUPPORTED("MemorySanitizer used an unrecognized TLS variable: %d", tlsIndex);
145 return nullptr;
146 }
147 }
148
149 } // namespace rr
150 #endif
151
152 namespace {
153
154 // TODO(b/174587935): Eliminate command-line parsing.
parseCommandLineOptionsOnce(int argc,const char * const * argv)155 bool parseCommandLineOptionsOnce(int argc, const char *const *argv)
156 {
157 // Use a static immediately invoked lambda to make this thread safe
158 static auto initialized = [=]() {
159 return llvm::cl::ParseCommandLineOptions(argc, argv);
160 }();
161
162 return initialized;
163 }
164
165 // JITGlobals is a singleton that holds all the immutable machine specific
166 // information for the host device.
167 class JITGlobals
168 {
169 public:
170 static JITGlobals *get();
171
172 llvm::orc::JITTargetMachineBuilder getTargetMachineBuilder() const;
173 const llvm::DataLayout &getDataLayout() const;
174 const llvm::Triple &getTargetTriple() const;
175
176 private:
177 JITGlobals(llvm::orc::JITTargetMachineBuilder &&jitTargetMachineBuilder, llvm::DataLayout &&dataLayout);
178
179 static llvm::CodeGenOpt::Level toLLVM(int level);
180
181 const llvm::orc::JITTargetMachineBuilder jitTargetMachineBuilder;
182 const llvm::DataLayout dataLayout;
183 };
184
get()185 JITGlobals *JITGlobals::get()
186 {
187 static JITGlobals instance = [] {
188 const char *argv[] = {
189 "Reactor",
190 #if defined(__i386__) || defined(__x86_64__)
191 "-x86-asm-syntax=intel", // Use Intel syntax rather than the default AT&T
192 #endif
193 #if LLVM_VERSION_MAJOR <= 12
194 "-warn-stack-size=524288", // Warn when a function uses more than 512 KiB of stack memory
195 #endif
196 };
197
198 parseCommandLineOptionsOnce(sizeof(argv) / sizeof(argv[0]), argv);
199
200 llvm::InitializeNativeTarget();
201 llvm::InitializeNativeTargetAsmPrinter();
202 llvm::InitializeNativeTargetAsmParser();
203
204 // TODO(b/171236524): JITTargetMachineBuilder::detectHost() currently uses the target triple of the host,
205 // rather than a valid triple for the current process. Once fixed, we can use that function instead.
206 llvm::orc::JITTargetMachineBuilder jitTargetMachineBuilder(llvm::Triple(LLVM_DEFAULT_TARGET_TRIPLE));
207
208 // Retrieve host CPU name and sub-target features and add them to builder.
209 // Relocation model, code model and codegen opt level are kept to default values.
210 llvm::StringMap<bool> cpuFeatures;
211 bool ok = llvm::sys::getHostCPUFeatures(cpuFeatures);
212
213 #if defined(__i386__) || defined(__x86_64__) || \
214 (defined(__linux__) && (defined(__arm__) || defined(__aarch64__)))
215 ASSERT_MSG(ok, "llvm::sys::getHostCPUFeatures returned false");
216 #else
217 (void)ok; // getHostCPUFeatures always returns false on other platforms
218 #endif
219
220 for(auto &feature : cpuFeatures)
221 {
222 jitTargetMachineBuilder.getFeatures().AddFeature(feature.first(), feature.second);
223 }
224
225 #if LLVM_VERSION_MAJOR >= 11 /* TODO(b/165000222): Unconditional after LLVM 11 upgrade */
226 jitTargetMachineBuilder.setCPU(std::string(llvm::sys::getHostCPUName()));
227 #else
228 jitTargetMachineBuilder.setCPU(llvm::sys::getHostCPUName());
229 #endif
230
231 // Reactor's MemorySanitizer support depends on intercepting __emutls_get_address calls.
232 ASSERT(!__has_feature(memory_sanitizer) || (jitTargetMachineBuilder.getOptions().ExplicitEmulatedTLS &&
233 jitTargetMachineBuilder.getOptions().EmulatedTLS));
234
235 auto dataLayout = jitTargetMachineBuilder.getDefaultDataLayoutForTarget();
236 ASSERT_MSG(dataLayout, "JITTargetMachineBuilder::getDefaultDataLayoutForTarget() failed");
237
238 return JITGlobals(std::move(jitTargetMachineBuilder), std::move(dataLayout.get()));
239 }();
240
241 return &instance;
242 }
243
getTargetMachineBuilder() const244 llvm::orc::JITTargetMachineBuilder JITGlobals::getTargetMachineBuilder() const
245 {
246 llvm::orc::JITTargetMachineBuilder out = jitTargetMachineBuilder;
247 out.setCodeGenOptLevel(toLLVM(rr::getPragmaState(rr::OptimizationLevel)));
248
249 return out;
250 }
251
getDataLayout() const252 const llvm::DataLayout &JITGlobals::getDataLayout() const
253 {
254 return dataLayout;
255 }
256
getTargetTriple() const257 const llvm::Triple &JITGlobals::getTargetTriple() const
258 {
259 return jitTargetMachineBuilder.getTargetTriple();
260 }
261
JITGlobals(llvm::orc::JITTargetMachineBuilder && jitTargetMachineBuilder,llvm::DataLayout && dataLayout)262 JITGlobals::JITGlobals(llvm::orc::JITTargetMachineBuilder &&jitTargetMachineBuilder, llvm::DataLayout &&dataLayout)
263 : jitTargetMachineBuilder(jitTargetMachineBuilder)
264 , dataLayout(dataLayout)
265 {
266 }
267
toLLVM(int level)268 llvm::CodeGenOpt::Level JITGlobals::toLLVM(int level)
269 {
270 // TODO(b/173257647): MemorySanitizer instrumentation produces IR which takes
271 // a lot longer to process by the machine code optimization passes. Disabling
272 // them has a negligible effect on code quality but compiles much faster.
273 if(__has_feature(memory_sanitizer))
274 {
275 return llvm::CodeGenOpt::None;
276 }
277
278 switch(level)
279 {
280 case 0: return llvm::CodeGenOpt::None;
281 case 1: return llvm::CodeGenOpt::Less;
282 case 2: return llvm::CodeGenOpt::Default;
283 case 3: return llvm::CodeGenOpt::Aggressive;
284 default: UNREACHABLE("Unknown Optimization Level %d", int(level));
285 }
286
287 return llvm::CodeGenOpt::Default;
288 }
289
290 class MemoryMapper final : public llvm::SectionMemoryManager::MemoryMapper
291 {
292 public:
MemoryMapper()293 MemoryMapper() {}
~MemoryMapper()294 ~MemoryMapper() final {}
295
allocateMappedMemory(llvm::SectionMemoryManager::AllocationPurpose purpose,size_t numBytes,const llvm::sys::MemoryBlock * const nearBlock,unsigned flags,std::error_code & errorCode)296 llvm::sys::MemoryBlock allocateMappedMemory(
297 llvm::SectionMemoryManager::AllocationPurpose purpose,
298 size_t numBytes, const llvm::sys::MemoryBlock *const nearBlock,
299 unsigned flags, std::error_code &errorCode) final
300 {
301 errorCode = std::error_code();
302
303 // Round up numBytes to page size.
304 size_t pageSize = rr::memoryPageSize();
305 numBytes = (numBytes + pageSize - 1) & ~(pageSize - 1);
306
307 bool need_exec =
308 purpose == llvm::SectionMemoryManager::AllocationPurpose::Code;
309 void *addr = rr::allocateMemoryPages(
310 numBytes, flagsToPermissions(flags), need_exec);
311 if(!addr)
312 return llvm::sys::MemoryBlock();
313 return llvm::sys::MemoryBlock(addr, numBytes);
314 }
315
protectMappedMemory(const llvm::sys::MemoryBlock & block,unsigned flags)316 std::error_code protectMappedMemory(const llvm::sys::MemoryBlock &block,
317 unsigned flags)
318 {
319 // Round down base address to align with a page boundary. This matches
320 // DefaultMMapper behavior.
321 void *addr = block.base();
322 size_t size = block.allocatedSize();
323 size_t pageSize = rr::memoryPageSize();
324 addr = reinterpret_cast<void *>(
325 reinterpret_cast<uintptr_t>(addr) & ~(pageSize - 1));
326 size += reinterpret_cast<uintptr_t>(block.base()) -
327 reinterpret_cast<uintptr_t>(addr);
328
329 rr::protectMemoryPages(addr, size, flagsToPermissions(flags));
330 return std::error_code();
331 }
332
releaseMappedMemory(llvm::sys::MemoryBlock & block)333 std::error_code releaseMappedMemory(llvm::sys::MemoryBlock &block)
334 {
335 size_t size = block.allocatedSize();
336
337 rr::deallocateMemoryPages(block.base(), size);
338 return std::error_code();
339 }
340
341 private:
flagsToPermissions(unsigned flags)342 int flagsToPermissions(unsigned flags)
343 {
344 int result = 0;
345 if(flags & llvm::sys::Memory::MF_READ)
346 {
347 result |= rr::PERMISSION_READ;
348 }
349 if(flags & llvm::sys::Memory::MF_WRITE)
350 {
351 result |= rr::PERMISSION_WRITE;
352 }
353 if(flags & llvm::sys::Memory::MF_EXEC)
354 {
355 result |= rr::PERMISSION_EXECUTE;
356 }
357 return result;
358 }
359 };
360
361 template<typename T>
alignUp(T val,T alignment)362 T alignUp(T val, T alignment)
363 {
364 return alignment * ((val + alignment - 1) / alignment);
365 }
366
alignedAlloc(size_t size,size_t alignment)367 void *alignedAlloc(size_t size, size_t alignment)
368 {
369 ASSERT(alignment < 256);
370 auto allocation = new uint8_t[size + sizeof(uint8_t) + alignment];
371 auto aligned = allocation;
372 aligned += sizeof(uint8_t); // Make space for the base-address offset.
373 aligned = reinterpret_cast<uint8_t *>(alignUp(reinterpret_cast<uintptr_t>(aligned), alignment)); // align
374 auto offset = static_cast<uint8_t>(aligned - allocation);
375 aligned[-1] = offset;
376 return aligned;
377 }
378
alignedFree(void * ptr)379 void alignedFree(void *ptr)
380 {
381 auto aligned = reinterpret_cast<uint8_t *>(ptr);
382 auto offset = aligned[-1];
383 auto allocation = aligned - offset;
384 delete[] allocation;
385 }
386
387 template<typename T>
atomicLoad(void * ptr,void * ret,llvm::AtomicOrdering ordering)388 static void atomicLoad(void *ptr, void *ret, llvm::AtomicOrdering ordering)
389 {
390 *reinterpret_cast<T *>(ret) = std::atomic_load_explicit<T>(reinterpret_cast<std::atomic<T> *>(ptr), rr::atomicOrdering(ordering));
391 }
392
393 template<typename T>
atomicStore(void * ptr,void * val,llvm::AtomicOrdering ordering)394 static void atomicStore(void *ptr, void *val, llvm::AtomicOrdering ordering)
395 {
396 std::atomic_store_explicit<T>(reinterpret_cast<std::atomic<T> *>(ptr), *reinterpret_cast<T *>(val), rr::atomicOrdering(ordering));
397 }
398
399 #ifdef __ANDROID__
400 template<typename F>
sync_fetch_and_op(uint32_t volatile * ptr,uint32_t val,F f)401 static uint32_t sync_fetch_and_op(uint32_t volatile *ptr, uint32_t val, F f)
402 {
403 // Build an arbitrary op out of looped CAS
404 for(;;)
405 {
406 uint32_t expected = *ptr;
407 uint32_t desired = f(expected, val);
408
409 if(expected == __sync_val_compare_and_swap_4(ptr, expected, desired))
410 {
411 return expected;
412 }
413 }
414 }
415 #endif
416
417 #if LLVM_VERSION_MAJOR >= 11 /* TODO(b/165000222): Unconditional after LLVM 11 upgrade */
418 class ExternalSymbolGenerator : public llvm::orc::DefinitionGenerator
419 #else
420 class ExternalSymbolGenerator : public llvm::orc::JITDylib::DefinitionGenerator
421 #endif
422 {
423 struct Atomic
424 {
load__anon2de977520111::ExternalSymbolGenerator::Atomic425 static void load(size_t size, void *ptr, void *ret, llvm::AtomicOrdering ordering)
426 {
427 switch(size)
428 {
429 case 1: atomicLoad<uint8_t>(ptr, ret, ordering); break;
430 case 2: atomicLoad<uint16_t>(ptr, ret, ordering); break;
431 case 4: atomicLoad<uint32_t>(ptr, ret, ordering); break;
432 case 8: atomicLoad<uint64_t>(ptr, ret, ordering); break;
433 default:
434 UNIMPLEMENTED_NO_BUG("Atomic::load(size: %d)", int(size));
435 }
436 }
store__anon2de977520111::ExternalSymbolGenerator::Atomic437 static void store(size_t size, void *ptr, void *ret, llvm::AtomicOrdering ordering)
438 {
439 switch(size)
440 {
441 case 1: atomicStore<uint8_t>(ptr, ret, ordering); break;
442 case 2: atomicStore<uint16_t>(ptr, ret, ordering); break;
443 case 4: atomicStore<uint32_t>(ptr, ret, ordering); break;
444 case 8: atomicStore<uint64_t>(ptr, ret, ordering); break;
445 default:
446 UNIMPLEMENTED_NO_BUG("Atomic::store(size: %d)", int(size));
447 }
448 }
449 };
450
nop()451 static void nop() {}
neverCalled()452 static void neverCalled() { UNREACHABLE("Should never be called"); }
453
coroutine_alloc_frame(size_t size)454 static void *coroutine_alloc_frame(size_t size) { return alignedAlloc(size, 16); }
coroutine_free_frame(void * ptr)455 static void coroutine_free_frame(void *ptr) { alignedFree(ptr); }
456
457 #ifdef __ANDROID__
458 // forwarders since we can't take address of builtins
sync_synchronize()459 static void sync_synchronize() { __sync_synchronize(); }
sync_fetch_and_add_4(uint32_t * ptr,uint32_t val)460 static uint32_t sync_fetch_and_add_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_add_4(ptr, val); }
sync_fetch_and_and_4(uint32_t * ptr,uint32_t val)461 static uint32_t sync_fetch_and_and_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_and_4(ptr, val); }
sync_fetch_and_or_4(uint32_t * ptr,uint32_t val)462 static uint32_t sync_fetch_and_or_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_or_4(ptr, val); }
sync_fetch_and_xor_4(uint32_t * ptr,uint32_t val)463 static uint32_t sync_fetch_and_xor_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_xor_4(ptr, val); }
sync_fetch_and_sub_4(uint32_t * ptr,uint32_t val)464 static uint32_t sync_fetch_and_sub_4(uint32_t *ptr, uint32_t val) { return __sync_fetch_and_sub_4(ptr, val); }
sync_lock_test_and_set_4(uint32_t * ptr,uint32_t val)465 static uint32_t sync_lock_test_and_set_4(uint32_t *ptr, uint32_t val) { return __sync_lock_test_and_set_4(ptr, val); }
sync_val_compare_and_swap_4(uint32_t * ptr,uint32_t expected,uint32_t desired)466 static uint32_t sync_val_compare_and_swap_4(uint32_t *ptr, uint32_t expected, uint32_t desired) { return __sync_val_compare_and_swap_4(ptr, expected, desired); }
467
sync_fetch_and_max_4(uint32_t * ptr,uint32_t val)468 static uint32_t sync_fetch_and_max_4(uint32_t *ptr, uint32_t val)
469 {
470 return sync_fetch_and_op(ptr, val, [](int32_t a, int32_t b) { return std::max(a, b); });
471 }
sync_fetch_and_min_4(uint32_t * ptr,uint32_t val)472 static uint32_t sync_fetch_and_min_4(uint32_t *ptr, uint32_t val)
473 {
474 return sync_fetch_and_op(ptr, val, [](int32_t a, int32_t b) { return std::min(a, b); });
475 }
sync_fetch_and_umax_4(uint32_t * ptr,uint32_t val)476 static uint32_t sync_fetch_and_umax_4(uint32_t *ptr, uint32_t val)
477 {
478 return sync_fetch_and_op(ptr, val, [](uint32_t a, uint32_t b) { return std::max(a, b); });
479 }
sync_fetch_and_umin_4(uint32_t * ptr,uint32_t val)480 static uint32_t sync_fetch_and_umin_4(uint32_t *ptr, uint32_t val)
481 {
482 return sync_fetch_and_op(ptr, val, [](uint32_t a, uint32_t b) { return std::min(a, b); });
483 }
484 #endif
485
486 class Resolver
487 {
488 public:
489 using FunctionMap = llvm::StringMap<void *>;
490
491 FunctionMap functions;
492
Resolver()493 Resolver()
494 {
495 #ifdef ENABLE_RR_PRINT
496 functions.try_emplace("rr::DebugPrintf", reinterpret_cast<void *>(rr::DebugPrintf));
497 #endif
498 functions.try_emplace("nop", reinterpret_cast<void *>(nop));
499 functions.try_emplace("floorf", reinterpret_cast<void *>(floorf));
500 functions.try_emplace("nearbyintf", reinterpret_cast<void *>(nearbyintf));
501 functions.try_emplace("truncf", reinterpret_cast<void *>(truncf));
502 functions.try_emplace("printf", reinterpret_cast<void *>(printf));
503 functions.try_emplace("puts", reinterpret_cast<void *>(puts));
504 functions.try_emplace("fmodf", reinterpret_cast<void *>(fmodf));
505
506 functions.try_emplace("sinf", reinterpret_cast<void *>(sinf));
507 functions.try_emplace("cosf", reinterpret_cast<void *>(cosf));
508 functions.try_emplace("asinf", reinterpret_cast<void *>(asinf));
509 functions.try_emplace("acosf", reinterpret_cast<void *>(acosf));
510 functions.try_emplace("atanf", reinterpret_cast<void *>(atanf));
511 functions.try_emplace("sinhf", reinterpret_cast<void *>(sinhf));
512 functions.try_emplace("coshf", reinterpret_cast<void *>(coshf));
513 functions.try_emplace("tanhf", reinterpret_cast<void *>(tanhf));
514 functions.try_emplace("asinhf", reinterpret_cast<void *>(asinhf));
515 functions.try_emplace("acoshf", reinterpret_cast<void *>(acoshf));
516 functions.try_emplace("atanhf", reinterpret_cast<void *>(atanhf));
517 functions.try_emplace("atan2f", reinterpret_cast<void *>(atan2f));
518 functions.try_emplace("powf", reinterpret_cast<void *>(powf));
519 functions.try_emplace("expf", reinterpret_cast<void *>(expf));
520 functions.try_emplace("logf", reinterpret_cast<void *>(logf));
521 functions.try_emplace("exp2f", reinterpret_cast<void *>(exp2f));
522 functions.try_emplace("log2f", reinterpret_cast<void *>(log2f));
523 functions.try_emplace("fmaf", reinterpret_cast<void *>(fmaf));
524
525 functions.try_emplace("fmod", reinterpret_cast<void *>(static_cast<double (*)(double, double)>(fmod)));
526 functions.try_emplace("sin", reinterpret_cast<void *>(static_cast<double (*)(double)>(sin)));
527 functions.try_emplace("cos", reinterpret_cast<void *>(static_cast<double (*)(double)>(cos)));
528 functions.try_emplace("asin", reinterpret_cast<void *>(static_cast<double (*)(double)>(asin)));
529 functions.try_emplace("acos", reinterpret_cast<void *>(static_cast<double (*)(double)>(acos)));
530 functions.try_emplace("atan", reinterpret_cast<void *>(static_cast<double (*)(double)>(atan)));
531 functions.try_emplace("sinh", reinterpret_cast<void *>(static_cast<double (*)(double)>(sinh)));
532 functions.try_emplace("cosh", reinterpret_cast<void *>(static_cast<double (*)(double)>(cosh)));
533 functions.try_emplace("tanh", reinterpret_cast<void *>(static_cast<double (*)(double)>(tanh)));
534 functions.try_emplace("asinh", reinterpret_cast<void *>(static_cast<double (*)(double)>(asinh)));
535 functions.try_emplace("acosh", reinterpret_cast<void *>(static_cast<double (*)(double)>(acosh)));
536 functions.try_emplace("atanh", reinterpret_cast<void *>(static_cast<double (*)(double)>(atanh)));
537 functions.try_emplace("atan2", reinterpret_cast<void *>(static_cast<double (*)(double, double)>(atan2)));
538 functions.try_emplace("pow", reinterpret_cast<void *>(static_cast<double (*)(double, double)>(pow)));
539 functions.try_emplace("exp", reinterpret_cast<void *>(static_cast<double (*)(double)>(exp)));
540 functions.try_emplace("log", reinterpret_cast<void *>(static_cast<double (*)(double)>(log)));
541 functions.try_emplace("exp2", reinterpret_cast<void *>(static_cast<double (*)(double)>(exp2)));
542 functions.try_emplace("log2", reinterpret_cast<void *>(static_cast<double (*)(double)>(log2)));
543
544 functions.try_emplace("atomic_load", reinterpret_cast<void *>(Atomic::load));
545 functions.try_emplace("atomic_store", reinterpret_cast<void *>(Atomic::store));
546
547 // FIXME(b/119409619): use an allocator here so we can control all memory allocations
548 functions.try_emplace("coroutine_alloc_frame", reinterpret_cast<void *>(coroutine_alloc_frame));
549 functions.try_emplace("coroutine_free_frame", reinterpret_cast<void *>(coroutine_free_frame));
550
551 functions.try_emplace("memset", reinterpret_cast<void *>(memset));
552
553 #ifdef __APPLE__
554 functions.try_emplace("__sincosf_stret", reinterpret_cast<void *>(__sincosf_stret));
555 #elif defined(__linux__)
556 functions.try_emplace("sincosf", reinterpret_cast<void *>(sincosf));
557 #elif defined(_WIN64)
558 functions.try_emplace("__chkstk", reinterpret_cast<void *>(__chkstk));
559 #elif defined(_WIN32)
560 functions.try_emplace("_chkstk", reinterpret_cast<void *>(_chkstk));
561 #endif
562
563 #ifdef __ARM_EABI__
564 functions.try_emplace("__aeabi_idivmod", reinterpret_cast<void *>(__aeabi_idivmod));
565 #endif
566 #ifdef __ANDROID__
567 functions.try_emplace("aeabi_unwind_cpp_pr0", reinterpret_cast<void *>(neverCalled));
568 functions.try_emplace("sync_synchronize", reinterpret_cast<void *>(sync_synchronize));
569 functions.try_emplace("sync_fetch_and_add_4", reinterpret_cast<void *>(sync_fetch_and_add_4));
570 functions.try_emplace("sync_fetch_and_and_4", reinterpret_cast<void *>(sync_fetch_and_and_4));
571 functions.try_emplace("sync_fetch_and_or_4", reinterpret_cast<void *>(sync_fetch_and_or_4));
572 functions.try_emplace("sync_fetch_and_xor_4", reinterpret_cast<void *>(sync_fetch_and_xor_4));
573 functions.try_emplace("sync_fetch_and_sub_4", reinterpret_cast<void *>(sync_fetch_and_sub_4));
574 functions.try_emplace("sync_lock_test_and_set_4", reinterpret_cast<void *>(sync_lock_test_and_set_4));
575 functions.try_emplace("sync_val_compare_and_swap_4", reinterpret_cast<void *>(sync_val_compare_and_swap_4));
576 functions.try_emplace("sync_fetch_and_max_4", reinterpret_cast<void *>(sync_fetch_and_max_4));
577 functions.try_emplace("sync_fetch_and_min_4", reinterpret_cast<void *>(sync_fetch_and_min_4));
578 functions.try_emplace("sync_fetch_and_umax_4", reinterpret_cast<void *>(sync_fetch_and_umax_4));
579 functions.try_emplace("sync_fetch_and_umin_4", reinterpret_cast<void *>(sync_fetch_and_umin_4));
580
581 # if defined(__i386__)
582 // TODO(b/172974501): Workaround for an x86-32 issue where an R_386_PC32 relocation is used
583 // When calling a C function from Reactor code, who's address is not associated with any symbol
584 // (since it's an absolute constant), but it still invokes the symbol resolver for "".
585 functions.try_emplace("", nullptr);
586 # endif
587 #endif
588 #if __has_feature(memory_sanitizer)
589 functions.try_emplace("__emutls_get_address", reinterpret_cast<void *>(rr::getTLSAddress));
590 functions.try_emplace("__emutls_v.__msan_param_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::param)));
591 functions.try_emplace("__emutls_v.__msan_param_origin_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::param_origin)));
592 functions.try_emplace("__emutls_v.__msan_retval_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::retval)));
593 functions.try_emplace("__emutls_v.__msan_retval_origin_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::retval_origin)));
594 functions.try_emplace("__emutls_v.__msan_va_arg_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::va_arg)));
595 functions.try_emplace("__emutls_v.__msan_va_arg_origin_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::va_arg_origin)));
596 functions.try_emplace("__emutls_v.__msan_va_arg_overflow_size_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::va_arg_overflow_size)));
597 functions.try_emplace("__emutls_v.__msan_origin_tls", reinterpret_cast<void *>(static_cast<uintptr_t>(rr::MSanTLS::origin)));
598
599 functions.try_emplace("__msan_unpoison", reinterpret_cast<void *>(__msan_unpoison));
600 functions.try_emplace("__msan_unpoison_param", reinterpret_cast<void *>(__msan_unpoison_param));
601 #endif
602 }
603 };
604
tryToGenerate(llvm::orc::LookupState & state,llvm::orc::LookupKind kind,llvm::orc::JITDylib & dylib,llvm::orc::JITDylibLookupFlags flags,const llvm::orc::SymbolLookupSet & set)605 llvm::Error tryToGenerate(
606 #if LLVM_VERSION_MAJOR >= 11 /* TODO(b/165000222): Unconditional after LLVM 11 upgrade */
607 llvm::orc::LookupState &state,
608 #endif
609 llvm::orc::LookupKind kind,
610 llvm::orc::JITDylib &dylib,
611 llvm::orc::JITDylibLookupFlags flags,
612 const llvm::orc::SymbolLookupSet &set) override
613 {
614 static Resolver resolver;
615
616 llvm::orc::SymbolMap symbols;
617
618 #if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
619 std::string missing;
620 #endif // !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
621
622 for(auto symbol : set)
623 {
624 auto name = symbol.first;
625
626 #if defined(__APPLE__)
627 // Trim the underscore from the start of the symbol. LLVM adds it for Mach-O mangling convention.
628 ASSERT((*name)[0] == '_');
629 auto unmangled = (*name).drop_front(1);
630 #else
631 auto unmangled = *name;
632 #endif
633
634 auto it = resolver.functions.find(unmangled.str());
635 if(it != resolver.functions.end())
636 {
637 symbols[name] = llvm::JITEvaluatedSymbol(
638 static_cast<llvm::JITTargetAddress>(reinterpret_cast<uintptr_t>(it->second)),
639 llvm::JITSymbolFlags::Exported);
640
641 continue;
642 }
643
644 #if __has_feature(memory_sanitizer) || (__has_feature(address_sanitizer) && ADDRESS_SANITIZER_INSTRUMENTATION_SUPPORTED)
645 // Sanitizers use a dynamically linked runtime. Instrumented routines reference some
646 // symbols from this library. Look them up dynamically in the default namespace.
647 // Note this approach should not be used for other symbols, since they might not be
648 // visible (e.g. due to static linking), we may wish to provide an alternate
649 // implementation, and/or it would be a security vulnerability.
650
651 void *address = dlsym(RTLD_DEFAULT, unmangled.data());
652
653 if(address)
654 {
655 symbols[name] = llvm::JITEvaluatedSymbol(
656 static_cast<llvm::JITTargetAddress>(reinterpret_cast<uintptr_t>(address)),
657 llvm::JITSymbolFlags::Exported);
658
659 continue;
660 }
661 #endif
662
663 #if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
664 missing += (missing.empty() ? "'" : ", '") + unmangled.str() + "'";
665 #endif
666 }
667
668 #if !defined(NDEBUG) || defined(DCHECK_ALWAYS_ON)
669 // Missing functions will likely make the module fail in non-obvious ways.
670 if(!missing.empty())
671 {
672 WARN("Missing external functions: %s", missing.c_str());
673 }
674 #endif
675
676 if(symbols.empty())
677 {
678 return llvm::Error::success();
679 }
680
681 return dylib.define(llvm::orc::absoluteSymbols(std::move(symbols)));
682 }
683 };
684
685 // As we must support different LLVM versions, add a generic Unwrap for functions that return Expected<T> or the actual T.
686 // TODO(b/165000222): Remove after LLVM 11 upgrade
687 template<typename T>
Unwrap(llvm::Expected<T> && v)688 T &Unwrap(llvm::Expected<T> &&v)
689 {
690 assert(v);
691 return v.get();
692 }
693 template<typename T>
Unwrap(T && v)694 T &Unwrap(T &&v)
695 {
696 return v;
697 }
698
699 // Sets *fatal to true if a diagnostic is received which makes a routine invalid or unusable.
700 struct FatalDiagnosticsHandler : public llvm::DiagnosticHandler
701 {
FatalDiagnosticsHandler__anon2de977520111::FatalDiagnosticsHandler702 FatalDiagnosticsHandler(bool *fatal)
703 : fatal(fatal)
704 {}
705
handleDiagnostics__anon2de977520111::FatalDiagnosticsHandler706 bool handleDiagnostics(const llvm::DiagnosticInfo &info) override
707 {
708 switch(info.getSeverity())
709 {
710 case llvm::DS_Error:
711 ASSERT_MSG(false, "LLVM JIT compilation failure");
712 *fatal = true;
713 break;
714 case llvm::DS_Warning:
715 if(info.getKind() == llvm::DK_StackSize)
716 {
717 // Stack size limit exceeded
718 *fatal = true;
719 }
720 break;
721 case llvm::DS_Remark:
722 break;
723 case llvm::DS_Note:
724 break;
725 }
726
727 return true; // Diagnostic handled, don't let LLVM print it.
728 }
729
730 bool *fatal;
731 };
732
733 // JITRoutine is a rr::Routine that holds a LLVM JIT session, compiler and
734 // object layer as each routine may require different target machine
735 // settings and no Reactor routine directly links against another.
736 class JITRoutine : public rr::Routine
737 {
738 public:
JITRoutine(std::unique_ptr<llvm::Module> module,std::unique_ptr<llvm::LLVMContext> context,const char * name,llvm::Function ** funcs,size_t count)739 JITRoutine(
740 std::unique_ptr<llvm::Module> module,
741 std::unique_ptr<llvm::LLVMContext> context,
742 const char *name,
743 llvm::Function **funcs,
744 size_t count)
745 : name(name)
746 #if LLVM_VERSION_MAJOR >= 13
747 , session(std::move(Unwrap(llvm::orc::SelfExecutorProcessControl::Create())))
748 #endif
749 , objectLayer(session, [this]() {
750 return std::make_unique<llvm::SectionMemoryManager>(&memoryMapper);
751 })
752 , addresses(count)
753 {
754 bool fatalCompileIssue = false;
755 context->setDiagnosticHandler(std::make_unique<FatalDiagnosticsHandler>(&fatalCompileIssue), true);
756
757 #ifdef ENABLE_RR_DEBUG_INFO
758 // TODO(b/165000222): Update this on next LLVM roll.
759 // https://github.com/llvm/llvm-project/commit/98f2bb4461072347dcca7d2b1b9571b3a6525801
760 // introduces RTDyldObjectLinkingLayer::registerJITEventListener().
761 // The current API does not appear to have any way to bind the
762 // rr::DebugInfo::NotifyFreeingObject event.
763 # if LLVM_VERSION_MAJOR >= 12
764 objectLayer.setNotifyLoaded([](llvm::orc::MaterializationResponsibility &R,
765 const llvm::object::ObjectFile &obj,
__anon2de977520902(llvm::orc::MaterializationResponsibility &R, const llvm::object::ObjectFile &obj, const llvm::RuntimeDyld::LoadedObjectInfo &l) 766 const llvm::RuntimeDyld::LoadedObjectInfo &l) {
767 static std::atomic<uint64_t> unique_key{ 0 };
768 rr::DebugInfo::NotifyObjectEmitted(unique_key++, obj, l);
769 });
770 # else
771 objectLayer.setNotifyLoaded([](llvm::orc::VModuleKey,
772 const llvm::object::ObjectFile &obj,
__anon2de977520a02(llvm::orc::VModuleKey, const llvm::object::ObjectFile &obj, const llvm::RuntimeDyld::LoadedObjectInfo &l) 773 const llvm::RuntimeDyld::LoadedObjectInfo &l) {
774 static std::atomic<uint64_t> unique_key{ 0 };
775 rr::DebugInfo::NotifyObjectEmitted(unique_key++, obj, l);
776 });
777 # endif
778 #endif // ENABLE_RR_DEBUG_INFO
779
780 if(JITGlobals::get()->getTargetTriple().isOSBinFormatCOFF())
781 {
782 // Hack to support symbol visibility in COFF.
783 // Matches hack in llvm::orc::LLJIT::createObjectLinkingLayer().
784 // See documentation on these functions for more detail.
785 objectLayer.setOverrideObjectFlagsWithResponsibilityFlags(true);
786 objectLayer.setAutoClaimResponsibilityForObjectSymbols(true);
787 }
788
789 llvm::SmallVector<llvm::orc::SymbolStringPtr, 8> functionNames(count);
790 llvm::orc::MangleAndInterner mangle(session, JITGlobals::get()->getDataLayout());
791
792 for(size_t i = 0; i < count; i++)
793 {
794 llvm::Function *func = funcs[i];
795
796 if(!func->hasName())
797 {
798 func->setName("f" + llvm::Twine(i).str());
799 }
800
801 functionNames[i] = mangle(func->getName());
802 }
803
804 #ifdef ENABLE_RR_EMIT_ASM_FILE
805 const auto asmFilename = rr::AsmFile::generateFilename(REACTOR_ASM_EMIT_DIR, name);
806 rr::AsmFile::emitAsmFile(asmFilename, JITGlobals::get()->getTargetMachineBuilder(), *module);
807 #endif
808
809 // Once the module is passed to the compileLayer, the llvm::Functions are freed.
810 // Make sure funcs are not referenced after this point.
811 funcs = nullptr;
812
813 llvm::orc::IRCompileLayer compileLayer(session, objectLayer, std::make_unique<llvm::orc::ConcurrentIRCompiler>(JITGlobals::get()->getTargetMachineBuilder()));
814 llvm::orc::JITDylib &dylib(Unwrap(session.createJITDylib("<routine>")));
815 dylib.addGenerator(std::make_unique<ExternalSymbolGenerator>());
816
817 llvm::cantFail(compileLayer.add(dylib, llvm::orc::ThreadSafeModule(std::move(module), std::move(context))));
818
819 // Resolve the function addresses.
820 for(size_t i = 0; i < count; i++)
821 {
822 fatalCompileIssue = false; // May be set to true by session.lookup()
823
824 // This is where the actual compilation happens.
825 auto symbol = session.lookup({ &dylib }, functionNames[i]);
826
827 ASSERT_MSG(symbol, "Failed to lookup address of routine function %d: %s",
828 (int)i, llvm::toString(symbol.takeError()).c_str());
829
830 if(fatalCompileIssue)
831 {
832 addresses[i] = nullptr;
833 }
834 else // Successful compilation
835 {
836 addresses[i] = reinterpret_cast<void *>(static_cast<intptr_t>(symbol->getAddress()));
837 }
838 }
839
840 #ifdef ENABLE_RR_EMIT_ASM_FILE
841 rr::AsmFile::fixupAsmFile(asmFilename, addresses);
842 #endif
843 }
844
~JITRoutine()845 ~JITRoutine()
846 {
847 #if LLVM_VERSION_MAJOR >= 11 /* TODO(b/165000222): Unconditional after LLVM 11 upgrade */
848 if(auto err = session.endSession())
849 {
850 session.reportError(std::move(err));
851 }
852 #endif
853 }
854
getEntry(int index) const855 const void *getEntry(int index) const override
856 {
857 return addresses[index];
858 }
859
860 private:
861 std::string name;
862 llvm::orc::ExecutionSession session;
863 MemoryMapper memoryMapper;
864 llvm::orc::RTDyldObjectLinkingLayer objectLayer;
865 std::vector<const void *> addresses;
866 };
867
868 } // anonymous namespace
869
870 namespace rr {
871
JITBuilder()872 JITBuilder::JITBuilder()
873 : context(new llvm::LLVMContext())
874 , module(new llvm::Module("", *context))
875 , builder(new llvm::IRBuilder<>(*context))
876 {
877 module->setTargetTriple(LLVM_DEFAULT_TARGET_TRIPLE);
878 module->setDataLayout(JITGlobals::get()->getDataLayout());
879
880 msanInstrumentation = getPragmaState(MemorySanitizerInstrumentation);
881 }
882
runPasses()883 void JITBuilder::runPasses()
884 {
885 #if defined(ENABLE_RR_LLVM_IR_VERIFICATION) || !defined(NDEBUG)
886 if(llvm::verifyModule(*module, &llvm::errs()))
887 {
888 llvm::report_fatal_error("Invalid LLVM module");
889 }
890 #endif
891
892 int optimizationLevel = getPragmaState(OptimizationLevel);
893
894 #ifdef ENABLE_RR_DEBUG_INFO
895 if(debugInfo != nullptr)
896 {
897 optimizationLevel = 0; // Don't optimize if we're generating debug info.
898 }
899 #endif // ENABLE_RR_DEBUG_INFO
900
901 #if LLVM_VERSION_MAJOR >= 13 // New pass manager
902 llvm::LoopAnalysisManager lam;
903 llvm::FunctionAnalysisManager fam;
904 llvm::CGSCCAnalysisManager cgam;
905 llvm::ModuleAnalysisManager mam;
906 llvm::PassBuilder pb;
907
908 pb.registerModuleAnalyses(mam);
909 pb.registerCGSCCAnalyses(cgam);
910 pb.registerFunctionAnalyses(fam);
911 pb.registerLoopAnalyses(lam);
912 pb.crossRegisterProxies(lam, fam, cgam, mam);
913
914 llvm::ModulePassManager pm;
915 llvm::FunctionPassManager fpm;
916
917 if(coroutine.id)
918 {
919 // Adds mandatory coroutine transforms.
920 pm = pb.buildO0DefaultPipeline(llvm::OptimizationLevel::O0);
921 }
922
923 if(optimizationLevel > 0)
924 {
925 fpm.addPass(llvm::SROAPass(llvm::SROAOptions::PreserveCFG));
926 fpm.addPass(llvm::InstCombinePass());
927 }
928
929 if(!fpm.isEmpty())
930 {
931 pm.addPass(llvm::createModuleToFunctionPassAdaptor(std::move(fpm)));
932 }
933
934 if(__has_feature(memory_sanitizer) && msanInstrumentation)
935 {
936 llvm::MemorySanitizerOptions msanOpts(0 /* TrackOrigins */, false /* Recover */, false /* Kernel */, true /* EagerChecks */);
937 pm.addPass(llvm::MemorySanitizerPass(msanOpts));
938 }
939
940 if(__has_feature(address_sanitizer) && ADDRESS_SANITIZER_INSTRUMENTATION_SUPPORTED)
941 {
942 pm.addPass(llvm::AddressSanitizerPass(llvm::AddressSanitizerOptions{}));
943 }
944
945 pm.run(*module, mam);
946 #else // Legacy pass manager
947 llvm::legacy::PassManager passManager;
948
949 if(coroutine.id)
950 {
951 // Run mandatory coroutine transforms.
952 passManager.add(llvm::createCoroEarlyLegacyPass());
953 passManager.add(llvm::createCoroSplitLegacyPass());
954 passManager.add(llvm::createCoroElideLegacyPass());
955 passManager.add(llvm::createBarrierNoopPass());
956 passManager.add(llvm::createCoroCleanupLegacyPass());
957 }
958
959 if(optimizationLevel > 0)
960 {
961 passManager.add(llvm::createSROAPass());
962 passManager.add(llvm::createInstructionCombiningPass());
963 }
964
965 if(__has_feature(memory_sanitizer) && msanInstrumentation)
966 {
967 llvm::MemorySanitizerOptions msanOpts(0 /* TrackOrigins */, false /* Recover */, false /* Kernel */);
968 passManager.add(llvm::createMemorySanitizerLegacyPassPass(msanOpts));
969 }
970
971 if(__has_feature(address_sanitizer) && ADDRESS_SANITIZER_INSTRUMENTATION_SUPPORTED)
972 {
973 passManager.add(llvm::createAddressSanitizerFunctionPass());
974 }
975
976 passManager.run(*module);
977 #endif
978 }
979
acquireRoutine(const char * name,llvm::Function ** funcs,size_t count)980 std::shared_ptr<rr::Routine> JITBuilder::acquireRoutine(const char *name, llvm::Function **funcs, size_t count)
981 {
982 ASSERT(module);
983 return std::make_shared<JITRoutine>(std::move(module), std::move(context), name, funcs, count);
984 }
985
986 } // namespace rr
987