1 /**************************************************************************
2 *
3 * Copyright 2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
18 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * The above copyright notice and this permission notice (including the
23 * next paragraph) shall be included in all copies or substantial portions
24 * of the Software.
25 *
26 **************************************************************************/
27
28
29 /**
30 * The purpose of this module is to expose LLVM functionality not available
31 * through the C++ bindings.
32 */
33
34
35 // Undef these vars just to silence warnings
36 #undef PACKAGE_BUGREPORT
37 #undef PACKAGE_NAME
38 #undef PACKAGE_STRING
39 #undef PACKAGE_TARNAME
40 #undef PACKAGE_VERSION
41
42
43 #include <stddef.h>
44
45 #include <llvm/Config/llvm-config.h>
46
47 #if LLVM_VERSION_MAJOR < 7
48 // Workaround http://llvm.org/PR23628
49 #pragma push_macro("DEBUG")
50 #undef DEBUG
51 #endif
52
53 #include <llvm/Config/llvm-config.h>
54 #include <llvm-c/Core.h>
55 #include <llvm-c/Support.h>
56 #include <llvm-c/ExecutionEngine.h>
57 #include <llvm/Target/TargetOptions.h>
58 #include <llvm/ExecutionEngine/ExecutionEngine.h>
59 #include <llvm/Analysis/TargetLibraryInfo.h>
60 #include <llvm/ExecutionEngine/SectionMemoryManager.h>
61 #include <llvm/Support/CommandLine.h>
62 #include <llvm/Support/PrettyStackTrace.h>
63 #include <llvm/ExecutionEngine/ObjectCache.h>
64 #include <llvm/Support/TargetSelect.h>
65 #include <llvm/CodeGen/SelectionDAGNodes.h>
66 #if LLVM_VERSION_MAJOR >= 15
67 #include <llvm/Support/MemoryBuffer.h>
68 #endif
69
70 #if LLVM_VERSION_MAJOR >= 17
71 #include <llvm/TargetParser/Host.h>
72 #include <llvm/TargetParser/Triple.h>
73 #else
74 #include <llvm/Support/Host.h>
75 #include <llvm/ADT/Triple.h>
76 #endif
77
78 #if LLVM_VERSION_MAJOR < 11
79 #include <llvm/IR/CallSite.h>
80 #endif
81 #include <llvm/IR/IRBuilder.h>
82 #include <llvm/IR/Module.h>
83 #include <llvm/Support/CBindingWrapping.h>
84
85 #include <llvm/Config/llvm-config.h>
86 #if LLVM_USE_INTEL_JITEVENTS
87 #include <llvm/ExecutionEngine/JITEventListener.h>
88 #endif
89
90 #if LLVM_VERSION_MAJOR < 7
91 // Workaround http://llvm.org/PR23628
92 #pragma pop_macro("DEBUG")
93 #endif
94
95 #include "c11/threads.h"
96 #include "util/u_thread.h"
97 #include "util/detect.h"
98 #include "util/u_debug.h"
99 #include "util/u_cpu_detect.h"
100
101 #include "lp_bld_misc.h"
102 #include "lp_bld_debug.h"
103
104 static void lp_run_atexit_for_destructors(void);
105
106 namespace {
107
108 class LLVMEnsureMultithreaded {
109 public:
LLVMEnsureMultithreaded()110 LLVMEnsureMultithreaded()
111 {
112 if (!LLVMIsMultithreaded()) {
113 LLVMStartMultithreaded();
114 }
115 }
116 };
117
118 static LLVMEnsureMultithreaded lLVMEnsureMultithreaded;
119
120 }
121
122 static once_flag init_native_targets_once_flag = ONCE_FLAG_INIT;
123
init_native_targets()124 static void init_native_targets()
125 {
126 // If we have a native target, initialize it to ensure it is linked in and
127 // usable by the JIT.
128 llvm::InitializeNativeTarget();
129
130 llvm::InitializeNativeTargetAsmPrinter();
131
132 llvm::InitializeNativeTargetDisassembler();
133 #if DEBUG
134 {
135 char *env_llc_options = getenv("GALLIVM_LLC_OPTIONS");
136 if (env_llc_options) {
137 char *option;
138 char *options[64] = {(char *) "llc"}; // Warning without cast
139 int n;
140 for (n = 0, option = strtok(env_llc_options, " "); option; n++, option = strtok(NULL, " ")) {
141 options[n + 1] = option;
142 }
143 if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) {
144 debug_printf("llc additional options (%d):\n", n);
145 for (int i = 1; i <= n; i++)
146 debug_printf("\t%s\n", options[i]);
147 debug_printf("\n");
148 }
149 LLVMParseCommandLineOptions(n + 1, options, NULL);
150 }
151 }
152 #endif
153 lp_run_atexit_for_destructors();
154 }
155
156 extern "C" void
lp_set_target_options(void)157 lp_set_target_options(void)
158 {
159 /* The llvm target registry is not thread-safe, so drivers and gallium frontends
160 * that want to initialize targets should use the lp_set_target_options()
161 * function to safely initialize targets.
162 *
163 * LLVM targets should be initialized before the driver or gallium frontend tries
164 * to access the registry.
165 */
166 call_once(&init_native_targets_once_flag, init_native_targets);
167 }
168
169 extern "C"
170 LLVMTargetLibraryInfoRef
gallivm_create_target_library_info(const char * triple)171 gallivm_create_target_library_info(const char *triple)
172 {
173 return reinterpret_cast<LLVMTargetLibraryInfoRef>(
174 new llvm::TargetLibraryInfoImpl(
175 llvm::Triple(triple)));
176 }
177
178 extern "C"
179 void
gallivm_dispose_target_library_info(LLVMTargetLibraryInfoRef library_info)180 gallivm_dispose_target_library_info(LLVMTargetLibraryInfoRef library_info)
181 {
182 delete reinterpret_cast<
183 llvm::TargetLibraryInfoImpl
184 *>(library_info);
185 }
186
187
188 typedef llvm::RTDyldMemoryManager BaseMemoryManager;
189
190
191 /*
192 * Delegating is tedious but the default manager class is hidden in an
193 * anonymous namespace in LLVM, so we cannot just derive from it to change
194 * its behavior.
195 */
196 class DelegatingJITMemoryManager : public BaseMemoryManager {
197
198 protected:
199 virtual BaseMemoryManager *mgr() const = 0;
200
201 public:
202 /*
203 * From RTDyldMemoryManager
204 */
allocateCodeSection(uintptr_t Size,unsigned Alignment,unsigned SectionID,llvm::StringRef SectionName)205 virtual uint8_t *allocateCodeSection(uintptr_t Size,
206 unsigned Alignment,
207 unsigned SectionID,
208 llvm::StringRef SectionName) {
209 return mgr()->allocateCodeSection(Size, Alignment, SectionID,
210 SectionName);
211 }
allocateDataSection(uintptr_t Size,unsigned Alignment,unsigned SectionID,llvm::StringRef SectionName,bool IsReadOnly)212 virtual uint8_t *allocateDataSection(uintptr_t Size,
213 unsigned Alignment,
214 unsigned SectionID,
215 llvm::StringRef SectionName,
216 bool IsReadOnly) {
217 return mgr()->allocateDataSection(Size, Alignment, SectionID,
218 SectionName,
219 IsReadOnly);
220 }
registerEHFrames(uint8_t * Addr,uint64_t LoadAddr,size_t Size)221 virtual void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
222 mgr()->registerEHFrames(Addr, LoadAddr, Size);
223 }
224 #if LLVM_VERSION_MAJOR >= 5
deregisterEHFrames()225 virtual void deregisterEHFrames() {
226 mgr()->deregisterEHFrames();
227 }
228 #else
deregisterEHFrames(uint8_t * Addr,uint64_t LoadAddr,size_t Size)229 virtual void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
230 mgr()->deregisterEHFrames(Addr, LoadAddr, Size);
231 }
232 #endif
getPointerToNamedFunction(const std::string & Name,bool AbortOnFailure=true)233 virtual void *getPointerToNamedFunction(const std::string &Name,
234 bool AbortOnFailure=true) {
235 return mgr()->getPointerToNamedFunction(Name, AbortOnFailure);
236 }
finalizeMemory(std::string * ErrMsg=0)237 virtual bool finalizeMemory(std::string *ErrMsg = 0) {
238 return mgr()->finalizeMemory(ErrMsg);
239 }
240 };
241
242
243 /*
244 * Delegate memory management to one shared manager for more efficient use
245 * of memory than creating a separate pool for each LLVM engine.
246 * Keep generated code until freeGeneratedCode() is called, instead of when
247 * memory manager is destroyed, which happens during engine destruction.
248 * This allows additional memory savings as we don't have to keep the engine
249 * around in order to use the code.
250 * All methods are delegated to the shared manager except destruction and
251 * deallocating code. For the latter we just remember what needs to be
252 * deallocated later. The shared manager is deleted once it is empty.
253 */
254 class ShaderMemoryManager : public DelegatingJITMemoryManager {
255
256 BaseMemoryManager *TheMM;
257
258 struct GeneratedCode {
259 typedef std::vector<void *> Vec;
260 Vec FunctionBody, ExceptionTable;
261 BaseMemoryManager *TheMM;
262
GeneratedCodeShaderMemoryManager::GeneratedCode263 GeneratedCode(BaseMemoryManager *MM) {
264 TheMM = MM;
265 }
266
~GeneratedCodeShaderMemoryManager::GeneratedCode267 ~GeneratedCode() {
268 }
269 };
270
271 GeneratedCode *code;
272
mgr() const273 BaseMemoryManager *mgr() const {
274 return TheMM;
275 }
276
277 public:
278
ShaderMemoryManager(BaseMemoryManager * MM)279 ShaderMemoryManager(BaseMemoryManager* MM) {
280 TheMM = MM;
281 code = new GeneratedCode(MM);
282 }
283
~ShaderMemoryManager()284 virtual ~ShaderMemoryManager() {
285 /*
286 * 'code' is purposely not deleted. It is the user's responsibility
287 * to call getGeneratedCode() and freeGeneratedCode().
288 */
289 }
290
getGeneratedCode()291 struct lp_generated_code *getGeneratedCode() {
292 return (struct lp_generated_code *) code;
293 }
294
freeGeneratedCode(struct lp_generated_code * code)295 static void freeGeneratedCode(struct lp_generated_code *code) {
296 delete (GeneratedCode *) code;
297 }
298
deallocateFunctionBody(void * Body)299 virtual void deallocateFunctionBody(void *Body) {
300 // remember for later deallocation
301 code->FunctionBody.push_back(Body);
302 }
303 };
304
305 class LPObjectCache : public llvm::ObjectCache {
306 private:
307 bool has_object;
308 struct lp_cached_code *cache_out;
309 public:
LPObjectCache(struct lp_cached_code * cache)310 LPObjectCache(struct lp_cached_code *cache) {
311 cache_out = cache;
312 has_object = false;
313 }
314
~LPObjectCache()315 ~LPObjectCache() {
316 }
notifyObjectCompiled(const llvm::Module * M,llvm::MemoryBufferRef Obj)317 void notifyObjectCompiled(const llvm::Module *M, llvm::MemoryBufferRef Obj) {
318 const std::string ModuleID = M->getModuleIdentifier();
319 if (has_object)
320 fprintf(stderr, "CACHE ALREADY HAS MODULE OBJECT\n");
321 has_object = true;
322 cache_out->data_size = Obj.getBufferSize();
323 cache_out->data = malloc(cache_out->data_size);
324 memcpy(cache_out->data, Obj.getBufferStart(), cache_out->data_size);
325 }
326
getObject(const llvm::Module * M)327 virtual std::unique_ptr<llvm::MemoryBuffer> getObject(const llvm::Module *M) {
328 if (cache_out->data_size) {
329 return llvm::MemoryBuffer::getMemBuffer(llvm::StringRef((const char *)cache_out->data, cache_out->data_size), "", false);
330 }
331 return NULL;
332 }
333
334 };
335
336 /**
337 * Same as LLVMCreateJITCompilerForModule, but:
338 * - allows using MCJIT and enabling AVX feature where available.
339 * - set target options
340 *
341 * See also:
342 * - llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp
343 * - llvm/tools/lli/lli.cpp
344 * - http://markmail.org/message/ttkuhvgj4cxxy2on#query:+page:1+mid:aju2dggerju3ivd3+state:results
345 */
346 extern "C"
347 LLVMBool
lp_build_create_jit_compiler_for_module(LLVMExecutionEngineRef * OutJIT,lp_generated_code ** OutCode,struct lp_cached_code * cache_out,LLVMModuleRef M,LLVMMCJITMemoryManagerRef CMM,unsigned OptLevel,char ** OutError)348 lp_build_create_jit_compiler_for_module(LLVMExecutionEngineRef *OutJIT,
349 lp_generated_code **OutCode,
350 struct lp_cached_code *cache_out,
351 LLVMModuleRef M,
352 LLVMMCJITMemoryManagerRef CMM,
353 unsigned OptLevel,
354 char **OutError)
355 {
356 using namespace llvm;
357
358 std::string Error;
359 EngineBuilder builder(std::unique_ptr<Module>(unwrap(M)));
360
361 /**
362 * LLVM 3.1+ haven't more "extern unsigned llvm::StackAlignmentOverride" and
363 * friends for configuring code generation options, like stack alignment.
364 */
365 TargetOptions options;
366 #if DETECT_ARCH_X86 && LLVM_VERSION_MAJOR < 13
367 options.StackAlignmentOverride = 4;
368 #endif
369
370 builder.setEngineKind(EngineKind::JIT)
371 .setErrorStr(&Error)
372 .setTargetOptions(options)
373 #if LLVM_VERSION_MAJOR >= 18
374 .setOptLevel((CodeGenOptLevel)OptLevel);
375 #else
376 .setOptLevel((CodeGenOpt::Level)OptLevel);
377 #endif
378
379 #if DETECT_OS_WINDOWS
380 /*
381 * MCJIT works on Windows, but currently only through ELF object format.
382 *
383 * XXX: We could use `LLVM_HOST_TRIPLE "-elf"` but LLVM_HOST_TRIPLE has
384 * different strings for MinGW/MSVC, so better play it safe and be
385 * explicit.
386 */
387 # if DETECT_ARCH_X86_64
388 LLVMSetTarget(M, "x86_64-pc-win32-elf");
389 # elif DETECT_ARCH_X86
390 LLVMSetTarget(M, "i686-pc-win32-elf");
391 # elif DETECT_ARCH_AARCH64
392 LLVMSetTarget(M, "aarch64-pc-win32-elf");
393 # else
394 # error Unsupported architecture for MCJIT on Windows.
395 # endif
396 #endif
397
398 llvm::SmallVector<std::string, 16> MAttrs;
399
400 #if DETECT_ARCH_ARM
401 /* llvm-3.3+ implements sys::getHostCPUFeatures for Arm,
402 * which allows us to enable/disable code generation based
403 * on the results of cpuid on these architectures.
404 */
405 llvm::StringMap<bool> features;
406 llvm::sys::getHostCPUFeatures(features);
407
408 for (StringMapIterator<bool> f = features.begin();
409 f != features.end();
410 ++f) {
411 MAttrs.push_back(((*f).second ? "+" : "-") + (*f).first().str());
412 }
413 #elif DETECT_ARCH_X86 || DETECT_ARCH_X86_64
414 /*
415 * Because we can override cpu caps with environment variables,
416 * so we do not use llvm::sys::getHostCPUFeatures to detect cpu features
417 * but using util_get_cpu_caps() instead.
418 */
419 #if DETECT_ARCH_X86_64
420 /*
421 * Without this, on some "buggy" qemu cpu setup, LLVM could crash
422 * if LLVM detects the wrong CPU type.
423 */
424 MAttrs.push_back("+64bit");
425 #endif
426 MAttrs.push_back(util_get_cpu_caps()->has_sse ? "+sse" : "-sse" );
427 MAttrs.push_back(util_get_cpu_caps()->has_sse2 ? "+sse2" : "-sse2" );
428 MAttrs.push_back(util_get_cpu_caps()->has_sse3 ? "+sse3" : "-sse3" );
429 MAttrs.push_back(util_get_cpu_caps()->has_ssse3 ? "+ssse3" : "-ssse3" );
430 MAttrs.push_back(util_get_cpu_caps()->has_sse4_1 ? "+sse4.1" : "-sse4.1");
431 MAttrs.push_back(util_get_cpu_caps()->has_sse4_2 ? "+sse4.2" : "-sse4.2");
432 /*
433 * AVX feature is not automatically detected from CPUID by the X86 target
434 * yet, because the old (yet default) JIT engine is not capable of
435 * emitting the opcodes. On newer llvm versions it is and at least some
436 * versions (tested with 3.3) will emit avx opcodes without this anyway.
437 */
438 MAttrs.push_back(util_get_cpu_caps()->has_avx ? "+avx" : "-avx");
439 MAttrs.push_back(util_get_cpu_caps()->has_f16c ? "+f16c" : "-f16c");
440 MAttrs.push_back(util_get_cpu_caps()->has_fma ? "+fma" : "-fma");
441 MAttrs.push_back(util_get_cpu_caps()->has_avx2 ? "+avx2" : "-avx2");
442
443 /* All avx512 have avx512f */
444 MAttrs.push_back(util_get_cpu_caps()->has_avx512f ? "+avx512f" : "-avx512f");
445 MAttrs.push_back(util_get_cpu_caps()->has_avx512cd ? "+avx512cd" : "-avx512cd");
446 MAttrs.push_back(util_get_cpu_caps()->has_avx512er ? "+avx512er" : "-avx512er");
447 MAttrs.push_back(util_get_cpu_caps()->has_avx512pf ? "+avx512pf" : "-avx512pf");
448 MAttrs.push_back(util_get_cpu_caps()->has_avx512bw ? "+avx512bw" : "-avx512bw");
449 MAttrs.push_back(util_get_cpu_caps()->has_avx512dq ? "+avx512dq" : "-avx512dq");
450 MAttrs.push_back(util_get_cpu_caps()->has_avx512vl ? "+avx512vl" : "-avx512vl");
451 #endif
452 #if DETECT_ARCH_ARM
453 if (!util_get_cpu_caps()->has_neon) {
454 MAttrs.push_back("-neon");
455 MAttrs.push_back("-crypto");
456 MAttrs.push_back("-vfp2");
457 }
458 #endif
459
460 #if DETECT_ARCH_PPC
461 MAttrs.push_back(util_get_cpu_caps()->has_altivec ? "+altivec" : "-altivec");
462 /*
463 * Bug 25503 is fixed, by the same fix that fixed
464 * bug 26775, in versions of LLVM later than 3.8 (starting with 3.8.1).
465 * BZ 33531 actually comprises more than one bug, all of
466 * which are fixed in LLVM 4.0.
467 *
468 * With LLVM 4.0 or higher:
469 * Make sure VSX instructions are ENABLED (if supported), unless
470 * VSX instructions are explicitly enabled/disabled via GALLIVM_VSX=1 or 0.
471 */
472 if (util_get_cpu_caps()->has_altivec) {
473 MAttrs.push_back(util_get_cpu_caps()->has_vsx ? "+vsx" : "-vsx");
474 }
475 #endif
476
477 #if DETECT_ARCH_MIPS64
478 MAttrs.push_back(util_get_cpu_caps()->has_msa ? "+msa" : "-msa");
479 /* MSA requires a 64-bit FPU register file */
480 MAttrs.push_back("+fp64");
481 #endif
482
483 builder.setMAttrs(MAttrs);
484
485 if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) {
486 int n = MAttrs.size();
487 if (n > 0) {
488 debug_printf("llc -mattr option(s): ");
489 for (int i = 0; i < n; i++)
490 debug_printf("%s%s", MAttrs[i].c_str(), (i < n - 1) ? "," : "");
491 debug_printf("\n");
492 }
493 }
494
495 StringRef MCPU = llvm::sys::getHostCPUName();
496 /*
497 * The cpu bits are no longer set automatically, so need to set mcpu manually.
498 * Note that the MAttrs set above will be sort of ignored (since we should
499 * not set any which would not be set by specifying the cpu anyway).
500 * It ought to be safe though since getHostCPUName() should include bits
501 * not only from the cpu but environment as well (for instance if it's safe
502 * to use avx instructions which need OS support). According to
503 * http://llvm.org/bugs/show_bug.cgi?id=19429 however if I understand this
504 * right it may be necessary to specify older cpu (or disable mattrs) though
505 * when not using MCJIT so no instructions are generated which the old JIT
506 * can't handle. Not entirely sure if we really need to do anything yet.
507 */
508
509 #if DETECT_ARCH_PPC_64
510 /*
511 * Large programs, e.g. gnome-shell and firefox, may tax the addressability
512 * of the Medium code model once dynamically generated JIT-compiled shader
513 * programs are linked in and relocated. Yet the default code model as of
514 * LLVM 8 is Medium or even Small.
515 * The cost of changing from Medium to Large is negligible:
516 * - an additional 8-byte pointer stored immediately before the shader entrypoint;
517 * - change an add-immediate (addis) instruction to a load (ld).
518 */
519 builder.setCodeModel(CodeModel::Large);
520
521 #if UTIL_ARCH_LITTLE_ENDIAN
522 /*
523 * Versions of LLVM prior to 4.0 lacked a table entry for "POWER8NVL",
524 * resulting in (big-endian) "generic" being returned on
525 * little-endian Power8NVL systems. The result was that code that
526 * attempted to load the least significant 32 bits of a 64-bit quantity
527 * from memory loaded the wrong half. This resulted in failures in some
528 * Piglit tests, e.g.
529 * .../arb_gpu_shader_fp64/execution/conversion/frag-conversion-explicit-double-uint
530 */
531 if (MCPU == "generic")
532 MCPU = "pwr8";
533 #endif
534 #endif
535
536 #if DETECT_ARCH_MIPS64
537 /*
538 * ls3a4000 CPU and ls2k1000 SoC is a mips64r5 compatible with MSA SIMD
539 * instruction set implemented, while ls3a3000 is mips64r2 compatible
540 * only. getHostCPUName() return "generic" on all loongson
541 * mips CPU currently. So we override the MCPU to mips64r5 if MSA is
542 * implemented, feedback to mips64r2 for all other ordinary mips64 cpu.
543 */
544 if (MCPU == "generic")
545 MCPU = util_get_cpu_caps()->has_msa ? "mips64r5" : "mips64r2";
546 #endif
547
548 builder.setMCPU(MCPU);
549 if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) {
550 debug_printf("llc -mcpu option: %s\n", MCPU.str().c_str());
551 }
552
553 ShaderMemoryManager *MM = NULL;
554 BaseMemoryManager* JMM = reinterpret_cast<BaseMemoryManager*>(CMM);
555 MM = new ShaderMemoryManager(JMM);
556 *OutCode = MM->getGeneratedCode();
557
558 builder.setMCJITMemoryManager(std::unique_ptr<RTDyldMemoryManager>(MM));
559 MM = NULL; // ownership taken by std::unique_ptr
560
561 ExecutionEngine *JIT;
562
563 JIT = builder.create();
564
565 if (cache_out) {
566 LPObjectCache *objcache = new LPObjectCache(cache_out);
567 JIT->setObjectCache(objcache);
568 cache_out->jit_obj_cache = (void *)objcache;
569 }
570
571 #if LLVM_USE_INTEL_JITEVENTS
572 JITEventListener *JEL = JITEventListener::createIntelJITEventListener();
573 JIT->RegisterJITEventListener(JEL);
574 #endif
575 if (JIT) {
576 *OutJIT = wrap(JIT);
577 return 0;
578 }
579 lp_free_generated_code(*OutCode);
580 *OutCode = 0;
581 delete MM;
582 *OutError = strdup(Error.c_str());
583 return 1;
584 }
585
586
587 extern "C"
588 void
lp_free_generated_code(struct lp_generated_code * code)589 lp_free_generated_code(struct lp_generated_code *code)
590 {
591 ShaderMemoryManager::freeGeneratedCode(code);
592 }
593
594 extern "C"
595 LLVMMCJITMemoryManagerRef
lp_get_default_memory_manager()596 lp_get_default_memory_manager()
597 {
598 BaseMemoryManager *mm;
599 mm = new llvm::SectionMemoryManager();
600 return reinterpret_cast<LLVMMCJITMemoryManagerRef>(mm);
601 }
602
603 extern "C"
604 void
lp_free_memory_manager(LLVMMCJITMemoryManagerRef memorymgr)605 lp_free_memory_manager(LLVMMCJITMemoryManagerRef memorymgr)
606 {
607 delete reinterpret_cast<BaseMemoryManager*>(memorymgr);
608 }
609
610 extern "C" void
lp_free_objcache(void * objcache_ptr)611 lp_free_objcache(void *objcache_ptr)
612 {
613 LPObjectCache *objcache = (LPObjectCache *)objcache_ptr;
614 delete objcache;
615 }
616
617 extern "C" LLVMValueRef
lp_get_called_value(LLVMValueRef call)618 lp_get_called_value(LLVMValueRef call)
619 {
620 return LLVMGetCalledValue(call);
621 }
622
623 extern "C" bool
lp_is_function(LLVMValueRef v)624 lp_is_function(LLVMValueRef v)
625 {
626 return LLVMGetValueKind(v) == LLVMFunctionValueKind;
627 }
628
629 extern "C" void
lp_set_module_stack_alignment_override(LLVMModuleRef MRef,unsigned align)630 lp_set_module_stack_alignment_override(LLVMModuleRef MRef, unsigned align)
631 {
632 #if LLVM_VERSION_MAJOR >= 13
633 llvm::Module *M = llvm::unwrap(MRef);
634 M->setOverrideStackAlignment(align);
635 #endif
636 }
637
638 using namespace llvm;
639
640 class GallivmRunAtExitForStaticDestructors : public SDNode
641 {
642 public:
643 /* getSDVTList (protected) calls getValueTypeList (private), which contains static variables. */
GallivmRunAtExitForStaticDestructors()644 GallivmRunAtExitForStaticDestructors(): SDNode(0, 0, DebugLoc(), getSDVTList(MVT::Other))
645 {
646 }
647 };
648
649 static void
lp_run_atexit_for_destructors(void)650 lp_run_atexit_for_destructors(void)
651 {
652 /* LLVM >= 16 registers static variable destructors on the first compile, which gcc
653 * implements by calling atexit there. Before that, u_queue registers its atexit
654 * handler to kill all threads. Since exit() runs atexit handlers in the reverse order,
655 * the LLVM destructors are called first while shader compiler threads may still be
656 * running, which crashes in LLVM in SelectionDAG.cpp.
657 *
658 * The solution is to run the code that declares the LLVM static variables first,
659 * so that atexit for LLVM is registered first and u_queue is registered after that,
660 * which ensures that all u_queue threads are terminated before LLVM destructors are
661 * called.
662 *
663 * This just executes the code that declares static variables.
664 */
665 GallivmRunAtExitForStaticDestructors();
666 }
667