1 /****************************************************************************
2 * Copyright (C) 2014-2018 Intel Corporation. All Rights Reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * @file JitManager.cpp
24 *
25 * @brief Implementation if the Jit Manager.
26 *
27 * Notes:
28 *
29 ******************************************************************************/
30 #include "jit_pch.hpp"
31
32 #include "JitManager.h"
33 #include "jit_api.h"
34 #include "fetch_jit.h"
35
36 #include "core/state.h"
37
38 #include "gen_state_llvm.h"
39
40 #include <sstream>
41 #if defined(_WIN32)
42 #include <psapi.h>
43 #include <cstring>
44
45 #define INTEL_OUTPUT_DIR "c:\\Intel"
46 #define SWR_OUTPUT_DIR INTEL_OUTPUT_DIR "\\SWR"
47 #define JITTER_OUTPUT_DIR SWR_OUTPUT_DIR "\\Jitter"
48 #endif // _WIN32
49
50 #if defined(__APPLE__) || defined(FORCE_LINUX) || defined(__linux__) || defined(__gnu_linux__)
51 #include <pwd.h>
52 #include <sys/stat.h>
53 #endif
54
55
56 using namespace llvm;
57 using namespace SwrJit;
58
59 //////////////////////////////////////////////////////////////////////////
60 /// @brief Contructor for JitManager.
61 /// @param simdWidth - SIMD width to be used in generated program.
JitManager(uint32_t simdWidth,const char * arch,const char * core)62 JitManager::JitManager(uint32_t simdWidth, const char* arch, const char* core) :
63 mContext(), mBuilder(mContext), mIsModuleFinalized(true), mJitNumber(0), mVWidth(simdWidth),
64 mArch(arch)
65 {
66 mpCurrentModule = nullptr;
67 mpExec = nullptr;
68
69 InitializeNativeTarget();
70 InitializeNativeTargetAsmPrinter();
71 InitializeNativeTargetDisassembler();
72
73
74 // force JIT to use the same CPU arch as the rest of swr
75 if (mArch.AVX512F())
76 {
77 #if USE_SIMD16_SHADERS
78 if (mArch.AVX512ER())
79 {
80 mHostCpuName = StringRef("knl");
81 }
82 else
83 {
84 mHostCpuName = StringRef("skylake-avx512");
85 }
86 mUsingAVX512 = true;
87 #else
88 mHostCpuName = StringRef("core-avx2");
89 #endif
90 if (mVWidth == 0)
91 {
92 mVWidth = 8;
93 }
94 }
95 else if (mArch.AVX2())
96 {
97 mHostCpuName = StringRef("core-avx2");
98 if (mVWidth == 0)
99 {
100 mVWidth = 8;
101 }
102 }
103 else if (mArch.AVX())
104 {
105 if (mArch.F16C())
106 {
107 mHostCpuName = StringRef("core-avx-i");
108 }
109 else
110 {
111 mHostCpuName = StringRef("corei7-avx");
112 }
113 if (mVWidth == 0)
114 {
115 mVWidth = 8;
116 }
117 }
118 else
119 {
120 SWR_INVALID("Jitting requires at least AVX ISA support");
121 }
122
123
124 mOptLevel = CodeGenOpt::Aggressive;
125
126 if (KNOB_JIT_OPTIMIZATION_LEVEL >= CodeGenOpt::None &&
127 KNOB_JIT_OPTIMIZATION_LEVEL <= CodeGenOpt::Aggressive)
128 {
129 mOptLevel = CodeGenOpt::Level(KNOB_JIT_OPTIMIZATION_LEVEL);
130 }
131
132 if (KNOB_JIT_ENABLE_CACHE)
133 {
134 mCache.Init(this, mHostCpuName, mOptLevel);
135 }
136
137 SetupNewModule();
138 mIsModuleFinalized = true;
139
140 // fetch function signature
141 #if USE_SIMD16_SHADERS
142 // typedef void(__cdecl *PFN_FETCH_FUNC)(SWR_FETCH_CONTEXT& fetchInfo, simd16vertex& out);
143 #else
144 // typedef void(__cdecl *PFN_FETCH_FUNC)(SWR_FETCH_CONTEXT& fetchInfo, simdvertex& out);
145 #endif
146 std::vector<Type*> fsArgs;
147
148 // llvm5 is picky and does not take a void * type
149 fsArgs.push_back(PointerType::get(Gen_SWR_FETCH_CONTEXT(this), 0));
150
151 fsArgs.push_back(Type::getInt8PtrTy(mContext));
152
153 fsArgs.push_back(PointerType::get(Gen_SWR_FETCH_CONTEXT(this), 0));
154 #if USE_SIMD16_SHADERS
155 fsArgs.push_back(PointerType::get(Gen_simd16vertex(this), 0));
156 #else
157 fsArgs.push_back(PointerType::get(Gen_simdvertex(this), 0));
158 #endif
159
160 mFetchShaderTy = FunctionType::get(Type::getVoidTy(mContext), fsArgs, false);
161
162 #if defined(_MSC_VER)
163 // explicitly instantiate used symbols from potentially staticly linked libs
164 sys::DynamicLibrary::AddSymbol("exp2f", &exp2f);
165 sys::DynamicLibrary::AddSymbol("log2f", &log2f);
166 sys::DynamicLibrary::AddSymbol("sinf", &sinf);
167 sys::DynamicLibrary::AddSymbol("cosf", &cosf);
168 sys::DynamicLibrary::AddSymbol("powf", &powf);
169 #endif
170
171 #if defined(_WIN32)
172 if (KNOB_DUMP_SHADER_IR)
173 {
174 CreateDirectoryPath(INTEL_OUTPUT_DIR);
175 CreateDirectoryPath(SWR_OUTPUT_DIR);
176 CreateDirectoryPath(JITTER_OUTPUT_DIR);
177 }
178 #endif
179 }
180
CreateExecEngine(std::unique_ptr<Module> pModule)181 void JitManager::CreateExecEngine(std::unique_ptr<Module> pModule)
182 {
183 TargetOptions tOpts;
184 tOpts.AllowFPOpFusion = FPOpFusion::Fast;
185 tOpts.NoInfsFPMath = false;
186 tOpts.NoNaNsFPMath = false;
187 tOpts.UnsafeFPMath = false;
188
189 // tOpts.PrintMachineCode = true;
190
191 mpExec = EngineBuilder(std::move(pModule))
192 .setTargetOptions(tOpts)
193 .setOptLevel(mOptLevel)
194 .setMCPU(mHostCpuName)
195 .create();
196
197 if (KNOB_JIT_ENABLE_CACHE)
198 {
199 mpExec->setObjectCache(&mCache);
200 }
201
202 #if LLVM_USE_INTEL_JITEVENTS
203 JITEventListener* vTune = JITEventListener::createIntelJITEventListener();
204 mpExec->RegisterJITEventListener(vTune);
205 #endif
206
207 mvExecEngines.push_back(mpExec);
208 }
209
210 //////////////////////////////////////////////////////////////////////////
211 /// @brief Create new LLVM module.
SetupNewModule()212 void JitManager::SetupNewModule()
213 {
214 SWR_ASSERT(mIsModuleFinalized == true && "Current module is not finalized!");
215
216 std::unique_ptr<Module> newModule(new Module("", mContext));
217 mpCurrentModule = newModule.get();
218 mpCurrentModule->setTargetTriple(sys::getProcessTriple());
219 CreateExecEngine(std::move(newModule));
220 mIsModuleFinalized = false;
221 }
222
223
224 DIType*
CreateDebugStructType(StructType * pType,const std::string & name,DIFile * pFile,uint32_t lineNum,const std::vector<std::pair<std::string,uint32_t>> & members)225 JitManager::CreateDebugStructType(StructType* pType,
226 const std::string& name,
227 DIFile* pFile,
228 uint32_t lineNum,
229 const std::vector<std::pair<std::string, uint32_t>>& members)
230 {
231 DIBuilder builder(*mpCurrentModule);
232 SmallVector<Metadata*, 8> ElemTypes;
233 DataLayout DL = DataLayout(mpCurrentModule);
234 uint32_t size = DL.getTypeAllocSizeInBits(pType);
235 uint32_t alignment = DL.getABITypeAlignment(pType);
236 DINode::DIFlags flags = DINode::DIFlags::FlagPublic;
237
238 DICompositeType* pDIStructTy = builder.createStructType(pFile,
239 name,
240 pFile,
241 lineNum,
242 size,
243 alignment,
244 flags,
245 nullptr,
246 builder.getOrCreateArray(ElemTypes));
247
248 // Register mapping now to break loops (in case struct contains itself or pointers to itself)
249 mDebugStructMap[pType] = pDIStructTy;
250
251 uint32_t idx = 0;
252 for (auto& elem : pType->elements())
253 {
254 std::string name = members[idx].first;
255 uint32_t lineNum = members[idx].second;
256 size = DL.getTypeAllocSizeInBits(elem);
257 alignment = DL.getABITypeAlignment(elem);
258 uint32_t offset = DL.getStructLayout(pType)->getElementOffsetInBits(idx);
259 llvm::DIType* pDebugTy = GetDebugType(elem);
260 ElemTypes.push_back(builder.createMemberType(
261 pDIStructTy, name, pFile, lineNum, size, alignment, offset, flags, pDebugTy));
262
263 idx++;
264 }
265
266 pDIStructTy->replaceElements(builder.getOrCreateArray(ElemTypes));
267 return pDIStructTy;
268 }
269
GetDebugArrayType(Type * pTy)270 DIType* JitManager::GetDebugArrayType(Type* pTy)
271 {
272 DIBuilder builder(*mpCurrentModule);
273 DataLayout DL = DataLayout(mpCurrentModule);
274 ArrayType* pArrayTy = cast<ArrayType>(pTy);
275 uint32_t size = DL.getTypeAllocSizeInBits(pArrayTy);
276 uint32_t alignment = DL.getABITypeAlignment(pArrayTy);
277
278 SmallVector<Metadata*, 8> Elems;
279 Elems.push_back(builder.getOrCreateSubrange(0, pArrayTy->getNumElements()));
280 return builder.createArrayType(
281 size, alignment, GetDebugType(pArrayTy->getElementType()), builder.getOrCreateArray(Elems));
282 }
283
284 // Create a DIType from llvm Type
GetDebugType(Type * pTy)285 DIType* JitManager::GetDebugType(Type* pTy)
286 {
287 DIBuilder builder(*mpCurrentModule);
288 Type::TypeID id = pTy->getTypeID();
289
290 switch (id)
291 {
292 case Type::VoidTyID:
293 return builder.createUnspecifiedType("void");
294 break;
295 case Type::HalfTyID:
296 return builder.createBasicType("float16", 16, dwarf::DW_ATE_float);
297 break;
298 case Type::FloatTyID:
299 return builder.createBasicType("float", 32, dwarf::DW_ATE_float);
300 break;
301 case Type::DoubleTyID:
302 return builder.createBasicType("double", 64, dwarf::DW_ATE_float);
303 break;
304 case Type::IntegerTyID:
305 return GetDebugIntegerType(pTy);
306 break;
307 case Type::StructTyID:
308 return GetDebugStructType(pTy);
309 break;
310 case Type::ArrayTyID:
311 return GetDebugArrayType(pTy);
312 break;
313 case Type::PointerTyID:
314 return builder.createPointerType(GetDebugType(pTy->getPointerElementType()), 64, 64);
315 break;
316 #if LLVM_VERSION_MAJOR >= 11
317 case Type::FixedVectorTyID:
318 #else
319 case Type::VectorTyID:
320 #endif
321 return GetDebugVectorType(pTy);
322 break;
323 case Type::FunctionTyID:
324 return GetDebugFunctionType(pTy);
325 break;
326 default:
327 SWR_ASSERT(false, "Unimplemented llvm type");
328 }
329 return nullptr;
330 }
331
332 // Create a DISubroutineType from an llvm FunctionType
GetDebugFunctionType(Type * pTy)333 DIType* JitManager::GetDebugFunctionType(Type* pTy)
334 {
335 SmallVector<Metadata*, 8> ElemTypes;
336 FunctionType* pFuncTy = cast<FunctionType>(pTy);
337 DIBuilder builder(*mpCurrentModule);
338
339 // Add result type
340 ElemTypes.push_back(GetDebugType(pFuncTy->getReturnType()));
341
342 // Add arguments
343 for (auto& param : pFuncTy->params())
344 {
345 ElemTypes.push_back(GetDebugType(param));
346 }
347
348 return builder.createSubroutineType(builder.getOrCreateTypeArray(ElemTypes));
349 }
350
GetDebugIntegerType(Type * pTy)351 DIType* JitManager::GetDebugIntegerType(Type* pTy)
352 {
353 DIBuilder builder(*mpCurrentModule);
354 IntegerType* pIntTy = cast<IntegerType>(pTy);
355 switch (pIntTy->getBitWidth())
356 {
357 case 1:
358 return builder.createBasicType("int1", 1, dwarf::DW_ATE_unsigned);
359 break;
360 case 8:
361 return builder.createBasicType("int8", 8, dwarf::DW_ATE_signed);
362 break;
363 case 16:
364 return builder.createBasicType("int16", 16, dwarf::DW_ATE_signed);
365 break;
366 case 32:
367 return builder.createBasicType("int", 32, dwarf::DW_ATE_signed);
368 break;
369 case 64:
370 return builder.createBasicType("int64", 64, dwarf::DW_ATE_signed);
371 break;
372 case 128:
373 return builder.createBasicType("int128", 128, dwarf::DW_ATE_signed);
374 break;
375 default:
376 SWR_ASSERT(false, "Unimplemented integer bit width");
377 }
378 return nullptr;
379 }
380
GetDebugVectorType(Type * pTy)381 DIType* JitManager::GetDebugVectorType(Type* pTy)
382 {
383 DIBuilder builder(*mpCurrentModule);
384 #if LLVM_VERSION_MAJOR >= 12
385 FixedVectorType* pVecTy = cast<FixedVectorType>(pTy);
386 #elif LLVM_VERSION_MAJOR >= 11
387 VectorType* pVecTy = cast<VectorType>(pTy);
388 #else
389 auto pVecTy = pTy;
390 #endif
391 DataLayout DL = DataLayout(mpCurrentModule);
392 uint32_t size = DL.getTypeAllocSizeInBits(pVecTy);
393 uint32_t alignment = DL.getABITypeAlignment(pVecTy);
394 SmallVector<Metadata*, 1> Elems;
395
396 #if LLVM_VERSION_MAJOR >= 11
397 Elems.push_back(builder.getOrCreateSubrange(0, pVecTy->getNumElements()));
398 #else
399 Elems.push_back(builder.getOrCreateSubrange(0, pVecTy->getVectorNumElements()));
400 #endif
401
402 return builder.createVectorType(size,
403 alignment,
404 #if LLVM_VERSION_MAJOR >= 11
405 GetDebugType(pVecTy->getElementType()),
406 #else
407 GetDebugType(pVecTy->getVectorElementType()),
408 #endif
409 builder.getOrCreateArray(Elems));
410 }
411
412 //////////////////////////////////////////////////////////////////////////
413 /// @brief Dump function x86 assembly to file.
414 /// @note This should only be called after the module has been jitted to x86 and the
415 /// module will not be further accessed.
DumpAsm(Function * pFunction,const char * fileName)416 void JitManager::DumpAsm(Function* pFunction, const char* fileName)
417 {
418 if (KNOB_DUMP_SHADER_IR)
419 {
420 #if defined(_WIN32)
421 DWORD pid = GetCurrentProcessId();
422 char procname[MAX_PATH];
423 GetModuleFileNameA(NULL, procname, MAX_PATH);
424 const char* pBaseName = strrchr(procname, '\\');
425 std::stringstream outDir;
426 outDir << JITTER_OUTPUT_DIR << pBaseName << "_" << pid << std::ends;
427 CreateDirectoryPath(outDir.str().c_str());
428 #endif
429
430 std::error_code EC;
431 Module* pModule = pFunction->getParent();
432 const char* funcName = pFunction->getName().data();
433 char fName[256];
434 #if defined(_WIN32)
435 sprintf(fName, "%s\\%s.%s.asm", outDir.str().c_str(), funcName, fileName);
436 #else
437 sprintf(fName, "%s.%s.asm", funcName, fileName);
438 #endif
439
440 raw_fd_ostream filestream(fName, EC, llvm::sys::fs::F_None);
441
442 legacy::PassManager* pMPasses = new legacy::PassManager();
443 auto* pTarget = mpExec->getTargetMachine();
444 pTarget->Options.MCOptions.AsmVerbose = true;
445 #if LLVM_VERSION_MAJOR >= 10
446 pTarget->addPassesToEmitFile(
447 *pMPasses, filestream, nullptr, CGFT_AssemblyFile);
448 #elif LLVM_VERSION_MAJOR >= 7
449 pTarget->addPassesToEmitFile(
450 *pMPasses, filestream, nullptr, TargetMachine::CGFT_AssemblyFile);
451 #else
452 pTarget->addPassesToEmitFile(*pMPasses, filestream, TargetMachine::CGFT_AssemblyFile);
453 #endif
454 pMPasses->run(*pModule);
455 delete pMPasses;
456 pTarget->Options.MCOptions.AsmVerbose = false;
457 }
458 }
459
GetOutputDir()460 std::string JitManager::GetOutputDir()
461 {
462 #if defined(_WIN32)
463 DWORD pid = GetCurrentProcessId();
464 char procname[MAX_PATH];
465 GetModuleFileNameA(NULL, procname, MAX_PATH);
466 const char* pBaseName = strrchr(procname, '\\');
467 std::stringstream outDir;
468 outDir << JITTER_OUTPUT_DIR << pBaseName << "_" << pid;
469 CreateDirectoryPath(outDir.str().c_str());
470 return outDir.str();
471 #endif
472 return "";
473 }
474
475 //////////////////////////////////////////////////////////////////////////
476 /// @brief Dump function to file.
DumpToFile(Module * M,const char * fileName,llvm::AssemblyAnnotationWriter * annotater)477 void JitManager::DumpToFile(Module* M,
478 const char* fileName,
479 llvm::AssemblyAnnotationWriter* annotater)
480 {
481 if (KNOB_DUMP_SHADER_IR)
482 {
483 std::string outDir = GetOutputDir();
484
485 std::error_code EC;
486 const char* funcName = M->getName().data();
487 char fName[256];
488 #if defined(_WIN32)
489 sprintf(fName, "%s\\%s.%s.ll", outDir.c_str(), funcName, fileName);
490 #else
491 sprintf(fName, "%s.%s.ll", funcName, fileName);
492 #endif
493 raw_fd_ostream fd(fName, EC, llvm::sys::fs::F_None);
494 M->print(fd, annotater);
495 fd.flush();
496 }
497 }
498
499 //////////////////////////////////////////////////////////////////////////
500 /// @brief Dump function to file.
DumpToFile(Function * f,const char * fileName)501 void JitManager::DumpToFile(Function* f, const char* fileName)
502 {
503 if (KNOB_DUMP_SHADER_IR)
504 {
505 std::string outDir = GetOutputDir();
506
507 std::error_code EC;
508 const char* funcName = f->getName().data();
509 char fName[256];
510 #if defined(_WIN32)
511 sprintf(fName, "%s\\%s.%s.ll", outDir.c_str(), funcName, fileName);
512 #else
513 sprintf(fName, "%s.%s.ll", funcName, fileName);
514 #endif
515 raw_fd_ostream fd(fName, EC, llvm::sys::fs::F_None);
516 f->print(fd, nullptr);
517
518 #if defined(_WIN32)
519 sprintf(fName, "%s\\cfg.%s.%s.dot", outDir.c_str(), funcName, fileName);
520 #else
521 sprintf(fName, "cfg.%s.%s.dot", funcName, fileName);
522 #endif
523 fd.flush();
524
525 raw_fd_ostream fd_cfg(fName, EC, llvm::sys::fs::F_Text);
526 WriteGraph(fd_cfg, (const Function*)f);
527
528 fd_cfg.flush();
529 }
530 }
531
532 extern "C" {
533 bool g_DllActive = true;
534
535 //////////////////////////////////////////////////////////////////////////
536 /// @brief Create JIT context.
537 /// @param simdWidth - SIMD width to be used in generated program.
JitCreateContext(uint32_t targetSimdWidth,const char * arch,const char * core)538 HANDLE JITCALL JitCreateContext(uint32_t targetSimdWidth, const char* arch, const char* core)
539 {
540 return new JitManager(targetSimdWidth, arch, core);
541 }
542
543 //////////////////////////////////////////////////////////////////////////
544 /// @brief Destroy JIT context.
JitDestroyContext(HANDLE hJitContext)545 void JITCALL JitDestroyContext(HANDLE hJitContext)
546 {
547 if (g_DllActive)
548 {
549 delete reinterpret_cast<JitManager*>(hJitContext);
550 }
551 }
552 }
553
554 //////////////////////////////////////////////////////////////////////////
555 /// JitCache
556 //////////////////////////////////////////////////////////////////////////
557
558 //////////////////////////////////////////////////////////////////////////
559 /// JitCacheFileHeader
560 //////////////////////////////////////////////////////////////////////////
561 struct JitCacheFileHeader
562 {
InitJitCacheFileHeader563 void Init(uint32_t llCRC,
564 uint32_t objCRC,
565 const std::string& moduleID,
566 const std::string& cpu,
567 uint32_t optLevel,
568 uint64_t objSize)
569 {
570 m_objSize = objSize;
571 m_llCRC = llCRC;
572 m_objCRC = objCRC;
573 strncpy(m_ModuleID, moduleID.c_str(), JC_STR_MAX_LEN - 1);
574 m_ModuleID[JC_STR_MAX_LEN - 1] = 0;
575 strncpy(m_Cpu, cpu.c_str(), JC_STR_MAX_LEN - 1);
576 m_Cpu[JC_STR_MAX_LEN - 1] = 0;
577 m_optLevel = optLevel;
578 }
579
580
581 bool
IsValidJitCacheFileHeader582 IsValid(uint32_t llCRC, const std::string& moduleID, const std::string& cpu, uint32_t optLevel)
583 {
584 if ((m_MagicNumber != JC_MAGIC_NUMBER) || (m_llCRC != llCRC) ||
585 (m_platformKey != JC_PLATFORM_KEY) || (m_optLevel != optLevel))
586 {
587 return false;
588 }
589
590 m_ModuleID[JC_STR_MAX_LEN - 1] = 0;
591 if (strncmp(moduleID.c_str(), m_ModuleID, JC_STR_MAX_LEN - 1))
592 {
593 return false;
594 }
595
596 m_Cpu[JC_STR_MAX_LEN - 1] = 0;
597 if (strncmp(cpu.c_str(), m_Cpu, JC_STR_MAX_LEN - 1))
598 {
599 return false;
600 }
601
602 return true;
603 }
604
GetObjectSizeJitCacheFileHeader605 uint64_t GetObjectSize() const { return m_objSize; }
GetObjectCRCJitCacheFileHeader606 uint64_t GetObjectCRC() const { return m_objCRC; }
607
608 private:
609 static const uint64_t JC_MAGIC_NUMBER = 0xfedcba9876543210ULL + 7;
610 static const size_t JC_STR_MAX_LEN = 32;
611 static const uint32_t JC_PLATFORM_KEY = (LLVM_VERSION_MAJOR << 24) |
612 (LLVM_VERSION_MINOR << 16) | (LLVM_VERSION_PATCH << 8) |
613 ((sizeof(void*) > sizeof(uint32_t)) ? 1 : 0);
614
615 uint64_t m_MagicNumber = JC_MAGIC_NUMBER;
616 uint64_t m_objSize = 0;
617 uint32_t m_llCRC = 0;
618 uint32_t m_platformKey = JC_PLATFORM_KEY;
619 uint32_t m_objCRC = 0;
620 uint32_t m_optLevel = 0;
621 char m_ModuleID[JC_STR_MAX_LEN] = {};
622 char m_Cpu[JC_STR_MAX_LEN] = {};
623 };
624
ComputeModuleCRC(const llvm::Module * M)625 static inline uint32_t ComputeModuleCRC(const llvm::Module* M)
626 {
627 std::string bitcodeBuffer;
628 raw_string_ostream bitcodeStream(bitcodeBuffer);
629
630 #if LLVM_VERSION_MAJOR >= 7
631 llvm::WriteBitcodeToFile(*M, bitcodeStream);
632 #else
633 llvm::WriteBitcodeToFile(M, bitcodeStream);
634 #endif
635 // M->print(bitcodeStream, nullptr, false);
636
637 bitcodeStream.flush();
638
639 return ComputeCRC(0, bitcodeBuffer.data(), bitcodeBuffer.size());
640 }
641
642 /// constructor
JitCache()643 JitCache::JitCache()
644 {
645 #if defined(__APPLE__) || defined(FORCE_LINUX) || defined(__linux__) || defined(__gnu_linux__)
646 if (strncmp(KNOB_JIT_CACHE_DIR.c_str(), "~/", 2) == 0)
647 {
648 char* homedir;
649 if (!(homedir = getenv("HOME")))
650 {
651 homedir = getpwuid(getuid())->pw_dir;
652 }
653 mCacheDir = homedir;
654 mCacheDir += (KNOB_JIT_CACHE_DIR.c_str() + 1);
655 }
656 else
657 #endif
658 {
659 mCacheDir = KNOB_JIT_CACHE_DIR;
660 }
661
662 // Create cache dir at startup to allow jitter to write debug.ll files
663 // to that directory.
664 if (!llvm::sys::fs::exists(mCacheDir.str()) &&
665 llvm::sys::fs::create_directories(mCacheDir.str()))
666 {
667 SWR_INVALID("Unable to create directory: %s", mCacheDir.c_str());
668 }
669
670 }
671
ExecUnhookedProcess(const std::string & CmdLine,std::string * pStdOut,std::string * pStdErr)672 int ExecUnhookedProcess(const std::string& CmdLine, std::string* pStdOut, std::string* pStdErr)
673 {
674
675 return ExecCmd(CmdLine, nullptr, pStdOut, pStdErr);
676 }
677
678 /// Calculate actual directory where module will be cached.
679 /// This is always a subdirectory of mCacheDir. Full absolute
680 /// path name will be stored in mCurrentModuleCacheDir
CalcModuleCacheDir()681 void JitCache::CalcModuleCacheDir()
682 {
683 mModuleCacheDir.clear();
684
685 llvm::SmallString<MAX_PATH> moduleDir = mCacheDir;
686
687 // Create 4 levels of directory hierarchy based on CRC, 256 entries each
688 uint8_t* pCRC = (uint8_t*)&mCurrentModuleCRC;
689 for (uint32_t i = 0; i < 4; ++i)
690 {
691 llvm::sys::path::append(moduleDir, std::to_string((int)pCRC[i]));
692 }
693
694 mModuleCacheDir = moduleDir;
695 }
696
697 /// notifyObjectCompiled - Provides a pointer to compiled code for Module M.
notifyObjectCompiled(const llvm::Module * M,llvm::MemoryBufferRef Obj)698 void JitCache::notifyObjectCompiled(const llvm::Module* M, llvm::MemoryBufferRef Obj)
699 {
700 const std::string& moduleID = M->getModuleIdentifier();
701 if (!moduleID.length())
702 {
703 return;
704 }
705
706 if (!mModuleCacheDir.size())
707 {
708 SWR_INVALID("Unset module cache directory");
709 return;
710 }
711
712 if (!llvm::sys::fs::exists(mModuleCacheDir.str()) &&
713 llvm::sys::fs::create_directories(mModuleCacheDir.str()))
714 {
715 SWR_INVALID("Unable to create directory: %s", mModuleCacheDir.c_str());
716 return;
717 }
718
719 JitCacheFileHeader header;
720
721 llvm::SmallString<MAX_PATH> filePath = mModuleCacheDir;
722 llvm::sys::path::append(filePath, moduleID);
723
724 llvm::SmallString<MAX_PATH> objPath = filePath;
725 objPath += JIT_OBJ_EXT;
726
727 {
728 std::error_code err;
729 llvm::raw_fd_ostream fileObj(objPath.c_str(), err, llvm::sys::fs::F_None);
730 fileObj << Obj.getBuffer();
731 fileObj.flush();
732 }
733
734
735 {
736 std::error_code err;
737 llvm::raw_fd_ostream fileObj(filePath.c_str(), err, llvm::sys::fs::F_None);
738
739 uint32_t objcrc = ComputeCRC(0, Obj.getBufferStart(), Obj.getBufferSize());
740
741 header.Init(mCurrentModuleCRC, objcrc, moduleID, mCpu, mOptLevel, Obj.getBufferSize());
742
743 fileObj.write((const char*)&header, sizeof(header));
744 fileObj.flush();
745 }
746 }
747
748 /// Returns a pointer to a newly allocated MemoryBuffer that contains the
749 /// object which corresponds with Module M, or 0 if an object is not
750 /// available.
getObject(const llvm::Module * M)751 std::unique_ptr<llvm::MemoryBuffer> JitCache::getObject(const llvm::Module* M)
752 {
753 const std::string& moduleID = M->getModuleIdentifier();
754 mCurrentModuleCRC = ComputeModuleCRC(M);
755
756 if (!moduleID.length())
757 {
758 return nullptr;
759 }
760
761 CalcModuleCacheDir();
762
763 if (!llvm::sys::fs::exists(mModuleCacheDir))
764 {
765 return nullptr;
766 }
767
768 llvm::SmallString<MAX_PATH> filePath = mModuleCacheDir;
769 llvm::sys::path::append(filePath, moduleID);
770
771 llvm::SmallString<MAX_PATH> objFilePath = filePath;
772 objFilePath += JIT_OBJ_EXT;
773
774 FILE* fpObjIn = nullptr;
775 FILE* fpIn = fopen(filePath.c_str(), "rb");
776 if (!fpIn)
777 {
778 return nullptr;
779 }
780
781 std::unique_ptr<llvm::MemoryBuffer> pBuf = nullptr;
782 do
783 {
784 JitCacheFileHeader header;
785 if (!fread(&header, sizeof(header), 1, fpIn))
786 {
787 break;
788 }
789
790 if (!header.IsValid(mCurrentModuleCRC, moduleID, mCpu, mOptLevel))
791 {
792 break;
793 }
794
795 fpObjIn = fopen(objFilePath.c_str(), "rb");
796 if (!fpObjIn)
797 {
798 break;
799 }
800
801 #if LLVM_VERSION_MAJOR < 6
802 pBuf = llvm::MemoryBuffer::getNewUninitMemBuffer(size_t(header.GetObjectSize()));
803 #else
804 pBuf = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(size_t(header.GetObjectSize()));
805 #endif
806 if (!fread(const_cast<char*>(pBuf->getBufferStart()), header.GetObjectSize(), 1, fpObjIn))
807 {
808 pBuf = nullptr;
809 break;
810 }
811
812 if (header.GetObjectCRC() != ComputeCRC(0, pBuf->getBufferStart(), pBuf->getBufferSize()))
813 {
814 SWR_TRACE("Invalid object cache file, ignoring: %s", filePath.c_str());
815 pBuf = nullptr;
816 break;
817 }
818
819 } while (0);
820
821 fclose(fpIn);
822
823 if (fpObjIn)
824 {
825 fclose(fpObjIn);
826 }
827
828
829 return pBuf;
830 }
831
emitInstructionAnnot(const llvm::Instruction * pInst,llvm::formatted_raw_ostream & OS)832 void InterleaveAssemblyAnnotater::emitInstructionAnnot(const llvm::Instruction* pInst,
833 llvm::formatted_raw_ostream& OS)
834 {
835 auto dbgLoc = pInst->getDebugLoc();
836 if (dbgLoc)
837 {
838 unsigned int line = dbgLoc.getLine();
839 if (line != mCurrentLineNo)
840 {
841 if (line > 0 && line <= mAssembly.size())
842 {
843 // HACK: here we assume that OS is a formatted_raw_ostream(ods())
844 // and modify the color accordingly. We can't do the color
845 // modification on OS because formatted_raw_ostream strips
846 // the color information. The only way to fix this behavior
847 // is to patch LLVM.
848 OS << "\n; " << line << ": " << mAssembly[line - 1] << "\n";
849 }
850 mCurrentLineNo = line;
851 }
852 }
853 }
854