1 // 2 // Copyright 2012-2016 Francisco Jerez 3 // Copyright 2012-2016 Advanced Micro Devices, Inc. 4 // Copyright 2014-2016 Jan Vesely 5 // Copyright 2014-2015 Serge Martin 6 // Copyright 2015 Zoltan Gilian 7 // 8 // Permission is hereby granted, free of charge, to any person obtaining a 9 // copy of this software and associated documentation files (the "Software"), 10 // to deal in the Software without restriction, including without limitation 11 // the rights to use, copy, modify, merge, publish, distribute, sublicense, 12 // and/or sell copies of the Software, and to permit persons to whom the 13 // Software is furnished to do so, subject to the following conditions: 14 // 15 // The above copyright notice and this permission notice shall be included in 16 // all copies or substantial portions of the Software. 17 // 18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 22 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 23 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 // OTHER DEALINGS IN THE SOFTWARE. 25 // 26 27 #include <llvm/IR/DiagnosticPrinter.h> 28 #include <llvm/IR/DiagnosticInfo.h> 29 #include <llvm/IR/LLVMContext.h> 30 #include <llvm/IR/Module.h> 31 #include <llvm/Support/raw_ostream.h> 32 #include <llvm/Transforms/IPO/Internalize.h> 33 #include <llvm-c/Target.h> 34 #include <llvm-c/TargetMachine.h> 35 #include <llvm-c/Transforms/PassBuilder.h> 36 #include <llvm/Support/CBindingWrapping.h> 37 #include <clang/CodeGen/CodeGenAction.h> 38 #include <clang/Lex/PreprocessorOptions.h> 39 #include <clang/Frontend/TextDiagnosticBuffer.h> 40 #include <clang/Frontend/TextDiagnosticPrinter.h> 41 #include <clang/Basic/TargetInfo.h> 42 43 #if LLVM_VERSION_MAJOR >= 20 44 #include <llvm/Support/VirtualFileSystem.h> 45 #endif 46 47 // We need to include internal headers last, because the internal headers 48 // include CL headers which have #define's like: 49 // 50 //#define cl_khr_gl_sharing 1 51 //#define cl_khr_icd 1 52 // 53 // Which will break the compilation of clang/Basic/OpenCLOptions.h 54 55 #include "core/error.hpp" 56 #include "llvm/codegen.hpp" 57 #include "llvm/compat.hpp" 58 #include "llvm/invocation.hpp" 59 #include "llvm/metadata.hpp" 60 #include "llvm/util.hpp" 61 #include "util/algorithm.hpp" 62 63 64 using clover::binary; 65 using clover::device; 66 using clover::build_error; 67 using clover::invalid_build_options_error; 68 using clover::map; 69 using clover::header_map; 70 using namespace clover::llvm; 71 72 using ::llvm::Function; 73 using ::llvm::LLVMContext; 74 using ::llvm::Module; 75 using ::llvm::raw_string_ostream; 76 77 namespace { 78 79 static const cl_version ANY_VERSION = CL_MAKE_VERSION(9, 9, 9); 80 const cl_version cl_versions[] = { 81 CL_MAKE_VERSION(1, 1, 0), 82 CL_MAKE_VERSION(1, 2, 0), 83 CL_MAKE_VERSION(2, 0, 0), 84 CL_MAKE_VERSION(2, 1, 0), 85 CL_MAKE_VERSION(2, 2, 0), 86 CL_MAKE_VERSION(3, 0, 0), 87 }; 88 89 struct clc_version_lang_std { 90 cl_version version_number; // CLC Version 91 clang::LangStandard::Kind clc_lang_standard; 92 }; 93 94 const clc_version_lang_std cl_version_lang_stds[] = { 95 { CL_MAKE_VERSION(1, 0, 0), clang::LangStandard::lang_opencl10}, 96 { CL_MAKE_VERSION(1, 1, 0), clang::LangStandard::lang_opencl11}, 97 { CL_MAKE_VERSION(1, 2, 0), clang::LangStandard::lang_opencl12}, 98 { CL_MAKE_VERSION(2, 0, 0), clang::LangStandard::lang_opencl20}, 99 #if LLVM_VERSION_MAJOR >= 12 100 { CL_MAKE_VERSION(3, 0, 0), clang::LangStandard::lang_opencl30}, 101 #endif 102 }; 103 104 bool are_equal(cl_version_khr version1,cl_version_khr version2,bool ignore_patch_version=false)105 are_equal(cl_version_khr version1, cl_version_khr version2, 106 bool ignore_patch_version = false) { 107 if (ignore_patch_version) { 108 version1 &= ~CL_VERSION_PATCH_MASK_KHR; 109 version2 &= ~CL_VERSION_PATCH_MASK_KHR; 110 } 111 return version1 == version2; 112 } 113 114 void init_targets()115 init_targets() { 116 static bool targets_initialized = false; 117 if (!targets_initialized) { 118 LLVMInitializeAllTargets(); 119 LLVMInitializeAllTargetInfos(); 120 LLVMInitializeAllTargetMCs(); 121 LLVMInitializeAllAsmParsers(); 122 LLVMInitializeAllAsmPrinters(); 123 targets_initialized = true; 124 } 125 } 126 127 void 128 #if LLVM_VERSION_MAJOR >= 19 diagnostic_handler(const::llvm::DiagnosticInfo * di,void * data)129 diagnostic_handler(const ::llvm::DiagnosticInfo *di, void *data) { 130 if (di->getSeverity() == ::llvm::DS_Error) { 131 #else 132 diagnostic_handler(const ::llvm::DiagnosticInfo &di, void *data) { 133 if (di.getSeverity() == ::llvm::DS_Error) { 134 #endif 135 raw_string_ostream os { *reinterpret_cast<std::string *>(data) }; 136 ::llvm::DiagnosticPrinterRawOStream printer { os }; 137 #if LLVM_VERSION_MAJOR >= 19 138 di->print(printer); 139 #else 140 di.print(printer); 141 #endif 142 throw build_error(); 143 } 144 } 145 146 std::unique_ptr<LLVMContext> 147 create_context(std::string &r_log) { 148 init_targets(); 149 std::unique_ptr<LLVMContext> ctx { new LLVMContext }; 150 151 ctx->setDiagnosticHandlerCallBack(diagnostic_handler, &r_log); 152 return ctx; 153 } 154 155 const struct clc_version_lang_std& 156 get_cl_lang_standard(unsigned requested, unsigned max = ANY_VERSION) { 157 for (const struct clc_version_lang_std &version : cl_version_lang_stds) { 158 if (version.version_number == max || 159 version.version_number == requested) { 160 return version; 161 } 162 } 163 throw build_error("Unknown/Unsupported language version"); 164 } 165 166 const cl_version 167 get_cl_version(cl_version requested, 168 cl_version max = ANY_VERSION) { 169 for (const auto &version : cl_versions) { 170 if (are_equal(version, max, true) || 171 are_equal(version, requested, true)) { 172 return version; 173 } 174 } 175 throw build_error("Unknown/Unsupported language version"); 176 } 177 178 clang::LangStandard::Kind 179 get_lang_standard_from_version(const cl_version input_version, 180 bool is_build_opt = false) { 181 182 //Per CL 2.0 spec, section 5.8.4.5: 183 // If it's an option, use the value directly. 184 // If it's a device version, clamp to max 1.x version, a.k.a. 1.2 185 const cl_version version = 186 get_cl_version(input_version, is_build_opt ? ANY_VERSION : 120); 187 188 const struct clc_version_lang_std standard = 189 get_cl_lang_standard(version); 190 191 return standard.clc_lang_standard; 192 } 193 194 clang::LangStandard::Kind 195 get_language_version(const std::vector<std::string> &opts, 196 const cl_version device_version) { 197 198 const std::string search = "-cl-std=CL"; 199 200 for (auto &opt: opts) { 201 auto pos = opt.find(search); 202 if (pos == 0){ 203 std::stringstream ver_str(opt.substr(pos + search.size())); 204 unsigned int ver_major = 0; 205 char separator = '\0'; 206 unsigned int ver_minor = 0; 207 ver_str >> ver_major >> separator >> ver_minor; 208 if (ver_str.fail() || ver_str.bad() || !ver_str.eof() || 209 separator != '.') { 210 throw build_error(); 211 } 212 const auto ver = CL_MAKE_VERSION_KHR(ver_major, ver_minor, 0); 213 const auto device_ver = get_cl_version(device_version); 214 const auto requested = get_cl_version(ver); 215 if (requested > device_ver) { 216 throw build_error(); 217 } 218 return get_lang_standard_from_version(ver, true); 219 } 220 } 221 222 return get_lang_standard_from_version(device_version); 223 } 224 225 std::unique_ptr<clang::CompilerInstance> 226 create_compiler_instance(const device &dev, const std::string& ir_target, 227 const std::vector<std::string> &opts, 228 std::string &r_log) { 229 std::unique_ptr<clang::CompilerInstance> c { new clang::CompilerInstance }; 230 clang::TextDiagnosticBuffer *diag_buffer = new clang::TextDiagnosticBuffer; 231 clang::DiagnosticsEngine diag { new clang::DiagnosticIDs, 232 new clang::DiagnosticOptions, diag_buffer }; 233 234 // Parse the compiler options. A file name should be present at the end 235 // and must have the .cl extension in order for the CompilerInvocation 236 // class to recognize it as an OpenCL source file. 237 #if LLVM_VERSION_MAJOR >= 12 238 std::vector<const char *> copts; 239 #if LLVM_VERSION_MAJOR == 15 || LLVM_VERSION_MAJOR == 16 240 // Before LLVM commit 702d5de4 opaque pointers were supported but not enabled 241 // by default when building LLVM. They were made default in commit 702d5de4. 242 // LLVM commit d69e9f9d introduced -opaque-pointers/-no-opaque-pointers cc1 243 // options to enable or disable them whatever the LLVM default is. 244 245 // Those two commits follow llvmorg-15-init and precede llvmorg-15.0.0-rc1 tags. 246 247 // Since LLVM commit d785a8ea, the CLANG_ENABLE_OPAQUE_POINTERS build option of 248 // LLVM is removed, meaning there is no way to build LLVM with opaque pointers 249 // enabled by default. 250 // It was said at the time it was still possible to explicitly disable opaque 251 // pointers via cc1 -no-opaque-pointers option, but it is known a later commit 252 // broke backward compatibility provided by -no-opaque-pointers as verified with 253 // arbitrary commit d7d586e5, so there is no way to use opaque pointers starting 254 // with LLVM 16. 255 256 // Those two commits follow llvmorg-16-init and precede llvmorg-16.0.0-rc1 tags. 257 258 // Since Mesa commit 977dbfc9 opaque pointers are properly implemented in Clover 259 // and used. 260 261 // If we don't pass -opaque-pointers to Clang on LLVM versions supporting opaque 262 // pointers but disabling them by default, there will be an API mismatch between 263 // Mesa and LLVM and Clover will not work. 264 copts.push_back("-opaque-pointers"); 265 #endif 266 for (auto &opt : opts) { 267 if (opt == "-cl-denorms-are-zero") 268 copts.push_back("-fdenormal-fp-math=positive-zero"); 269 else 270 copts.push_back(opt.c_str()); 271 } 272 #else 273 const std::vector<const char *> copts = 274 map(std::mem_fn(&std::string::c_str), opts); 275 #endif 276 277 const target &target = ir_target; 278 const cl_version device_clc_version = dev.device_clc_version(); 279 280 if (!compat::create_compiler_invocation_from_args( 281 c->getInvocation(), copts, diag)) 282 throw invalid_build_options_error(); 283 284 diag_buffer->FlushDiagnostics(diag); 285 if (diag.hasErrorOccurred()) 286 throw invalid_build_options_error(); 287 288 c->getTargetOpts().CPU = target.cpu; 289 c->getTargetOpts().Triple = target.triple; 290 c->getLangOpts().NoBuiltin = true; 291 292 #if LLVM_VERSION_MAJOR >= 13 293 c->getTargetOpts().OpenCLExtensionsAsWritten.push_back("-__opencl_c_generic_address_space"); 294 c->getTargetOpts().OpenCLExtensionsAsWritten.push_back("-__opencl_c_pipes"); 295 c->getTargetOpts().OpenCLExtensionsAsWritten.push_back("-__opencl_c_device_enqueue"); 296 c->getTargetOpts().OpenCLExtensionsAsWritten.push_back("-__opencl_c_program_scope_global_variables"); 297 c->getTargetOpts().OpenCLExtensionsAsWritten.push_back("-__opencl_c_subgroups"); 298 c->getTargetOpts().OpenCLExtensionsAsWritten.push_back("-__opencl_c_work_group_collective_functions"); 299 c->getTargetOpts().OpenCLExtensionsAsWritten.push_back("-__opencl_c_atomic_scope_device"); 300 c->getTargetOpts().OpenCLExtensionsAsWritten.push_back("-__opencl_c_atomic_order_seq_cst"); 301 #endif 302 303 // This is a workaround for a Clang bug which causes the number 304 // of warnings and errors to be printed to stderr. 305 // http://www.llvm.org/bugs/show_bug.cgi?id=19735 306 c->getDiagnosticOpts().ShowCarets = false; 307 308 compat::compiler_set_lang_defaults(c, compat::ik_opencl, 309 ::llvm::Triple(target.triple), 310 get_language_version(opts, device_clc_version)); 311 312 c->createDiagnostics( 313 #if LLVM_VERSION_MAJOR >= 20 314 *llvm::vfs::getRealFileSystem(), 315 #endif 316 new clang::TextDiagnosticPrinter( 317 *new raw_string_ostream(r_log), 318 &c->getDiagnosticOpts(), true)); 319 320 c->setTarget(clang::TargetInfo::CreateTargetInfo( 321 c->getDiagnostics(), c->getInvocation().TargetOpts)); 322 323 return c; 324 } 325 326 std::unique_ptr<Module> 327 compile(LLVMContext &ctx, clang::CompilerInstance &c, 328 const std::string &name, const std::string &source, 329 const header_map &headers, const device &dev, 330 const std::string &opts, bool use_libclc, std::string &r_log) { 331 c.getFrontendOpts().ProgramAction = clang::frontend::EmitLLVMOnly; 332 c.getHeaderSearchOpts().UseBuiltinIncludes = true; 333 c.getHeaderSearchOpts().UseStandardSystemIncludes = true; 334 c.getHeaderSearchOpts().ResourceDir = CLANG_RESOURCE_DIR; 335 336 if (use_libclc) { 337 // Add libclc generic search path 338 c.getHeaderSearchOpts().AddPath(LIBCLC_INCLUDEDIR, 339 clang::frontend::Angled, 340 false, false); 341 342 // Add libclc include 343 c.getPreprocessorOpts().Includes.push_back("clc/clc.h"); 344 } else { 345 // Add opencl-c generic search path 346 c.getHeaderSearchOpts().AddPath(CLANG_RESOURCE_DIR, 347 clang::frontend::Angled, 348 false, false); 349 350 // Add opencl include 351 c.getPreprocessorOpts().Includes.push_back("opencl-c.h"); 352 } 353 354 // Add definition for the OpenCL version 355 const auto dev_version = dev.device_version(); 356 c.getPreprocessorOpts().addMacroDef("__OPENCL_VERSION__=" + 357 std::to_string(CL_VERSION_MAJOR_KHR(dev_version)) + 358 std::to_string(CL_VERSION_MINOR_KHR(dev_version)) + "0"); 359 360 if (CL_VERSION_MAJOR(dev.version) >= 3) { 361 const auto features = dev.opencl_c_features(); 362 for (const auto &feature : features) 363 c.getPreprocessorOpts().addMacroDef(feature.name); 364 } 365 366 // clc.h requires that this macro be defined: 367 c.getPreprocessorOpts().addMacroDef("cl_clang_storage_class_specifiers"); 368 c.getPreprocessorOpts().addRemappedFile( 369 name, ::llvm::MemoryBuffer::getMemBuffer(source).release()); 370 371 if (headers.size()) { 372 const std::string tmp_header_path = "/tmp/clover/"; 373 374 c.getHeaderSearchOpts().AddPath(tmp_header_path, 375 clang::frontend::Angled, 376 false, false); 377 378 for (const auto &header : headers) 379 c.getPreprocessorOpts().addRemappedFile( 380 tmp_header_path + header.first, 381 ::llvm::MemoryBuffer::getMemBuffer(header.second).release()); 382 } 383 384 // Tell clang to link this file before performing any 385 // optimizations. This is required so that we can replace calls 386 // to the OpenCL C barrier() builtin with calls to target 387 // intrinsics that have the noduplicate attribute. This 388 // attribute will prevent Clang from creating illegal uses of 389 // barrier() (e.g. Moving barrier() inside a conditional that is 390 // no executed by all threads) during its optimizaton passes. 391 if (use_libclc) { 392 clang::CodeGenOptions::BitcodeFileToLink F; 393 394 F.Filename = LIBCLC_LIBEXECDIR + dev.ir_target() + ".bc"; 395 F.PropagateAttrs = true; 396 F.LinkFlags = ::llvm::Linker::Flags::None; 397 c.getCodeGenOpts().LinkBitcodeFiles.emplace_back(F); 398 } 399 400 // undefine __IMAGE_SUPPORT__ for device without image support 401 if (!dev.image_support()) 402 c.getPreprocessorOpts().addMacroUndef("__IMAGE_SUPPORT__"); 403 404 // Compile the code 405 clang::EmitLLVMOnlyAction act(&ctx); 406 if (!c.ExecuteAction(act)) 407 throw build_error(); 408 409 return act.takeModule(); 410 } 411 } 412 413 binary 414 clover::llvm::compile_program(const std::string &source, 415 const header_map &headers, 416 const device &dev, 417 const std::string &opts, 418 std::string &r_log) { 419 if (has_flag(debug::clc)) 420 debug::log(".cl", "// Options: " + opts + '\n' + source); 421 422 auto ctx = create_context(r_log); 423 auto c = create_compiler_instance(dev, dev.ir_target(), 424 tokenize(opts + " input.cl"), r_log); 425 auto mod = compile(*ctx, *c, "input.cl", source, headers, dev, opts, true, 426 r_log); 427 428 if (has_flag(debug::llvm)) 429 debug::log(".ll", print_module_bitcode(*mod)); 430 431 return build_module_library(*mod, binary::section::text_intermediate); 432 } 433 434 namespace { 435 void 436 optimize(Module &mod, 437 const std::string& ir_target, 438 unsigned optimization_level, 439 bool internalize_symbols) { 440 // By default, the function internalizer pass will look for a function 441 // called "main" and then mark all other functions as internal. Marking 442 // functions as internal enables the optimizer to perform optimizations 443 // like function inlining and global dead-code elimination. 444 // 445 // When there is no "main" function in a binary, the internalize pass will 446 // treat the binary like a library, and it won't internalize any functions. 447 // Since there is no "main" function in our kernels, we need to tell 448 // the internalizer pass that this binary is not a library by passing a 449 // list of kernel functions to the internalizer. The internalizer will 450 // treat the functions in the list as "main" functions and internalize 451 // all of the other functions. 452 if (internalize_symbols) { 453 std::vector<std::string> names = 454 map(std::mem_fn(&Function::getName), get_kernels(mod)); 455 internalizeModule(mod, 456 [=](const ::llvm::GlobalValue &gv) { 457 return std::find(names.begin(), names.end(), 458 gv.getName()) != names.end(); 459 }); 460 } 461 462 463 const char *opt_str = NULL; 464 LLVMCodeGenOptLevel level; 465 switch (optimization_level) { 466 case 0: 467 default: 468 opt_str = "default<O0>"; 469 level = LLVMCodeGenLevelNone; 470 break; 471 case 1: 472 opt_str = "default<O1>"; 473 level = LLVMCodeGenLevelLess; 474 break; 475 case 2: 476 opt_str = "default<O2>"; 477 level = LLVMCodeGenLevelDefault; 478 break; 479 case 3: 480 opt_str = "default<O3>"; 481 level = LLVMCodeGenLevelAggressive; 482 break; 483 } 484 485 const target &target = ir_target; 486 LLVMTargetRef targ; 487 char *err_message; 488 489 if (LLVMGetTargetFromTriple(target.triple.c_str(), &targ, &err_message)) 490 return; 491 LLVMTargetMachineRef tm = 492 LLVMCreateTargetMachine(targ, target.triple.c_str(), 493 target.cpu.c_str(), "", level, 494 LLVMRelocDefault, LLVMCodeModelDefault); 495 496 if (!tm) 497 return; 498 LLVMPassBuilderOptionsRef opts = LLVMCreatePassBuilderOptions(); 499 LLVMRunPasses(wrap(&mod), opt_str, tm, opts); 500 501 LLVMDisposeTargetMachine(tm); 502 LLVMDisposePassBuilderOptions(opts); 503 } 504 505 std::unique_ptr<Module> 506 link(LLVMContext &ctx, const clang::CompilerInstance &c, 507 const std::vector<binary> &binaries, std::string &r_log) { 508 std::unique_ptr<Module> mod { new Module("link", ctx) }; 509 std::unique_ptr< ::llvm::Linker> linker { new ::llvm::Linker(*mod) }; 510 511 for (auto &b : binaries) { 512 if (linker->linkInModule(parse_module_library(b, ctx, r_log))) 513 throw build_error(); 514 } 515 516 return mod; 517 } 518 } 519 520 binary 521 clover::llvm::link_program(const std::vector<binary> &binaries, 522 const device &dev, const std::string &opts, 523 std::string &r_log) { 524 std::vector<std::string> options = tokenize(opts + " input.cl"); 525 const bool create_library = count("-create-library", options); 526 erase_if(equals("-create-library"), options); 527 528 auto ctx = create_context(r_log); 529 auto c = create_compiler_instance(dev, dev.ir_target(), options, r_log); 530 auto mod = link(*ctx, *c, binaries, r_log); 531 532 optimize(*mod, dev.ir_target(), c->getCodeGenOpts().OptimizationLevel, !create_library); 533 534 static std::atomic_uint seq(0); 535 const std::string id = "." + mod->getModuleIdentifier() + "-" + 536 std::to_string(seq++); 537 538 if (has_flag(debug::llvm)) 539 debug::log(id + ".ll", print_module_bitcode(*mod)); 540 541 if (create_library) { 542 return build_module_library(*mod, binary::section::text_library); 543 544 } else if (dev.ir_format() == PIPE_SHADER_IR_NATIVE) { 545 if (has_flag(debug::native)) 546 debug::log(id + ".asm", print_module_native(*mod, dev.ir_target())); 547 548 return build_module_native(*mod, dev.ir_target(), *c, r_log); 549 550 } else { 551 unreachable("Unsupported IR."); 552 } 553 } 554