1 /*
2 * Copyright (c) 2023-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "compiler/aot/compiled_method.h"
17 #include "compiler/code_info/code_info.h"
18 #include "compiler/generated/pipeline_includes.h"
19 #include "events/events.h"
20 #include "optimizer/ir/graph.h"
21 #include "optimizer/ir/graph_checker.h"
22 #include "optimizer/ir_builder/ir_builder.h"
23 #include "optimizer/optimizations/balance_expressions.h"
24 #include "optimizer/optimizations/branch_elimination.h"
25 #include "optimizer/optimizations/checks_elimination.h"
26 #include "optimizer/optimizations/code_sink.h"
27 #include "optimizer/optimizations/escape.h"
28 #include "optimizer/optimizations/if_merging.h"
29 #include "optimizer/optimizations/inlining.h"
30 #include "optimizer/optimizations/licm.h"
31 #include "optimizer/optimizations/licm_conditions.h"
32 #include "optimizer/optimizations/lowering.h"
33 #include "optimizer/optimizations/loop_idioms.h"
34 #include "optimizer/optimizations/loop_peeling.h"
35 #include "optimizer/optimizations/loop_unroll.h"
36 #include "optimizer/optimizations/loop_unswitch.h"
37 #include "optimizer/optimizations/lse.h"
38 #include "optimizer/optimizations/memory_barriers.h"
39 #include "optimizer/optimizations/object_type_check_elimination.h"
40 #include "optimizer/optimizations/optimize_string_concat.h"
41 #include "optimizer/optimizations/peepholes.h"
42 #include "optimizer/optimizations/redundant_loop_elimination.h"
43 #include "optimizer/optimizations/reserve_string_builder_buffer.h"
44 #include "optimizer/optimizations/savestate_optimization.h"
45 #include "optimizer/optimizations/simplify_string_builder.h"
46 #include "optimizer/optimizations/try_catch_resolving.h"
47 #include "optimizer/optimizations/vn.h"
48 #include "optimizer/analysis/monitor_analysis.h"
49 #include "optimizer/optimizations/cleanup.h"
50 #include "runtime/include/method.h"
51 #include "runtime/include/thread.h"
52 #include "compiler_options.h"
53
54 #include "llvm_aot_compiler.h"
55 #include "llvm_logger.h"
56 #include "llvm_options.h"
57 #include "mir_compiler.h"
58 #include "target_machine_builder.h"
59
60 #include "lowering/llvm_ir_constructor.h"
61 #include "object_code/code_info_producer.h"
62 #include "transforms/passes/ark_frame_lowering/frame_lowering.h"
63
64 #include <llvm/IR/Constants.h>
65 #include <llvm/IR/Instructions.h>
66 #include <llvm/CodeGen/Passes.h>
67 #include <llvm/CodeGen/MachineFunctionPass.h>
68 // Suppress warning about forward Pass declaration defined in another namespace
69 #include <llvm/Pass.h>
70 #include <llvm/Support/Debug.h>
71
72 /* Be careful this file also includes elf.h, which has a lot of defines */
73 #include "aot/aot_builder/llvm_aot_builder.h"
74
75 #define DEBUG_TYPE "llvm-aot-compiler"
76
77 static constexpr unsigned LIMIT_MULTIPLIER = 2U;
78
79 static constexpr int32_t MAX_DEPTH = 32;
80
81 namespace {
82
83 class ScopedCompilerThread {
84 public:
ScopedCompilerThread(ark::compiler::RuntimeInterface::ThreadPtr parent,ark::compiler::RuntimeInterface * runtimeInterface)85 ScopedCompilerThread(ark::compiler::RuntimeInterface::ThreadPtr parent,
86 ark::compiler::RuntimeInterface *runtimeInterface)
87 : runtimeInterface_ {runtimeInterface}
88 {
89 ASSERT(parent != nullptr);
90 old_ = runtimeInterface->GetCurrentThread();
91 runtimeInterface->SetCurrentThread(parent);
92 compiler_ = runtimeInterface->CreateCompilerThread();
93 runtimeInterface->SetCurrentThread(compiler_);
94 }
95
96 ScopedCompilerThread(ScopedCompilerThread &) = delete;
97 ScopedCompilerThread &operator=(ScopedCompilerThread &) = delete;
98 ScopedCompilerThread(ScopedCompilerThread &&) = delete;
99 ScopedCompilerThread &operator=(ScopedCompilerThread &&) = delete;
100
~ScopedCompilerThread()101 ~ScopedCompilerThread()
102 {
103 ASSERT(runtimeInterface_->GetCurrentThread() == compiler_);
104 runtimeInterface_->DestroyCompilerThread(compiler_);
105 runtimeInterface_->SetCurrentThread(old_);
106 }
107
108 private:
109 ark::compiler::RuntimeInterface::ThreadPtr old_ {nullptr};
110 ark::compiler::RuntimeInterface::ThreadPtr compiler_ {nullptr};
111 ark::compiler::RuntimeInterface *runtimeInterface_;
112 };
113
MarkAsInlineObject(llvm::GlobalObject * globalObject)114 void MarkAsInlineObject(llvm::GlobalObject *globalObject)
115 {
116 globalObject->addMetadata(ark::llvmbackend::LLVMArkInterface::FUNCTION_MD_INLINE_MODULE,
117 *llvm::MDNode::get(globalObject->getContext(), {}));
118 }
119
UnmarkAsInlineObject(llvm::GlobalObject * globalObject)120 void UnmarkAsInlineObject(llvm::GlobalObject *globalObject)
121 {
122 ASSERT_PRINT(globalObject->hasMetadata(ark::llvmbackend::LLVMArkInterface::FUNCTION_MD_INLINE_MODULE),
123 "Must be marked to unmark");
124
125 globalObject->setMetadata(ark::llvmbackend::LLVMArkInterface::FUNCTION_MD_INLINE_MODULE, nullptr);
126 }
127
GetFrameSlotSize(ark::Arch arch)128 constexpr unsigned GetFrameSlotSize(ark::Arch arch)
129 {
130 constexpr size_t SPILLS = 0;
131 ark::CFrameLayout layout {arch, SPILLS};
132 return layout.GetSlotSize();
133 }
134
MonitorsCorrect(ark::compiler::Graph * graph,bool nonCatchOnly)135 bool MonitorsCorrect(ark::compiler::Graph *graph, bool nonCatchOnly)
136 {
137 if (!nonCatchOnly) {
138 return graph->RunPass<ark::compiler::MonitorAnalysis>();
139 }
140 graph->GetAnalysis<ark::compiler::MonitorAnalysis>().SetCheckNonCatchOnly(true);
141 bool ok = graph->RunPass<ark::compiler::MonitorAnalysis>();
142 graph->InvalidateAnalysis<ark::compiler::MonitorAnalysis>();
143 graph->GetAnalysis<ark::compiler::MonitorAnalysis>().SetCheckNonCatchOnly(false);
144 return ok;
145 }
146
147 } // namespace
148
149 namespace ark::llvmbackend {
150
151 class FixedCountSpreader : public Spreader {
152 public:
153 using MethodCount = uint64_t;
154 using ModuleFactory = std::function<std::shared_ptr<WrappedModule>(uint32_t moduleId)>;
155
FixedCountSpreader(compiler::RuntimeInterface * runtime,MethodCount limit,ModuleFactory moduleFactory)156 explicit FixedCountSpreader(compiler::RuntimeInterface *runtime, MethodCount limit, ModuleFactory moduleFactory)
157 : moduleFactory_(std::move(moduleFactory)), runtime_(runtime), limit_(limit)
158 {
159 // To avoid overflow in HardLimit()
160 ASSERT(limit_ < std::numeric_limits<MethodCount>::max() / LIMIT_MULTIPLIER);
161 ASSERT(limit_ > 0);
162 }
163
GetModuleForMethod(compiler::RuntimeInterface::MethodPtr method)164 std::shared_ptr<WrappedModule> GetModuleForMethod(compiler::RuntimeInterface::MethodPtr method) override
165 {
166 auto &module = modules_[method];
167 if (module != nullptr) {
168 return module;
169 }
170 ASSERT(!IsOrderViolated(method));
171 if (currentModule_ == nullptr) {
172 currentModule_ = moduleFactory_(moduleCount_++);
173 }
174 if (currentModuleSize_ + 1 >= limit_) {
175 bool sizeExceeds = currentModuleSize_ + 1 >= HardLimit();
176 if (sizeExceeds || runtime_->GetClass(method) != lastClass_) {
177 LLVM_LOG(DEBUG, INFRA) << "Creating new module, since "
178 << (sizeExceeds ? "the module's size would exceed HardLimit" : "class changed")
179 << ", class of current method = '" << runtime_->GetClassNameFromMethod(method)
180 << "', lastClass_ = '"
181 << (lastClass_ == nullptr ? "nullptr" : runtime_->GetClassName(lastClass_))
182 << "', limit_ = " << limit_ << ", HardLimit = " << HardLimit()
183 << ", currentModuleSize_ = " << currentModuleSize_;
184 currentModuleSize_ = 0;
185 currentModule_ = moduleFactory_(moduleCount_++);
186 moduleCount_++;
187 }
188 }
189
190 currentModuleSize_++;
191 lastClass_ = runtime_->GetClass(method);
192 module = currentModule_;
193 LLVM_LOG(DEBUG, INFRA) << "Getting module for graph (" << runtime_->GetMethodFullName(method, true) << "), "
194 << currentModuleSize_ << " / " << limit_ << ", HardLimit = " << HardLimit();
195
196 ASSERT(module != nullptr);
197 ASSERT(!module->IsCompiled());
198 ASSERT(currentModuleSize_ < HardLimit());
199 return module;
200 }
201
GetModules()202 std::unordered_set<std::shared_ptr<WrappedModule>> GetModules() override
203 {
204 std::unordered_set<std::shared_ptr<WrappedModule>> modules;
205 for (auto &entry : modules_) {
206 modules.insert(entry.second);
207 }
208 return modules;
209 }
210
211 private:
HardLimit()212 constexpr uint64_t HardLimit()
213 {
214 return limit_ * LIMIT_MULTIPLIER;
215 }
216 #ifndef NDEBUG
IsOrderViolated(compiler::RuntimeInterface::MethodPtr method)217 bool IsOrderViolated(compiler::RuntimeInterface::MethodPtr method)
218 {
219 auto clazz = runtime_->GetClass(method);
220 auto notSeen = seenClasses_.insert(clazz).second;
221 if (notSeen) {
222 return false;
223 }
224 return clazz != lastClass_;
225 }
226 #endif
227
228 private:
229 #ifndef NDEBUG
230 std::unordered_set<compiler::RuntimeInterface::ClassPtr> seenClasses_;
231 #endif
232 ModuleFactory moduleFactory_;
233 compiler::RuntimeInterface *runtime_;
234 const uint64_t limit_;
235 uint64_t currentModuleSize_ {0};
236 uint32_t moduleCount_ {0};
237 compiler::RuntimeInterface::ClassPtr lastClass_ {nullptr};
238 std::shared_ptr<WrappedModule> currentModule_ {nullptr};
239 std::unordered_map<compiler::RuntimeInterface::MethodPtr, std::shared_ptr<WrappedModule>> modules_;
240 };
241
TryAddGraph(compiler::Graph * graph)242 Expected<bool, std::string> LLVMAotCompiler::TryAddGraph(compiler::Graph *graph)
243 {
244 ASSERT(graph != nullptr && graph->GetArch() == GetArch() && graph->GetAotData()->GetUseCha());
245 ASSERT(graph->GetRuntime()->IsCompressedStringsEnabled() && graph->SupportManagedCode());
246
247 if (graph->IsDynamicMethod()) {
248 return false;
249 }
250
251 auto method = static_cast<Method *>(graph->GetMethod());
252 auto module = spreader_->GetModuleForMethod(method);
253 if (currentModule_ == nullptr) {
254 currentModule_ = module;
255 }
256 if (threadPool_ == nullptr && currentModule_ != module) {
257 // Compile finished module immediately before emitting the next one
258 CompileModule(*currentModule_);
259 }
260 if (threadPool_ != nullptr && currentModule_ != module) {
261 // Launch compilation on parallel worker
262 auto thread = runtime_->GetCurrentThread();
263 threadPool_->async(
264 [this, thread](auto it) -> void {
265 ScopedCompilerThread scopedCompilerThread {thread, runtime_};
266 CompileModule(*it);
267 {
268 std::lock_guard<decltype(mutex_)> lock {mutex_};
269 semaphore_++;
270 }
271 cv_.notify_all();
272 },
273 currentModule_);
274 // If all workers busy, wait here before emitting next module
275 {
276 std::unique_lock<decltype(mutex_)> lock {mutex_};
277 cv_.wait(lock, [&]() { return semaphore_ > 0; });
278 --semaphore_;
279 }
280 }
281 currentModule_ = module;
282
283 // Proceed with the Graph
284 auto result = AddGraphToModule(graph, *module, AddGraphMode::PRIMARY_FUNCTION);
285 if (result.HasValue() && result.Value()) {
286 module->AddMethod(method);
287 methods_.push_back(method);
288 if (!compiler::g_options.IsCompilerNonOptimizing()) {
289 AddInlineFunctionsByDepth(*module, graph, 0);
290 }
291 }
292 return result;
293 }
294
RunArkPasses(compiler::Graph * graph)295 bool LLVMAotCompiler::RunArkPasses(compiler::Graph *graph)
296 {
297 auto llvmPreOpt = g_options.GetLlvmPreOpt();
298 ASSERT(llvmPreOpt <= 2U);
299 graph->RunPass<compiler::Cleanup>(false);
300 if (!compiler::g_options.IsCompilerNonOptimizing()) {
301 // Same logic as in ark::compiler::Pipeline::RunOptimizations
302 if (!(compiler::g_options.WasSetCompilerLoopUnroll() && compiler::g_options.IsCompilerLoopUnroll())) {
303 graph->SetUnrollComplete();
304 }
305 graph->RunPass<compiler::Peepholes>();
306 graph->RunPass<compiler::BranchElimination>();
307 graph->RunPass<compiler::OptimizeStringConcat>();
308 graph->RunPass<compiler::SimplifyStringBuilder>();
309 graph->RunPass<compiler::Inlining>(llvmPreOpt == 0);
310 graph->RunPass<compiler::Cleanup>();
311 graph->RunPass<compiler::CatchInputs>();
312 graph->RunPass<compiler::TryCatchResolving>();
313 if (!MonitorsCorrect(graph, false)) {
314 return false;
315 }
316 graph->RunPass<compiler::Peepholes>();
317 graph->RunPass<compiler::BranchElimination>();
318 graph->RunPass<compiler::ValNum>();
319 graph->RunPass<compiler::IfMerging>();
320 graph->RunPass<compiler::Cleanup>(false);
321 graph->RunPass<compiler::Peepholes>();
322 graph->RunPass<compiler::ChecksElimination>();
323 if (llvmPreOpt == 2U) {
324 PreOpt2(graph);
325 }
326 }
327 #ifndef NDEBUG
328 if (compiler::g_options.IsCompilerCheckGraph() && compiler::g_options.IsCompilerCheckFinal()) {
329 compiler::GraphChecker(graph, "LLVM").Check();
330 }
331 #endif
332
333 return !compiler::g_options.IsCompilerNonOptimizing() || MonitorsCorrect(graph, true);
334 }
335
PreOpt2(compiler::Graph * graph)336 void LLVMAotCompiler::PreOpt2(compiler::Graph *graph)
337 {
338 graph->RunPass<compiler::Licm>(compiler::g_options.GetCompilerLicmHoistLimit());
339 graph->RunPass<compiler::LicmConditions>();
340 graph->RunPass<compiler::RedundantLoopElimination>();
341 graph->RunPass<compiler::LoopPeeling>();
342 graph->RunPass<compiler::LoopUnswitch>(compiler::g_options.GetCompilerLoopUnswitchMaxLevel(),
343 compiler::g_options.GetCompilerLoopUnswitchMaxInsts());
344 graph->RunPass<compiler::Lse>();
345 graph->RunPass<compiler::ValNum>();
346 if (graph->RunPass<compiler::Peepholes>() && graph->RunPass<compiler::BranchElimination>()) {
347 graph->RunPass<compiler::Peepholes>();
348 graph->RunPass<compiler::ValNum>();
349 }
350 graph->RunPass<compiler::Cleanup>();
351
352 graph->RunPass<compiler::LoopIdioms>();
353 graph->RunPass<compiler::ChecksElimination>();
354 if (graph->RunPass<compiler::DeoptimizeElimination>()) {
355 graph->RunPass<compiler::Peepholes>();
356 }
357 if (compiler::g_options.WasSetCompilerLoopUnroll()) {
358 graph->RunPass<compiler::LoopUnroll>(compiler::g_options.GetCompilerLoopUnrollInstLimit(),
359 compiler::g_options.GetCompilerLoopUnrollFactor());
360 }
361 compiler::OptimizationsAfterUnroll(graph);
362 graph->RunPass<compiler::Peepholes>();
363 graph->RunPass<compiler::EscapeAnalysis>();
364 graph->RunPass<compiler::ReserveStringBuilderBuffer>();
365 ASSERT(graph->IsUnrollComplete());
366
367 graph->RunPass<compiler::Peepholes>();
368 graph->RunPass<compiler::BranchElimination>();
369 graph->RunPass<compiler::BalanceExpressions>();
370 graph->RunPass<compiler::ValNum>();
371 graph->RunPass<compiler::SaveStateOptimization>();
372 graph->RunPass<compiler::Peepholes>();
373 #ifndef NDEBUG
374 graph->SetLowLevelInstructionsEnabled();
375 #endif // NDEBUG
376 graph->RunPass<compiler::Cleanup>(false);
377 graph->RunPass<compiler::CodeSink>();
378 graph->RunPass<compiler::Cleanup>(false);
379 }
380
AddGraphToModule(compiler::Graph * graph,WrappedModule & module,AddGraphMode addGraphMode)381 Expected<bool, std::string> LLVMAotCompiler::AddGraphToModule(compiler::Graph *graph, WrappedModule &module,
382 AddGraphMode addGraphMode)
383 {
384 auto method = graph->GetMethod();
385 if (module.HasFunctionDefinition(method) && addGraphMode == AddGraphMode::PRIMARY_FUNCTION) {
386 auto function = module.GetFunctionByMethodPtr(method);
387 UnmarkAsInlineObject(function);
388 return true;
389 }
390 LLVM_LOG(DEBUG, INFRA) << "Adding graph for method = '" << runtime_->GetMethodFullName(graph->GetMethod()) << "'";
391 if (!RunArkPasses(graph)) {
392 LLVM_LOG(WARNING, INFRA) << "Monitors are unbalanced for method = '"
393 << runtime_->GetMethodFullName(graph->GetMethod()) << "'";
394 return false;
395 }
396 std::string graphError = compiler::LLVMIrConstructor::CheckGraph(graph);
397 if (!graphError.empty()) {
398 return Unexpected(graphError);
399 }
400
401 compiler::LLVMIrConstructor ctor(graph, module.GetModule().get(), module.GetLLVMContext().get(),
402 module.GetLLVMArkInterface().get(), module.GetDebugData());
403 auto llvmFunction = ctor.GetFunc();
404 bool noInline = IsInliningDisabled(graph);
405 if (addGraphMode == AddGraphMode::INLINE_FUNCTION) {
406 MarkAsInlineObject(llvmFunction);
407 }
408 bool builtIr = ctor.BuildIr(noInline);
409 if (!builtIr) {
410 if (addGraphMode == AddGraphMode::PRIMARY_FUNCTION) {
411 irFailed_ = true;
412 LLVM_LOG(ERROR, INFRA) << "LLVM AOT failed to build IR for method '"
413 << module.GetLLVMArkInterface()->GetUniqMethodName(method) << "'";
414 } else {
415 ASSERT(addGraphMode == AddGraphMode::INLINE_FUNCTION);
416 LLVM_LOG(WARNING, INFRA) << "LLVM AOT failed to build IR for inline function '"
417 << module.GetLLVMArkInterface()->GetUniqMethodName(method) << "'";
418 }
419 llvmFunction->deleteBody();
420 return Unexpected(std::string("LLVM AOT failed to build IR"));
421 }
422
423 LLVM_LOG(DEBUG, INFRA) << "LLVM AOT built LLVM IR for method "
424 << module.GetLLVMArkInterface()->GetUniqMethodName(method);
425 return true;
426 }
427
PrepareAotGot(WrappedModule * wrappedModule)428 void LLVMAotCompiler::PrepareAotGot(WrappedModule *wrappedModule)
429 {
430 static constexpr size_t ARRAY_LENGTH = 0;
431 auto array64 = llvm::ArrayType::get(llvm::Type::getInt64Ty(*wrappedModule->GetLLVMContext()), ARRAY_LENGTH);
432
433 auto aotGot =
434 new llvm::GlobalVariable(*wrappedModule->GetModule(), array64, false, llvm::GlobalValue::ExternalLinkage,
435 llvm::ConstantAggregateZero::get(array64), "__aot_got");
436
437 aotGot->setSection(AOT_GOT_SECTION);
438 aotGot->setVisibility(llvm::GlobalVariable::ProtectedVisibility);
439 }
440
CreateModule(uint32_t moduleId)441 WrappedModule LLVMAotCompiler::CreateModule(uint32_t moduleId)
442 {
443 auto ttriple = GetTripleForArch(GetArch());
444 auto llvmContext = std::make_unique<llvm::LLVMContext>();
445 auto module = std::make_unique<llvm::Module>("panda-llvmaot-module", *llvmContext);
446 auto options = InitializeLLVMCompilerOptions();
447 // clang-format off
448 auto targetMachine = cantFail(TargetMachineBuilder {}
449 .SetCPU(GetCPUForArch(GetArch()))
450 .SetOptLevel(static_cast<llvm::CodeGenOpt::Level>(options.optlevel))
451 .SetFeatures(GetFeaturesForArch(GetArch()))
452 .SetTriple(ttriple)
453 .Build());
454 // clang-format on
455 auto layout = targetMachine->createDataLayout();
456 module->setDataLayout(layout);
457 module->setTargetTriple(ttriple.getTriple());
458 auto debugData = std::make_unique<DebugDataBuilder>(module.get(), filename_);
459 auto arkInterface = std::make_unique<LLVMArkInterface>(runtime_, ttriple, aotBuilder_, &lock_);
460 arkInterface.get()->CreateRequiredIntrinsicFunctionTypes(*llvmContext.get());
461 compiler::LLVMIrConstructor::InsertArkFrameInfo(module.get(), GetArch());
462 compiler::LLVMIrConstructor::ProvideSafepointPoll(module.get(), arkInterface.get(), GetArch());
463 WrappedModule wrappedModule {
464 std::move(llvmContext), //
465 std::move(module), //
466 std::move(targetMachine), //
467 std::move(arkInterface), //
468 std::move(debugData) //
469 };
470 wrappedModule.SetId(moduleId);
471
472 PrepareAotGot(&wrappedModule);
473 return wrappedModule;
474 }
475
CreateLLVMAotCompiler(compiler::RuntimeInterface * runtime,ArenaAllocator * allocator,compiler::LLVMAotBuilder * aotBuilder,const std::string & cmdline,const std::string & filename)476 std::unique_ptr<CompilerInterface> CreateLLVMAotCompiler(compiler::RuntimeInterface *runtime, ArenaAllocator *allocator,
477 compiler::LLVMAotBuilder *aotBuilder,
478 const std::string &cmdline, const std::string &filename)
479 {
480 return std::make_unique<LLVMAotCompiler>(runtime, allocator, aotBuilder, cmdline, filename);
481 }
482
LLVMAotCompiler(compiler::RuntimeInterface * runtime,ArenaAllocator * allocator,compiler::LLVMAotBuilder * aotBuilder,std::string cmdline,std::string filename)483 LLVMAotCompiler::LLVMAotCompiler(compiler::RuntimeInterface *runtime, ArenaAllocator *allocator,
484 compiler::LLVMAotBuilder *aotBuilder, std::string cmdline, std::string filename)
485 : LLVMCompiler(aotBuilder->GetArch()),
486 methods_(allocator->Adapter()),
487 aotBuilder_(aotBuilder),
488 cmdline_(std::move(cmdline)),
489 filename_(std::move(filename)),
490 runtime_(runtime)
491 {
492 auto arch = aotBuilder->GetArch();
493 llvm::Triple ttriple = GetTripleForArch(arch);
494 auto llvmCompilerOptions = InitializeLLVMCompilerOptions();
495 SetLLVMOption("spill-slot-min-size-bytes", std::to_string(GetFrameSlotSize(arch)));
496 // Disable some options of PlaceSafepoints pass
497 SetLLVMOption("spp-no-entry", "true");
498 SetLLVMOption("spp-no-call", "true");
499 // Set limit to skip Safepoints on entry for small functions
500 SetLLVMOption("isp-on-entry-limit", std::to_string(compiler::g_options.GetCompilerSafepointEliminationLimit()));
501 SetLLVMOption("enable-implicit-null-checks", compiler::g_options.IsCompilerImplicitNullCheck() ? "true" : "false");
502 SetLLVMOption("imp-null-check-page-size", std::to_string(runtime->GetProtectedMemorySize()));
503 if (arch == Arch::AARCH64) {
504 SetLLVMOption("aarch64-enable-ptr32", "true");
505 }
506 if (static_cast<mem::GCType>(aotBuilder->GetGcType()) != mem::GCType::G1_GC) {
507 // NOTE(zdenis): workaround to prevent vector stores of adjacent references
508 SetLLVMOption("vectorize-slp", "false");
509 }
510
511 spreader_ = std::make_unique<FixedCountSpreader>(
512 runtime_, g_options.GetLlvmaotMethodsPerModule(),
513 [this](uint32_t moduleId) { return std::make_shared<WrappedModule>(CreateModule(moduleId)); });
514 int32_t numThreads = g_options.GetLlvmaotThreads();
515 if (numThreads >= 0) {
516 auto strategy = llvm::hardware_concurrency(numThreads);
517 threadPool_ = std::make_unique<llvm::ThreadPool>(strategy);
518
519 numThreads = strategy.compute_thread_count();
520 ASSERT(numThreads > 0);
521 semaphore_ = numThreads;
522 }
523 LLVMCompiler::InitializeLLVMOptions();
524 }
525
526 /* static */
GetFeaturesForArch(Arch arch)527 std::vector<std::string> LLVMAotCompiler::GetFeaturesForArch(Arch arch)
528 {
529 std::vector<std::string> features;
530 features.reserve(2U);
531 switch (arch) {
532 case Arch::AARCH64: {
533 features.emplace_back(std::string("+reserve-x") + std::to_string(GetThreadReg(arch)));
534 if (compiler::g_options.IsCpuFeatureEnabled(compiler::CRC32)) {
535 features.emplace_back(std::string("+crc"));
536 }
537 return features;
538 }
539 case Arch::X86_64:
540 features.emplace_back(std::string("+fixed-r") + std::to_string(GetThreadReg(arch)));
541 if (compiler::g_options.IsCpuFeatureEnabled(compiler::SSE42)) {
542 features.emplace_back("+sse4.2");
543 }
544 return features;
545 default:
546 return {};
547 }
548 }
549
CollectAotBuilderOffsets(const std::unordered_set<std::shared_ptr<WrappedModule>> & modules)550 AotBuilderOffsets LLVMAotCompiler::CollectAotBuilderOffsets(
551 const std::unordered_set<std::shared_ptr<WrappedModule>> &modules)
552 {
553 const auto &methodPtrs = aotBuilder_->GetMethods();
554 const auto &headers = aotBuilder_->GetMethodHeaders();
555 auto methodsIt = std::begin(methods_);
556 for (size_t i = 0; i < headers.size(); ++i) {
557 if (methodPtrs[i].GetMethod() != *methodsIt) {
558 continue;
559 }
560
561 auto module = spreader_->GetModuleForMethod(*methodsIt);
562 auto fullMethodName = module->GetLLVMArkInterface()->GetUniqMethodName(*methodsIt);
563 std::unordered_map<std::string, CreatedObjectFile::StackMapSymbol> stackmapInfo;
564
565 auto section = module->GetObjectFile()->GetSectionByFunctionName(fullMethodName);
566 auto compiledMethod = AdaptCode(*methodsIt, {section.GetMemory(), section.GetSize()});
567 aotBuilder_->AdjustMethodHeader(compiledMethod, i);
568 EVENT_PAOC("Compiling " + runtime_->GetMethodFullName(*methodsIt) + " using llvm");
569 methodsIt++;
570 }
571 ASSERT(methodsIt == std::end(methods_));
572
573 std::vector<compiler::RoData> roDatas;
574 for (auto &module : modules) {
575 auto roDataSections = module->GetObjectFile()->GetRoDataSections();
576 for (auto &item : roDataSections) {
577 roDatas.push_back(compiler::RoData {
578 item.ContentToVector(), item.GetName() + std::to_string(module->GetModuleId()), item.GetAlignment()});
579 }
580 }
581 std::sort(roDatas.begin(), roDatas.end(),
582 [](const compiler::RoData &a, const compiler::RoData &b) -> bool { return a.name < b.name; });
583 aotBuilder_->SetRoDataSections(roDatas);
584
585 // All necessary information is supplied to the aotBuilder.
586 // And it can return the offsets of sections and methods where they will be located.
587 auto sectionAddresses = aotBuilder_->GetSectionsAddresses(cmdline_, filename_);
588 for (const auto &item : sectionAddresses) {
589 LLVM_LOG(DEBUG, INFRA) << item.first << " starts at " << item.second;
590 }
591
592 std::unordered_map<std::string, size_t> methodOffsets;
593 for (size_t i = 0; i < headers.size(); ++i) {
594 auto method =
595 const_cast<compiler::RuntimeInterface::MethodPtr>(static_cast<const void *>(methodPtrs[i].GetMethod()));
596 std::string methodName =
597 spreader_->GetModuleForMethod(method)->GetLLVMArkInterface()->GetUniqMethodName(methodPtrs[i].GetMethod());
598 methodOffsets[methodName] = headers[i].codeOffset;
599 }
600 return AotBuilderOffsets {std::move(sectionAddresses), std::move(methodOffsets)};
601 }
602
AdaptCode(Method * method,Span<const uint8_t> machineCode)603 compiler::CompiledMethod LLVMAotCompiler::AdaptCode(Method *method, Span<const uint8_t> machineCode)
604 {
605 ASSERT(method != nullptr);
606 ASSERT(!machineCode.empty());
607
608 ArenaAllocator allocator {SpaceType::SPACE_TYPE_COMPILER};
609
610 ArenaVector<uint8_t> code(allocator.Adapter());
611 code.insert(code.begin(), machineCode.cbegin(), machineCode.cend());
612
613 auto compiledMethod = compiler::CompiledMethod(GetArch(), method, 0);
614 compiledMethod.SetCode(Span<const uint8_t>(code));
615
616 compiler::CodeInfoBuilder codeInfoBuilder(GetArch(), &allocator);
617 spreader_->GetModuleForMethod(method)->GetCodeInfoProducer()->Produce(method, &codeInfoBuilder);
618
619 ArenaVector<uint8_t> codeInfo(allocator.Adapter());
620 codeInfoBuilder.Encode(&codeInfo);
621 ASSERT(!codeInfo.empty());
622 compiledMethod.SetCodeInfo(Span<const uint8_t>(codeInfo));
623
624 return compiledMethod;
625 }
626
LinkModule(WrappedModule * wrappedModule,ArkAotLinker * linker,AotBuilderOffsets * offsets)627 ArkAotLinker::RoDataSections LLVMAotCompiler::LinkModule(WrappedModule *wrappedModule, ArkAotLinker *linker,
628 AotBuilderOffsets *offsets)
629 {
630 exitOnErr_(linker->LoadObject(wrappedModule->TakeObjectFile()));
631 exitOnErr_(linker->RelocateSections(offsets->GetSectionAddresses(), offsets->GetMethodOffsets(),
632 wrappedModule->GetModuleId()));
633 const auto &methodPtrs = aotBuilder_->GetMethods();
634 const auto &headers = aotBuilder_->GetMethodHeaders();
635 ASSERT((methodPtrs.size() == headers.size()) && (methodPtrs.size() >= methods_.size()));
636 if (wrappedModule->GetMethods().empty()) {
637 return {};
638 }
639 auto methodsIt = wrappedModule->GetMethods().begin();
640
641 for (size_t i = 0; i < headers.size(); ++i) {
642 if (methodsIt == wrappedModule->GetMethods().end()) {
643 break;
644 }
645 if (methodPtrs[i].GetMethod() != *methodsIt) {
646 continue;
647 }
648 auto methodName = wrappedModule->GetLLVMArkInterface()->GetUniqMethodName(*methodsIt);
649 auto functionSection = linker->GetLinkedFunctionSection(methodName);
650 Span<const uint8_t> code(functionSection.GetMemory(), functionSection.GetSize());
651 auto compiledMethod = AdaptCode(static_cast<Method *>(*methodsIt), code);
652 if (g_options.IsLlvmDumpCodeinfo()) {
653 DumpCodeInfo(compiledMethod);
654 }
655 aotBuilder_->AdjustMethod(compiledMethod, i);
656 methodsIt++;
657 }
658 return linker->GetLinkedRoDataSections();
659 }
660
FinishCompile()661 void LLVMAotCompiler::FinishCompile()
662 {
663 // No need to do anything if we have no methods
664 if (methods_.empty()) {
665 return;
666 }
667 ASSERT(currentModule_ != nullptr);
668 ASSERT_PRINT(!compiled_, "Cannot compile twice");
669
670 if (threadPool_ != nullptr) {
671 // Proceed with last module
672 auto thread = runtime_->GetCurrentThread();
673 threadPool_->async(
674 [this, thread](auto it) -> void {
675 ScopedCompilerThread scopedCompilerThread {thread, runtime_};
676 CompileModule(*it);
677 },
678 currentModule_);
679
680 // Wait all workers to finish
681 threadPool_->wait();
682 } else {
683 // Last module
684 CompileModule(*currentModule_);
685 }
686
687 auto modules = spreader_->GetModules();
688 std::vector<compiler::RoData> roDatas;
689 auto offsets = CollectAotBuilderOffsets(modules);
690 for (auto &wrappedModule : modules) {
691 size_t functionHeaderSize = compiler::CodeInfo::GetCodeOffset(GetArch());
692 ArkAotLinker linker {functionHeaderSize};
693 auto newRodatas = LinkModule(wrappedModule.get(), &linker, &offsets);
694 for (auto &item : newRodatas) {
695 roDatas.push_back(compiler::RoData {item.ContentToVector(),
696 item.GetName() + std::to_string(wrappedModule->GetModuleId()),
697 item.GetAlignment()});
698 }
699 }
700 compiled_ = true;
701
702 std::sort(roDatas.begin(), roDatas.end(),
703 [](const compiler::RoData &a, const compiler::RoData &b) -> bool { return a.name < b.name; });
704 aotBuilder_->SetRoDataSections(std::move(roDatas));
705 }
706
AddInlineMethodByDepth(WrappedModule & module,compiler::Graph * caller,compiler::RuntimeInterface::MethodPtr method,int32_t depth)707 void LLVMAotCompiler::AddInlineMethodByDepth(WrappedModule &module, compiler::Graph *caller,
708 compiler::RuntimeInterface::MethodPtr method, int32_t depth)
709 {
710 if (!runtime_->IsMethodCanBeInlined(method) || IsInliningDisabled(runtime_, method)) {
711 LLVM_LOG(DEBUG, INFRA) << "Will not add inline function = '" << runtime_->GetMethodFullName(method)
712 << "' for caller = '" << runtime_->GetMethodFullName(caller->GetMethod())
713 << "' because !IsMethodCanBeInlined or IsInliningDisabled or the inline "
714 "function is external for the caller";
715 return;
716 }
717
718 auto function = module.GetFunctionByMethodPtr(method);
719 if (function != nullptr && !function->isDeclarationForLinker()) {
720 return;
721 }
722 ArenaAllocator allocator {SpaceType::SPACE_TYPE_COMPILER};
723 ArenaAllocator localAllocator {SpaceType::SPACE_TYPE_COMPILER};
724
725 auto graph = CreateGraph(allocator, localAllocator, *static_cast<Method *>(method));
726 if (!graph) {
727 [[maybe_unused]] auto message = llvm::toString(graph.takeError());
728 LLVM_LOG(WARNING, INFRA) << "Could not add inline function = '" << runtime_->GetMethodFullName(method)
729 << "' for caller = '" << runtime_->GetMethodFullName(caller->GetMethod())
730 << "' because '" << message << "'";
731 return;
732 }
733 auto callee = llvm::cantFail(std::move(graph));
734 auto result = AddGraphToModule(callee, module, AddGraphMode::INLINE_FUNCTION);
735 if (!result.HasValue() || !result.Value()) {
736 LLVM_LOG(WARNING, INFRA) << "Could not add inline function = '" << runtime_->GetMethodFullName(method)
737 << "' for caller = '" << runtime_->GetMethodFullName(caller->GetMethod()) << "'";
738 return;
739 }
740
741 LLVM_LOG(DEBUG, INFRA) << "Added inline function = '" << runtime_->GetMethodFullName(callee->GetMethod(), true)
742 << "because '" << runtime_->GetMethodFullName(caller->GetMethod(), true) << "' calls it";
743 AddInlineFunctionsByDepth(module, callee, depth + 1);
744 }
745
AddInlineFunctionsByDepth(WrappedModule & module,compiler::Graph * caller,int32_t depth)746 void LLVMAotCompiler::AddInlineFunctionsByDepth(WrappedModule &module, compiler::Graph *caller, int32_t depth)
747 {
748 ASSERT(depth >= 0);
749 LLVM_LOG(DEBUG, INFRA) << "Adding inline functions, depth = " << depth << ", caller = '"
750 << caller->GetRuntime()->GetMethodFullName(caller->GetMethod()) << "'";
751
752 // Max depth for framework.abc is 26.
753 // Limit the depth to avoid excessively large call depth
754 if (depth >= MAX_DEPTH) {
755 LLVM_LOG(DEBUG, INFRA) << "Exiting, inlining depth exceeded";
756 return;
757 }
758
759 for (auto basicBlock : caller->GetBlocksRPO()) {
760 for (auto instruction : basicBlock->AllInsts()) {
761 if (instruction->GetOpcode() != compiler::Opcode::CallStatic) {
762 continue;
763 }
764
765 auto method = instruction->CastToCallStatic()->GetCallMethod();
766 ASSERT(method != nullptr);
767 AddInlineMethodByDepth(module, caller, method, depth);
768 }
769 }
770 }
771
CreateGraph(ArenaAllocator & allocator,ArenaAllocator & localAllocator,Method & method)772 llvm::Expected<compiler::Graph *> LLVMAotCompiler::CreateGraph(ArenaAllocator &allocator,
773 ArenaAllocator &localAllocator, Method &method)
774 {
775 // Part of Paoc::CompileAot
776 auto sourceLang = method.GetClass()->GetSourceLang();
777 bool isDynamic = panda_file::IsDynamicLanguage(sourceLang);
778
779 auto graph = allocator.New<compiler::Graph>(
780 compiler::Graph::GraphArgs {&allocator, &localAllocator, aotBuilder_->GetArch(), &method, runtime_}, nullptr,
781 false, isDynamic);
782 if (graph == nullptr) {
783 return llvm::createStringError(llvm::inconvertibleErrorCode(), "Couldn't create graph");
784 }
785 graph->SetLanguage(sourceLang);
786
787 uintptr_t codeAddress = aotBuilder_->GetCurrentCodeAddress();
788 auto aotData = graph->GetAllocator()->New<compiler::AotData>(
789 compiler::AotData::AotDataArgs {method.GetPandaFile(),
790 graph,
791 nullptr,
792 codeAddress,
793 aotBuilder_->GetIntfInlineCacheIndex(),
794 {aotBuilder_->GetGotPlt(), aotBuilder_->GetGotVirtIndexes(),
795 aotBuilder_->GetGotClass(), aotBuilder_->GetGotString()},
796 {aotBuilder_->GetGotIntfInlineCache(), aotBuilder_->GetGotCommon()}});
797
798 aotData->SetUseCha(true);
799 graph->SetAotData(aotData);
800
801 if (!graph->RunPass<compiler::IrBuilder>()) {
802 return llvm::createStringError(llvm::inconvertibleErrorCode(), "IrBuilder failed");
803 }
804 return graph;
805 }
806
CompileModule(WrappedModule & module)807 void LLVMAotCompiler::CompileModule(WrappedModule &module)
808 {
809 auto compilerOptions = InitializeLLVMCompilerOptions();
810 module.GetDebugData()->Finalize();
811 auto arkInterface = module.GetLLVMArkInterface().get();
812 auto llvmIrModule = module.GetModule().get();
813
814 auto &targetMachine = module.GetTargetMachine();
815 LLVMOptimizer llvmOptimizer {compilerOptions, arkInterface, targetMachine};
816
817 // Dumps require lock to be not messed up between different modules compiled in parallel.
818 // NB! When ark is built with TSAN there are warnings coming from libLLVM.so.
819 // They do not fire in tests because we do not run parallel compilation in tests.
820 // We have to use suppression file, to run parallel compilation under tsan
821
822 { // Dump before optimization
823 llvm::sys::ScopedLock scopedLock {lock_};
824 llvmOptimizer.DumpModuleBefore(llvmIrModule);
825 }
826
827 // Optimize IR
828 llvmOptimizer.OptimizeModule(llvmIrModule);
829
830 { // Dump after optimization
831 llvm::sys::ScopedLock scopedLock {lock_};
832 llvmOptimizer.DumpModuleAfter(llvmIrModule);
833 }
834
835 // clang-format off
836 MIRCompiler mirCompiler {targetMachine, [&arkInterface](InsertingPassManager *manager) -> void {
837 manager->InsertBefore(&llvm::FEntryInserterID, CreateFrameLoweringPass(arkInterface));
838 }};
839 // clang-format on
840
841 // Create machine code
842 auto file = exitOnErr_(mirCompiler.CompileModule(*llvmIrModule));
843 if (file->HasSection(".llvm_stackmaps")) {
844 auto section = file->GetSection(".llvm_stackmaps");
845 module.GetCodeInfoProducer()->SetStackMap(section.GetMemory(), section.GetSize());
846
847 auto stackmapInfo = file->GetStackMapInfo();
848
849 for (auto method : module.GetMethods()) {
850 auto fullMethodName = module.GetLLVMArkInterface()->GetUniqMethodName(method);
851 if (stackmapInfo.find(fullMethodName) != stackmapInfo.end()) {
852 module.GetCodeInfoProducer()->AddSymbol(static_cast<Method *>(method), stackmapInfo.at(fullMethodName));
853 }
854 }
855 }
856 if (file->HasSection(".llvm_faultmaps")) {
857 auto section = file->GetSection(".llvm_faultmaps");
858 module.GetCodeInfoProducer()->SetFaultMaps(section.GetMemory(), section.GetSize());
859
860 auto faultMapInfo = file->GetFaultMapInfo();
861
862 for (auto method : module.GetMethods()) {
863 auto fullMethodName = module.GetLLVMArkInterface()->GetUniqMethodName(method);
864 if (faultMapInfo.find(fullMethodName) != faultMapInfo.end()) {
865 module.GetCodeInfoProducer()->AddFaultMapSymbol(static_cast<Method *>(method),
866 faultMapInfo.at(fullMethodName));
867 }
868 }
869 }
870
871 if (g_options.IsLlvmDumpObj()) {
872 uint32_t moduleNumber = compiledModules_++;
873 std::string fileName = "llvm-output-" + std::to_string(moduleNumber) + ".o";
874 file->WriteTo(fileName);
875 }
876 module.SetCompiled(std::move(file));
877 }
878
DumpCodeInfo(compiler::CompiledMethod & method) const879 void LLVMAotCompiler::DumpCodeInfo(compiler::CompiledMethod &method) const
880 {
881 LLVM_LOG(DEBUG, INFRA) << "Dumping code info for method: " << runtime_->GetMethodFullName(method.GetMethod());
882 std::stringstream ss;
883 compiler::CodeInfo info;
884 info.Decode(method.GetCodeInfo());
885 info.Dump(ss);
886 LLVM_LOG(DEBUG, INFRA) << ss.str();
887 }
888
889 } // namespace ark::llvmbackend
890