• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "libpandafile/class_data_accessor.h"
17 #include "paoc_options.h"
18 #include "aot/compiled_method.h"
19 #include "paoc_clusters.h"
20 #include "compiler.h"
21 #include "compiler_options.h"
22 #include "compiler_events_gen.h"
23 #include "compiler_logger.h"
24 #include "events/events.h"
25 #include "include/runtime.h"
26 #include "mem/gc/gc_types.h"
27 #include "optimizer_run.h"
28 #include "optimizer/ir_builder/ir_builder.h"
29 #include "os/filesystem.h"
30 #include "generated/base_options.h"
31 
32 #include "paoc.h"
33 #ifdef PANDA_LLVM_AOT
34 #include "paoc_llvm.h"
35 #endif
36 
37 using namespace ark::compiler;  // NOLINT(*-build-using-namespace)
38 
39 namespace ark::paoc {
40 
41 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
42 #define LOG_PAOC(level) LOG(level, COMPILER) << "PAOC: "
43 
CompilingContext(Method * methodPtr,size_t methodIndex,std::ofstream * statisticsDump)44 Paoc::CompilingContext::CompilingContext(Method *methodPtr, size_t methodIndex, std::ofstream *statisticsDump)
45     : method(methodPtr),
46       allocator(ark::SpaceType::SPACE_TYPE_COMPILER, PandaVM::GetCurrent()->GetMemStats(), true),
47       graphLocalAllocator(ark::SpaceType::SPACE_TYPE_COMPILER, PandaVM::GetCurrent()->GetMemStats(), true),
48       index(methodIndex),
49       stats(statisticsDump)
50 {
51 }
52 
~CompilingContext()53 Paoc::CompilingContext::~CompilingContext()
54 {
55     if (graph != nullptr) {
56         graph->~Graph();
57     }
58     ASSERT(stats != nullptr);
59     if (compilationStatus && *stats) {
60         DumpStatistics();
61     }
62 }
DumpStatistics() const63 void Paoc::CompilingContext::DumpStatistics() const
64 {
65     ASSERT(stats);
66     char sep = ',';
67     *stats << method->GetFullName() << sep;
68     *stats << "paoc-summary" << sep;
69     *stats << allocator.GetAllocatedSize() << sep;
70     *stats << graphLocalAllocator.GetAllocatedSize() << '\n';
71 }
72 
73 /// A class that encloses paoc's initialization:
74 class Paoc::PaocInitializer {
75 public:
PaocInitializer(Paoc * paoc)76     explicit PaocInitializer(Paoc *paoc) : paoc_(paoc) {}
77     // NOLINTNEXTLINE(modernize-avoid-c-arrays)
Init(const ark::Span<const char * > & args)78     int Init(const ark::Span<const char *> &args)
79     {
80         ASSERT(args.Data() != nullptr);
81         ark::PandArgParser paParser;
82 
83         paoc_->runtimeOptions_ = std::make_unique<decltype(paoc_->runtimeOptions_)::element_type>(args[0]);
84         paoc_->runtimeOptions_->AddOptions(&paParser);
85         paoc_->paocOptions_ = std::make_unique<decltype(paoc_->paocOptions_)::element_type>(args[0]);
86         paoc_->paocOptions_->AddOptions(&paParser);
87         ark::compiler::g_options.AddOptions(&paParser);
88 
89         base_options::Options baseOptions("");
90         baseOptions.AddOptions((&paParser));
91 
92         paoc_->AddExtraOptions(&paParser);
93 
94         if (!paParser.Parse(args.Size(), args.Data())) {
95             std::cerr << "Error: " << paParser.GetErrorString() << "\n";
96             return -1;
97         }
98         Logger::Initialize(baseOptions);
99         if (paoc_->paocOptions_->GetPaocPandaFiles().empty()) {
100             paoc_->PrintUsage(paParser);
101             return 1;
102         }
103         if (!os::IsDirExists(os::GetParentDir(paoc_->paocOptions_->GetPaocOutput()))) {
104             std::cerr << "Error: directory for output file \"" << paoc_->paocOptions_->GetPaocOutput()
105                       << "\" doesn't exist "
106                       << "\n";
107             return -1;
108         }
109         if (InitRuntime() != 0) {
110             return -1;
111         }
112         if (InitCompiler() != 0) {
113             return -1;
114         }
115         if (paoc_->paocOptions_->WasSetPaocClusters() && !InitPaocClusters(&paParser)) {
116             return -1;
117         }
118         if (paoc_->IsLLVMAotMode()) {
119             paoc_->PrepareLLVM(args);
120         }
121 
122         return 0;
123     }
124 
125 private:
InitRuntime()126     int InitRuntime()
127     {
128         auto runtimeOptionsErr = paoc_->runtimeOptions_->Validate();
129         if (runtimeOptionsErr) {
130             std::cerr << "Error: " << runtimeOptionsErr.value().GetMessage() << std::endl;
131             return 1;
132         }
133 #ifndef PANDA_ASAN_ON
134         if (!paoc_->runtimeOptions_->WasSetInternalMemorySizeLimit()) {
135             // 2Gb - for emitting code
136             constexpr size_t CODE_SIZE_LIMIT = 0x7f000000;
137             paoc_->runtimeOptions_->SetInternalMemorySizeLimit(CODE_SIZE_LIMIT);
138         }
139 #endif
140         paoc_->runtimeOptions_->SetArkAot(true);
141         if (!paoc_->runtimeOptions_->WasSetTaskmanagerWorkersCount()) {
142             paoc_->runtimeOptions_->SetTaskmanagerWorkersCount(1);
143         }
144         if (!ark::Runtime::Create(*paoc_->runtimeOptions_)) {
145             std::cerr << "Failed to create runtime!\n";
146             return -1;
147         }
148         paoc_->runtime_ = Runtime::GetCurrent()->GetPandaVM()->GetCompilerRuntimeInterface();
149         return 0;
150     }
151 
InitCompiler()152     int InitCompiler()
153     {
154         CompilerLogger::Init(ark::compiler::g_options.GetCompilerLog());
155 
156         if (ark::compiler::g_options.IsCompilerEnableEvents()) {
157             EventWriter::Init(ark::compiler::g_options.GetCompilerEventsPath());
158         }
159         ValidateCompilerOptions();
160 
161         if (paoc_->paocOptions_->WasSetPaocMethodsFromFile()) {
162             InitPaocMethodsFromFile();
163         }
164         paoc_->skipInfo_.isFirstCompiled = !paoc_->paocOptions_->WasSetPaocSkipUntil();
165         paoc_->skipInfo_.isLastCompiled = false;
166 
167         if (!InitAotBuilder()) {
168             return 1;
169         }
170         if (paoc_->paocOptions_->WasSetPaocDumpStatsCsv()) {
171             paoc_->statisticsDump_ = std::ofstream(paoc_->paocOptions_->GetPaocDumpStatsCsv(), std::ofstream::trunc);
172         }
173         InitPaocMode();
174         paoc_->codeAllocator_ = new ArenaAllocator(ark::SpaceType::SPACE_TYPE_INTERNAL, nullptr, true);
175         paoc_->loader_ = ark::Runtime::GetCurrent()->GetClassLinker();
176 
177         return 0;
178     }
179 
CheckOptionsErr()180     void CheckOptionsErr()
181     {
182         auto compilerOptionsErr = ark::compiler::g_options.Validate();
183         if (compilerOptionsErr) {
184             LOG_PAOC(FATAL) << compilerOptionsErr.value().GetMessage();
185         }
186         auto paocOptionsErr = paoc_->paocOptions_->Validate();
187         if (paocOptionsErr) {
188             LOG_PAOC(FATAL) << paocOptionsErr.value().GetMessage();
189         }
190     }
191 
ValidateCompilerOptions()192     void ValidateCompilerOptions()
193     {
194         CheckOptionsErr();
195         paoc_->ValidateExtraOptions();
196 
197         if (paoc_->paocOptions_->WasSetPaocOutput() && paoc_->paocOptions_->WasSetPaocBootOutput()) {
198             LOG_PAOC(FATAL) << "combination of --paoc-output and --paoc-boot-output is not compatible\n";
199         }
200         if (paoc_->paocOptions_->WasSetPaocBootPandaLocations()) {
201             if (paoc_->paocOptions_->WasSetPaocBootLocation()) {
202                 LOG_PAOC(FATAL) << "We can't use --paoc-boot-panda-locations with --paoc-boot-location\n";
203             }
204             auto locations = paoc_->paocOptions_->GetPaocBootPandaLocations();
205             size_t i = 0;
206             // In fact boot files are preloaded by Runtime
207             for (auto bpf : Runtime::GetCurrent()->GetClassLinker()->GetBootPandaFiles()) {
208                 if (i >= locations.size()) {
209                     EVENT_PAOC("Numbers of files in --paoc-boot-panda-locations and --boot-panda-files are different");
210                     LOG_PAOC(FATAL)
211                         << "number of locations in --paoc-boot-panda-locations less then files in --boot-panda-files\n";
212                 }
213                 auto filename = locations[i];
214                 auto bpfName = bpf->GetFilename();
215                 if (!CompareBootFiles(bpfName, filename)) {
216                     EVENT_PAOC("different files in --paoc-boot-panda-locations and --boot-panda-files");
217                     LOG_PAOC(ERROR) << "The file from --paoc-boot-panda-locations: " << filename;
218                     LOG_PAOC(ERROR) << "isn't eqaul the file from --boot-panda-files: " << bpf->GetFilename();
219                     LOG_PAOC(FATAL) << "Files posotions " << i;
220                 }
221                 paoc_->locationMapping_[bpfName] = filename;
222                 paoc_->preloadedFiles_[paoc_->GetFilePath(bpfName)] = bpf;
223                 ++i;
224             }
225             if (i != locations.size()) {
226                 EVENT_PAOC("Numbers of files in --paoc-boot-panda-locations and --boot-panda-files are different");
227                 LOG_PAOC(FATAL)
228                     << "number of locations in --paoc-boot-panda-locations more then files in --boot-panda-files\n";
229             }
230         }
231         // In fact boot files are preloaded by Runtime
232         for (auto bpf : Runtime::GetCurrent()->GetClassLinker()->GetBootPandaFiles()) {
233             if (!paoc_->paocOptions_->GetPaocBootLocation().empty()) {
234                 std::string filename = GetFileLocation(*bpf, paoc_->paocOptions_->GetPaocBootLocation());
235                 paoc_->locationMapping_[bpf->GetFilename()] = filename;
236             }
237             paoc_->preloadedFiles_[paoc_->GetFilePath(bpf->GetFilename())] = bpf;
238         }
239     }
240 
InitPaocMethodsFromFile()241     void InitPaocMethodsFromFile()
242     {
243         std::ifstream mfile(paoc_->paocOptions_->GetPaocMethodsFromFile());
244         std::string line;
245         if (mfile) {
246             while (getline(mfile, line)) {
247                 paoc_->methodsList_.insert(line);
248             }
249         }
250         LOG_PAOC(INFO) << "Method list size: " << paoc_->methodsList_.size();
251     }
252 
InitPaocMode()253     void InitPaocMode()
254     {
255         const auto &modeStr = paoc_->paocOptions_->GetPaocMode();
256         if (modeStr == "aot") {
257             paoc_->mode_ = PaocMode::AOT;
258         } else if (modeStr == "jit") {
259             paoc_->mode_ = PaocMode::JIT;
260         } else if (modeStr == "osr") {
261             paoc_->mode_ = PaocMode::OSR;
262         } else if (modeStr == "llvm") {
263             paoc_->mode_ = PaocMode::LLVM;
264         } else {
265             UNREACHABLE();
266         }
267     }
268 
InitAotBuilder()269     bool InitAotBuilder()
270     {
271         Arch arch = RUNTIME_ARCH;
272         bool crossCompilation = false;
273         if (compiler::g_options.WasSetCompilerCrossArch()) {
274             arch = GetArchFromString(compiler::g_options.GetCompilerCrossArch());
275             crossCompilation = arch != RUNTIME_ARCH;
276         }
277         ark::compiler::g_options.AdjustCpuFeatures(crossCompilation);
278 
279         if (arch == Arch::NONE) {
280             LOG_PAOC(ERROR) << "Invalid --compiler-cross-arch option:" << compiler::g_options.GetCompilerCrossArch();
281             return false;
282         }
283         if (!BackendSupport(arch)) {
284             LOG_PAOC(ERROR) << "Backend is not supported: " << compiler::g_options.GetCompilerCrossArch();
285             return false;
286         }
287         paoc_->aotBuilder_ = paoc_->CreateAotBuilder();
288         paoc_->aotBuilder_->SetArch(arch);
289 
290         // Initialize GC:
291         auto runtimeLang = paoc_->runtimeOptions_->GetRuntimeType();
292         // Fix it in issue 8164:
293         auto gcType = ark::mem::GCTypeFromString(paoc_->runtimeOptions_->GetGcType(runtimeLang));
294         ASSERT(gcType != ark::mem::GCType::INVALID_GC);
295 
296         paoc_->aotBuilder_->SetGcType(static_cast<uint32_t>(gcType));
297 
298 #ifndef NDEBUG
299         paoc_->aotBuilder_->SetGenerateSymbols(true);
300 #else
301         paoc_->aotBuilder_->SetGenerateSymbols(paoc_->paocOptions_->IsPaocGenerateSymbols());
302 #endif
303 #ifdef PANDA_COMPILER_DEBUG_INFO
304         paoc_->aotBuilder_->SetEmitDebugInfo(ark::compiler::g_options.IsCompilerEmitDebugInfo());
305 #endif
306         return true;
307     }
308 
InitPaocClusters(ark::PandArgParser * paParser)309     bool InitPaocClusters(ark::PandArgParser *paParser)
310     {
311         std::ifstream fstream(paoc_->paocOptions_->GetPaocClusters());
312         if (!fstream) {
313             LOG_PAOC(ERROR) << "Can't open `paoc-clusters` file";
314             return false;
315         }
316         JsonObject obj(fstream.rdbuf());
317         fstream.close();
318         paoc_->clustersInfo_.Init(obj, paParser);
319         return true;
320     }
321 
322 private:
323     Paoc *paoc_ {nullptr};
324 };
325 
Run(const ark::Span<const char * > & args)326 int Paoc::Run(const ark::Span<const char *> &args)
327 {
328     if (PaocInitializer(this).Init(args) != 0) {
329         return -1;
330     }
331     auto allocator = Runtime::GetCurrent()->GetInternalAllocator();
332     if (compiler::g_options.WasSetCompilerProfile()) {
333         auto res = runtime_->AddProfile(compiler::g_options.GetCompilerProfile());
334         if (!res) {
335             LOG(FATAL, COMPILER) << "Cannot open profile `" << compiler::g_options.GetCompilerProfile()
336                                  << "`: " << res.Error();
337         }
338     }
339     aotBuilder_->SetRuntime(runtime_);
340     slowPathData_ = allocator->New<compiler::SharedSlowPathData>();
341 
342     if (!LoadPandaFiles()) {
343         LOG_PAOC(FATAL) << "Can not load panda files";
344     }
345     bool errorOccurred = !CompileFiles();
346     bool attemptedToCompile = (compilationIndex_ != 0);
347     errorOccurred |= !attemptedToCompile;
348     if (IsAotMode()) {
349         if (aotBuilder_->GetCurrentCodeAddress() != 0 || !aotBuilder_->GetClassHashTableSize()->empty() ||
350             IsLLVMAotMode()) {
351             RunAotMode(args);
352         }
353         if (aotBuilder_->GetCurrentCodeAddress() == 0 && aotBuilder_->GetClassHashTableSize()->empty()) {
354             LOG_PAOC(ERROR) << "There are no compiled methods";
355             Clear(allocator);
356             return -1;
357         }
358     }
359     if (!attemptedToCompile) {
360         LOG_PAOC(WARNING) << "There are no compiled methods";
361     }
362     Clear(allocator);
363     return ShouldIgnoreFailures() ? 0 : (errorOccurred ? 1 : 0);
364 }
365 
RunAotMode(const ark::Span<const char * > & args)366 void Paoc::RunAotMode(const ark::Span<const char *> &args)
367 {
368     std::string cmdline;
369     for (auto arg : args) {
370         cmdline += arg;
371         cmdline += " ";
372     }
373     std::string classCtx;
374     loader_->EnumeratePandaFiles([this, &classCtx](const panda_file::File &pf) {
375         if (locationMapping_.find(pf.GetFilename()) == locationMapping_.end()) {
376             classCtx += GetFilePath(pf.GetFilename());
377         } else {
378             classCtx += locationMapping_[pf.GetFilename()];
379         }
380         classCtx += '*';
381         classCtx += pf.GetPaddedChecksum();
382         classCtx += ':';
383         return true;
384     });
385     classCtx.pop_back();
386     aotBuilder_->SetClassContext(classCtx);
387     LOG(DEBUG, COMPILER) << "PAOC: ClassContext '" << classCtx << '\'';
388     auto outputFile = paocOptions_->GetPaocOutput();
389     if (paocOptions_->WasSetPaocBootOutput()) {
390         aotBuilder_->SetBootAot(true);
391         outputFile = paocOptions_->GetPaocBootOutput();
392     }
393     aotBuilder_->SetWithCha(paocOptions_->IsPaocUseCha());
394 
395     if (IsLLVMAotMode()) {
396         bool writeAot = EndLLVM();
397         if (!writeAot) {
398             return;
399         }
400     }
401 
402     aotBuilder_->Write(cmdline, outputFile);
403     LOG_PAOC(INFO) << "METHODS COMPILED (success/all): " << successMethods_ << '/' << successMethods_ + failedMethods_;
404     LOG_PAOC(DEBUG) << "Successfully compiled to " << outputFile;
405 }
406 
Clear(ark::mem::InternalAllocatorPtr allocator)407 void Paoc::Clear(ark::mem::InternalAllocatorPtr allocator)
408 {
409     delete codeAllocator_;
410     codeAllocator_ = nullptr;
411     allocator->Delete(slowPathData_);
412     methodsList_.clear();
413     ark::Runtime::Destroy();
414 }
415 
StartAotFile(const panda_file::File & pfileRef)416 void Paoc::StartAotFile(const panda_file::File &pfileRef)
417 {
418     ASSERT(IsAotMode());
419     std::string filename;
420     if (paocOptions_->GetPaocLocation().empty()) {
421         filename = GetFilePath(pfileRef.GetFilename());
422     } else {
423         filename = GetFileLocation(pfileRef, paocOptions_->GetPaocLocation());
424         locationMapping_[pfileRef.GetFilename()] = filename;
425     }
426     ASSERT(!filename.empty());
427     aotBuilder_->StartFile(filename, pfileRef.GetHeader()->checksum);
428 }
429 
TryLoadPandaFile(const std::string & fileName,PandaVM * vm)430 bool Paoc::TryLoadPandaFile(const std::string &fileName, PandaVM *vm)
431 {
432     const panda_file::File *pfile;
433     bool errorOccurred = false;
434 
435     auto filePath = GetFilePath(fileName);
436     if (preloadedFiles_.find(filePath) != preloadedFiles_.end()) {
437         pfile = preloadedFiles_[filePath];
438     } else {
439         auto file = vm->OpenPandaFile(fileName);
440         if (!file) {
441             if (!ShouldIgnoreFailures()) {
442                 LOG_PAOC(FATAL) << "Can not open file: " << fileName;
443             }
444             LOG_PAOC(WARNING) << "Can not open file: " << fileName;
445             return false;
446         }
447         pfile = file.get();
448         loader_->AddPandaFile(std::move(file));
449         LOG_PAOC(DEBUG) << "Added panda file: " << fileName;
450     }
451     auto &pfileRef = *pfile;
452 
453     if (IsAotMode()) {
454         StartAotFile(pfileRef);
455     }
456 
457     if (!CompilePandaFile(pfileRef)) {
458         errorOccurred = true;
459     }
460 
461     if (IsAotMode()) {
462         aotBuilder_->EndFile();
463     }
464     return !errorOccurred || ShouldIgnoreFailures();
465 }
466 
467 /**
468  * Iterate over `--paoc-panda-files`.
469  * @return `false` on error.
470  */
CompileFiles()471 bool Paoc::CompileFiles()
472 {
473     auto pfiles = paocOptions_->GetPaocPandaFiles();
474     auto *vm = ark::Runtime::GetCurrent()->GetPandaVM();
475     for (auto &fileName : pfiles) {
476         if (!TryLoadPandaFile(fileName, vm)) {
477             return false;
478         }
479     }
480     return true;
481 }
482 
GetFilePath(std::string fileName)483 std::string Paoc::GetFilePath(std::string fileName)
484 {
485     if (runtimeOptions_->IsAotVerifyAbsPath()) {
486         return os::GetAbsolutePath(fileName);
487     }
488     return fileName;
489 }
490 
491 /**
492  * Compile every method from loaded panda files that matches `--compiler-regex`,
493  * `--paoc-skip-until`, `--paoc-compile-until` and `--paoc-methods-from-file` options:
494  * @return `false` on error.
495  */
CompilePandaFile(const panda_file::File & pfileRef)496 bool Paoc::CompilePandaFile(const panda_file::File &pfileRef)
497 {
498     auto classes = pfileRef.GetClasses();
499     bool errorOccurred = false;
500     for (auto &classId : classes) {
501         panda_file::File::EntityId id(classId);
502         ark::Class *klass = ResolveClass(pfileRef, id);
503         std::string className = ClassHelper::GetName(pfileRef.GetStringData(id).data);
504 
505         if (!PossibleToCompile(pfileRef, klass, id)) {
506             if (paocOptions_->IsPaocVerbose()) {
507                 LOG_PAOC(DEBUG) << "Ignoring a class `" << className << "` (id = " << id << ", file `"
508                                 << pfileRef.GetFilename() << "`)";
509             }
510             continue;
511         }
512 
513         ASSERT(klass != nullptr);
514         if (paocOptions_->IsPaocVerbose()) {
515             LOG_PAOC(DEBUG) << "Compiling class `" << className << "` (id = " << id << ", file `"
516                             << pfileRef.GetFilename() << "`)";
517         }
518 
519         // Check that all of the methods are compiled correctly:
520         ASSERT(klass->GetPandaFile() != nullptr);
521         if (!Compile(klass, *klass->GetPandaFile())) {
522             errorOccurred = true;
523             std::string errMsg = "Class `" + className + "` from " + pfileRef.GetFilename() + " compiled with errors";
524             PrintError(errMsg);
525             if (!ShouldIgnoreFailures()) {
526                 break;
527             }
528         }
529     }
530 
531     BuildClassHashTable(pfileRef);
532 
533     return !errorOccurred;
534 }
535 
ResolveClass(const panda_file::File & pfileRef,panda_file::File::EntityId classId)536 ark::Class *Paoc::ResolveClass(const panda_file::File &pfileRef, panda_file::File::EntityId classId)
537 {
538     ErrorHandler handler;
539     ScopedMutatorLock lock;
540     if (pfileRef.IsExternal(classId)) {
541         return nullptr;
542     }
543     panda_file::ClassDataAccessor cda(pfileRef, classId);
544     LanguageContext ctx = Runtime::GetCurrent()->GetLanguageContext(&cda);
545     auto klass = loader_->GetExtension(ctx)->GetClass(pfileRef, classId, nullptr, &handler);
546     return klass;
547 }
548 
549 /**
550  * Check if it is possible to compile a class.
551  * @return true if the class isn't external, abstract, interface and isn't an array class.
552  */
PossibleToCompile(const panda_file::File & pfileRef,const ark::Class * klass,panda_file::File::EntityId classId)553 bool Paoc::PossibleToCompile(const panda_file::File &pfileRef, const ark::Class *klass,
554                              panda_file::File::EntityId classId)
555 {
556     std::string className = ClassHelper::GetName(pfileRef.GetStringData(classId).data);
557     if (klass == nullptr) {
558         if (paocOptions_->IsPaocVerbose()) {
559             LOG_PAOC(DEBUG) << "Class is nullptr: `" << className << "`";
560             LOG_PAOC(ERROR) << "ClassLinker can't load a class `" << className << "`";
561         }
562         return false;
563     }
564 
565     // Skip external classes
566     if (pfileRef.IsExternal(classId) || klass->GetFileId().GetOffset() != classId.GetOffset()) {
567         if (paocOptions_->IsPaocVerbose()) {
568             LOG_PAOC(DEBUG) << "Can't compile external class `" << className << "`";
569         }
570         return false;
571     }
572     // Skip object array class
573     if (klass->IsArrayClass()) {
574         if (paocOptions_->IsPaocVerbose()) {
575             LOG_PAOC(DEBUG) << "Can't compile array class `" << className << "`";
576         }
577         return false;
578     }
579 
580     if (klass->IsAbstract()) {
581         if (paocOptions_->IsPaocVerbose()) {
582             LOG_PAOC(DEBUG) << "Will compile abstract class `" << className << "`";
583         }
584     }
585 
586     return true;
587 }
588 
589 /**
590  * Compile the class.
591  * @return `false` on error.
592  */
Compile(Class * klass,const panda_file::File & pfileRef)593 bool Paoc::Compile(Class *klass, const panda_file::File &pfileRef)
594 {
595     ASSERT(klass != nullptr);
596 
597     if (IsAotMode()) {
598         aotBuilder_->StartClass(*klass);
599     }
600     bool errorOccurred = false;
601     auto methods = klass->GetMethods();
602     panda_file::ClassDataAccessor cda(pfileRef, klass->GetFileId());
603     size_t methodIndex = 0;
604     size_t smethodIdx = klass->GetNumVirtualMethods();
605     size_t vmethodIdx = 0;
606     cda.EnumerateMethods([this, &smethodIdx, &vmethodIdx, &methods, &methodIndex, &pfileRef,
607                           &errorOccurred](panda_file::MethodDataAccessor &methodDataAccessor) {
608         if (errorOccurred && !ShouldIgnoreFailures()) {
609             return;
610         }
611         // NOTE(pishin, msherstennikov): revisit
612         // Method (or the whole class?) may already have a definition in another file,
613         // in this case it should not be added into AOT file.
614         Method &method = methodDataAccessor.IsStatic() ? methods[smethodIdx++] : methods[vmethodIdx++];
615         auto methodName = runtime_->GetMethodFullName(&method, false);
616         if (method.GetPandaFile()->GetFilename() == pfileRef.GetFilename() && !Skip(&method) &&
617             IsMethodInList(methodName) && g_options.MatchesRegex(methodName) && !Compile(&method, methodIndex)) {
618             errorOccurred = true;
619         }
620         methodIndex++;
621     });
622 
623     if (IsAotMode()) {
624         aotBuilder_->EndClass();
625     }
626 
627     return !errorOccurred;
628 }
629 
Compile(Method * method,size_t methodIndex)630 bool Paoc::Compile(Method *method, size_t methodIndex)
631 {
632     if (method == nullptr) {
633         LOG_PAOC(WARNING) << "Method is nullptr";
634         return false;
635     }
636 #ifndef NDEBUG
637     const auto *pfile = method->GetPandaFile();
638     auto methodId = method->GetFileId();
639     ASSERT(pfile != nullptr);
640     ASSERT(!pfile->IsExternal(methodId));
641 #endif
642 
643     if (method->IsAbstract() || method->IsNative() || method->IsIntrinsic()) {
644         return true;
645     }
646     auto methodName = runtime_->GetMethodFullName(method, false);
647 
648     ++compilationIndex_;
649     LOG_PAOC(INFO) << "[" << compilationIndex_ << "] Compile method (id=" << method->GetFileId() << "): " << methodName;
650 
651     CompilingContext ctx(method, methodIndex, &statisticsDump_);
652 
653     PaocClusters::ScopedApplySpecialOptions to(methodName, &clustersInfo_);
654     switch (mode_) {
655         case PaocMode::AOT:
656         case PaocMode::LLVM:
657             if (!CompileAot(&ctx)) {
658                 EVENT_COMPILATION(methodName, false, ctx.method->GetCodeSize(), 0, 0, 0,
659                                   events::CompilationStatus::FAILED);
660                 failedMethods_++;
661                 ctx.compilationStatus = false;
662             } else {
663                 successMethods_++;
664             }
665             break;
666         case PaocMode::JIT:
667             ctx.compilationStatus = CompileJit(&ctx);
668             break;
669         case PaocMode::OSR:
670             ctx.compilationStatus = CompileOsr(&ctx);
671             break;
672         default:
673             UNREACHABLE();
674     }
675     return ctx.compilationStatus;
676 }
677 
CompileInGraph(CompilingContext * ctx,std::string methodName,bool isOsr)678 bool Paoc::CompileInGraph(CompilingContext *ctx, std::string methodName, bool isOsr)
679 {
680     compiler::InPlaceCompilerTaskRunner taskRunner;
681     auto &taskCtx = taskRunner.GetContext();
682     taskCtx.SetMethod(ctx->method);
683     taskCtx.SetOsr(isOsr);
684     taskCtx.SetAllocator(&ctx->allocator);
685     taskCtx.SetLocalAllocator(&ctx->graphLocalAllocator);
686     taskCtx.SetMethodName(std::move(methodName));
687     taskRunner.AddFinalize(
688         [&graph = ctx->graph](InPlaceCompilerContext &compilerCtx) { graph = compilerCtx.GetGraph(); });
689 
690     bool success = true;
691     taskRunner.AddCallbackOnFail([&success]([[maybe_unused]] InPlaceCompilerContext &compilerCtx) { success = false; });
692     auto arch = ChooseArch(Arch::NONE);
693     bool isDynamic = ark::panda_file::IsDynamicLanguage(ctx->method->GetClass()->GetSourceLang());
694     compiler::CompileInGraph<compiler::INPLACE_MODE>(runtime_, isDynamic, arch, std::move(taskRunner));
695     return success;
696 }
697 
698 /**
699  * Compiles a method in JIT mode (i.e. no code generated).
700  * @return `false` on error.
701  */
CompileJit(CompilingContext * ctx)702 bool Paoc::CompileJit(CompilingContext *ctx)
703 {
704     ASSERT(ctx != nullptr);
705     ASSERT(mode_ == PaocMode::JIT);
706     auto name = runtime_->GetMethodFullName(ctx->method, false);
707     if (!CompileInGraph(ctx, name, false)) {
708         std::string errMsg = "Failed to JIT-compile method: " + name;
709         PrintError(errMsg);
710         return false;
711     }
712     EVENT_COMPILATION(name, false, ctx->method->GetCodeSize(), 0, 0, 0, events::CompilationStatus::COMPILED);
713     return true;
714 }
715 
716 /**
717  * Compiles a method in OSR mode.
718  * @return `false` on error.
719  */
CompileOsr(CompilingContext * ctx)720 bool Paoc::CompileOsr(CompilingContext *ctx)
721 {
722     ASSERT(ctx != nullptr);
723     ASSERT(mode_ == PaocMode::OSR);
724     auto name = runtime_->GetMethodFullName(ctx->method, false);
725     if (!CompileInGraph(ctx, name, true)) {
726         std::string errMsg = "Failed to OSR-compile method: " + name;
727         PrintError(errMsg);
728         return false;
729     }
730     EVENT_COMPILATION(name, true, ctx->method->GetCodeSize(), 0, 0, 0, events::CompilationStatus::COMPILED);
731     return true;
732 }
733 
TryCreateGraph(CompilingContext * ctx)734 bool Paoc::TryCreateGraph(CompilingContext *ctx)
735 {
736     auto sourceLang = ctx->method->GetClass()->GetSourceLang();
737     bool isDynamic = ark::panda_file::IsDynamicLanguage(sourceLang);
738 
739     ctx->graph = ctx->allocator.New<Graph>(&ctx->allocator, &ctx->graphLocalAllocator, aotBuilder_->GetArch(),
740                                            ctx->method, runtime_, false, nullptr, isDynamic);
741     if (ctx->graph == nullptr) {
742         PrintError("Graph creation failed!");
743         return false;
744     }
745     ctx->graph->SetLanguage(sourceLang);
746     return true;
747 }
748 
FinalizeCompileAot(CompilingContext * ctx,uintptr_t codeAddress)749 bool Paoc::FinalizeCompileAot(CompilingContext *ctx, [[maybe_unused]] uintptr_t codeAddress)
750 {
751     CompiledMethod compiledMethod(ctx->graph->GetArch(), ctx->method, ctx->index);
752     compiledMethod.SetCode(ctx->graph->GetCode().ToConst());
753     compiledMethod.SetCodeInfo(ctx->graph->GetCodeInfoData());
754 #ifdef PANDA_COMPILER_DEBUG_INFO
755     compiledMethod.SetCfiInfo(ctx->graph->GetCallingConvention()->GetCfiInfo());
756 #endif
757     if (ctx->graph->GetCode().empty() || ctx->graph->GetCodeInfoData().empty()) {
758         LOG(INFO, COMPILER) << "Emit code failed";
759         return false;
760     }
761 
762     LOG(INFO, COMPILER) << "Ark AOT successfully compiled method: " << runtime_->GetMethodFullName(ctx->method, false);
763     EVENT_PAOC("Compiling " + runtime_->GetMethodFullName(ctx->method, false) + " using panda");
764     ASSERT(ctx->graph != nullptr);
765 
766     aotBuilder_->AddMethod(compiledMethod);
767 
768     EVENT_COMPILATION(runtime_->GetMethodFullName(ctx->method, false), false, ctx->method->GetCodeSize(), codeAddress,
769                       compiledMethod.GetCode().size(), compiledMethod.GetCodeInfo().size(),
770                       events::CompilationStatus::COMPILED);
771     return true;
772 }
773 
RunOptimizations(CompilingContext * ctx)774 bool Paoc::RunOptimizations(CompilingContext *ctx)
775 {
776     compiler::InPlaceCompilerTaskRunner taskRunner;
777     taskRunner.GetContext().SetGraph(ctx->graph);
778     bool success = true;
779     taskRunner.AddCallbackOnFail([&success]([[maybe_unused]] InPlaceCompilerContext &compilerCtx) { success = false; });
780 
781     compiler::RunOptimizations<compiler::INPLACE_MODE>(std::move(taskRunner));
782     return success;
783 }
784 
785 /**
786  * Compiles a method in AOT mode.
787  * @return `false` on error.
788  */
CompileAot(CompilingContext * ctx)789 bool Paoc::CompileAot(CompilingContext *ctx)
790 {
791     ASSERT(ctx != nullptr);
792     ASSERT(IsAotMode());
793 
794     LOG_IF(IsLLVMAotMode() && !paocOptions_->IsPaocUseCha(), FATAL, COMPILER)
795         << "LLVM AOT compiler supports only --paoc-use-cha=true";
796 
797     std::string className = ClassHelper::GetName(ctx->method->GetClassName().data);
798     if (runtimeOptions_->WasSetEventsOutput()) {
799         EVENT_PAOC("Trying to compile method: " + className +
800                    "::" + reinterpret_cast<const char *>(ctx->method->GetName().data));
801     }
802 
803     if (!TryCreateGraph(ctx)) {
804         return false;
805     }
806 
807     uintptr_t codeAddress = aotBuilder_->GetCurrentCodeAddress();
808     auto aotData = ctx->graph->GetAllocator()->New<AotData>(
809         ctx->method->GetPandaFile(), ctx->graph, codeAddress, aotBuilder_->GetIntfInlineCacheIndex(),
810         aotBuilder_->GetGotPlt(), aotBuilder_->GetGotVirtIndexes(), aotBuilder_->GetGotClass(),
811         aotBuilder_->GetGotString(), aotBuilder_->GetGotIntfInlineCache(), aotBuilder_->GetGotCommon(), slowPathData_);
812 
813     aotData->SetUseCha(paocOptions_->IsPaocUseCha());
814     ctx->graph->SetAotData(aotData);
815 
816     if (!ctx->graph->RunPass<IrBuilder>()) {
817         PrintError("IrBuilder failed!");
818         return false;
819     }
820 
821     if (IsLLVMAotMode()) {
822         auto result = TryLLVM(ctx);
823         switch (result) {
824             case LLVMCompilerStatus::COMPILED:
825                 return true;
826             case LLVMCompilerStatus::SKIP:
827                 return false;
828             case LLVMCompilerStatus::ERROR:
829                 LOG_PAOC(FATAL) << "LLVM AOT failed (unknown instruction)";
830                 break;
831             case LLVMCompilerStatus::FALLBACK:
832                 // Fallback to Ark Compiler AOT compilation
833                 break;
834             default:
835                 UNREACHABLE();
836                 break;
837         }
838         LOG_PAOC(INFO) << "LLVM fallback to ARK AOT on method: " << runtime_->GetMethodFullName(ctx->method, false);
839     }
840 
841     if (!RunOptimizations(ctx)) {
842         PrintError("RunOptimizations failed!");
843         return false;
844     }
845 
846     return FinalizeCompileAot(ctx, codeAddress);
847 }
848 
PrintError(const std::string & error)849 void Paoc::PrintError(const std::string &error)
850 {
851     if (ShouldIgnoreFailures()) {
852         LOG_PAOC(WARNING) << error;
853     } else {
854         LOG_PAOC(ERROR) << error;
855     }
856 }
857 
ShouldIgnoreFailures()858 bool Paoc::ShouldIgnoreFailures()
859 {
860     return compiler::g_options.IsCompilerIgnoreFailures();
861 }
862 
PrintUsage(const ark::PandArgParser & paParser)863 void Paoc::PrintUsage(const ark::PandArgParser &paParser)
864 {
865     std::cerr << "Usage: ark_aot [OPTIONS] --paoc-panda-files <list>\n";
866     std::cerr << "    --paoc-panda-files          list of input panda files, it is a mandatory option\n";
867     std::cerr << "    --paoc-mode                 specify compilation mode (aot, llvm, jit or osr) \n";
868     std::cerr << "    --paoc-output               path to output file, default is out.an\n";
869     std::cerr << "    --paoc-location             location path of the input panda file, that will be written into"
870                  " the AOT file\n";
871     std::cerr << "    --paoc-skip-until           first method to compile, skip all previous\n";
872     std::cerr << "    --paoc-compile-until        last method to compile, skip all following\n";
873     std::cerr << "    --paoc-methods-from-file    path to file which contains methods to compile\n";
874 #ifndef NDEBUG
875     std::cerr << "    --paoc-generate-symbols     generate symbols for compiled methods, disabled by default\n";
876 #endif
877     std::cerr << "    --compiler-ignore-failures  ignore failures in methods/classes/files compilation\n";
878     std::cerr << " You can also use other Ark compiler options\n";
879 
880     std::cerr << paParser.GetHelpString() << std::endl;
881 }
882 
IsMethodInList(const std::string & methodFullName)883 bool Paoc::IsMethodInList(const std::string &methodFullName)
884 {
885     return !paocOptions_->WasSetPaocMethodsFromFile() || (methodsList_.find(methodFullName) != methodsList_.end());
886 }
887 
888 /*
889  * Check if needed to skip method, considering  'paoc-skip-until' and 'paoc-compile-until' options
890  */
Skip(Method * method)891 bool Paoc::Skip(Method *method)
892 {
893     if (method == nullptr) {
894         return true;
895     }
896 
897     auto methodName = runtime_->GetMethodFullName(method, false);
898     if (!skipInfo_.isFirstCompiled) {
899         if (methodName == paocOptions_->GetPaocSkipUntil()) {
900             skipInfo_.isFirstCompiled = true;
901         } else {
902             return true;
903         }
904     }
905     if (skipInfo_.isLastCompiled) {
906         return true;
907     }
908     if (paocOptions_->WasSetPaocCompileUntil() && methodName == paocOptions_->GetPaocCompileUntil()) {
909         skipInfo_.isLastCompiled = true;
910     }
911     return false;
912 }
913 
GetFileLocation(const panda_file::File & pfileRef,std::string location)914 std::string Paoc::GetFileLocation(const panda_file::File &pfileRef, std::string location)
915 {
916     auto &filename = pfileRef.GetFilename();
917     if (auto pos = filename.rfind('/'); pos != std::string::npos) {
918         if (location.back() == '/') {
919             pos++;
920         }
921         location += filename.substr(pos);
922     } else {
923         location += '/' + filename;
924     }
925     return location;
926 }
927 
CompareBootFiles(std::string filename,std::string paocLocation)928 bool Paoc::CompareBootFiles(std::string filename, std::string paocLocation)
929 {
930     if (auto pos = filename.rfind('/'); pos != std::string::npos) {
931         filename = filename.substr(++pos);
932     }
933     if (auto pos = paocLocation.rfind('/'); pos != std::string::npos) {
934         paocLocation = paocLocation.substr(++pos);
935     }
936 
937     return paocLocation == filename;
938 }
939 
LoadPandaFiles()940 bool Paoc::LoadPandaFiles()
941 {
942     bool errorOccurred = false;
943     auto *vm = ark::Runtime::GetCurrent()->GetPandaVM();
944     auto pfiles = runtimeOptions_->GetPandaFiles();
945     for (auto &fileName : pfiles) {
946         auto pfile = vm->OpenPandaFile(fileName);
947         if (!pfile) {
948             errorOccurred = true;
949             if (!ShouldIgnoreFailures()) {
950                 LOG_PAOC(FATAL) << "Can not open file: " << fileName;
951             }
952             LOG_PAOC(WARNING) << "Can not open file: " << fileName;
953             continue;
954         }
955 
956         if (!paocOptions_->GetPaocLocation().empty()) {
957             std::string filename = GetFileLocation(*pfile, paocOptions_->GetPaocLocation());
958             locationMapping_[pfile->GetFilename()] = filename;
959         }
960 
961         preloadedFiles_[GetFilePath(fileName)] = pfile.get();
962         loader_->AddPandaFile(std::move(pfile));
963     }
964     return !errorOccurred;
965 }
966 
BuildClassHashTable(const panda_file::File & pfileRef)967 void Paoc::BuildClassHashTable(const panda_file::File &pfileRef)
968 {
969     if (!pfileRef.GetClasses().empty()) {
970         aotBuilder_->AddClassHashTable(pfileRef);
971     }
972 }
973 
974 #undef LOG_PAOC
975 
976 }  // namespace ark::paoc
977 
main(int argc,const char * argv[])978 int main(int argc, const char *argv[])
979 {
980 #ifdef PANDA_LLVM_AOT
981     ark::paoc::PaocLLVM paoc;
982 #else
983     ark::paoc::Paoc paoc;
984 #endif
985     ark::Span<const char *> args(argv, argc);
986     return paoc.Run(args);
987 }
988