OpenHarmony-v5.0.1-Release/s

/*
 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "optimizer_run.h"
#include "compile_method.h"
#include "mem/pool_manager.h"
#include "mem/code_allocator.h"
#include "include/class.h"
#include "include/method.h"
#include "optimizer/ir/ir_constructor.h"
#include "optimizer/ir/runtime_interface.h"
#include "optimizer/analysis/loop_analyzer.h"
#include "optimizer/pass.h"
#include "optimizer/ir_builder/ir_builder.h"
#include "utils/logger.h"
#include "code_info/code_info.h"
#include "events/events.h"
#include "trace/trace.h"
#include "optimizer/optimizations/regalloc/reg_alloc_linear_scan.h"
#include "optimizer/code_generator/codegen.h"

namespace ark::compiler {

#ifdef PANDA_COMPILER_DEBUG_INFO
static Span<uint8_t> EmitElf(Graph *graph, CodeAllocator *codeAllocator, ArenaAllocator *gdbDebugInfoAllocator,
                             const std::string &methodName);
#endif

void JITStats::SetCompilationStart()
{
    ASSERT(startTime_ == 0);
    startTime_ = time::GetCurrentTimeInNanos();
}
void JITStats::EndCompilationWithStats(const std::string &methodName, bool isOsr, size_t bcSize, size_t codeSize)
{
    ASSERT(startTime_ != 0);
    auto time = time::GetCurrentTimeInNanos() - startTime_;
    statsList_.push_back(Entry {PandaString(methodName, internalAllocator_->Adapter()), isOsr, bcSize, codeSize, time});
    startTime_ = 0;
}

void JITStats::ResetCompilationStart()
{
    startTime_ = 0;
}

void JITStats::DumpCsv(char sep)
{
    ASSERT(g_options.WasSetCompilerDumpJitStatsCsv());
    std::ofstream csv(g_options.GetCompilerDumpJitStatsCsv(), std::ofstream::trunc);
    for (const auto &i : statsList_) {
        csv << "\"" << i.methodName << "\"" << sep;
        csv << i.isOsr << sep;
        csv << i.bcSize << sep;
        csv << i.codeSize << sep;
        csv << i.time;
        csv << '\n';
    }
}

static void EndCompilation(const std::string &methodName, bool isOsr, size_t bcSize, [[maybe_unused]] uintptr_t address,
                           size_t codeSize, [[maybe_unused]] size_t infoSize,
                           [[maybe_unused]] events::CompilationStatus status, JITStats *jitStats)
{
    EVENT_COMPILATION(methodName, isOsr, bcSize, address, codeSize, infoSize, status);
    if (jitStats != nullptr) {
        ASSERT((codeSize != 0) == (status == events::CompilationStatus::COMPILED));
        jitStats->EndCompilationWithStats(methodName, isOsr, bcSize, codeSize);
    }
}

Arch ChooseArch(Arch arch)
{
    if (arch != Arch::NONE) {
        return arch;
    }

    arch = RUNTIME_ARCH;
    if (RUNTIME_ARCH == Arch::X86_64 && g_options.WasSetCompilerCrossArch()) {
        arch = GetArchFromString(g_options.GetCompilerCrossArch());
    }

    return arch;
}

static bool CheckSingleImplementation(Graph *graph)
{
    // Check that all methods that were inlined due to its single implementation property, still have this property,
    // otherwise we must drop compiled code.
    // NOTE(compiler): we need to reset hotness counter hereby avoid yet another warmup phase.
    auto cha = graph->GetRuntime()->GetCha();
    for (auto siMethod : graph->GetSingleImplementationList()) {
        if (!cha->IsSingleImplementation(siMethod)) {
            LOG(WARNING, COMPILER)
                << "Method lost single-implementation property after compilation, so we need to drop "
                   "whole compiled code: "
                << graph->GetRuntime()->GetMethodFullName(siMethod);
            return false;
        }
    }
    return true;
}

static Span<uint8_t> EmitCode(const Graph *graph, CodeAllocator *allocator)
{
    size_t codeOffset = RoundUp(CodePrefix::STRUCT_SIZE, GetCodeAlignment(graph->GetArch()));
    CodePrefix prefix;
    prefix.codeSize = graph->GetCode().size();
    prefix.codeInfoOffset = codeOffset + RoundUp(graph->GetCode().size(), sizeof(uint32_t));
    prefix.codeInfoSize = graph->GetCodeInfoData().size();
    size_t codeSize = prefix.codeInfoOffset + prefix.codeInfoSize;
    auto memRange = allocator->AllocateCodeUnprotected(codeSize);
    if (memRange.GetSize() == 0) {
        return Span<uint8_t> {};
    }

    auto data = reinterpret_cast<uint8_t *>(memRange.GetData());
    memcpy_s(data, sizeof(CodePrefix), &prefix, sizeof(CodePrefix));
    // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
    memcpy_s(&data[codeOffset], graph->GetCode().size(), graph->GetCode().data(), graph->GetCode().size());
    // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
    memcpy_s(&data[prefix.codeInfoOffset], graph->GetCodeInfoData().size(), graph->GetCodeInfoData().data(),
             graph->GetCodeInfoData().size());

    allocator->ProtectCode(memRange);

    return Span<uint8_t>(reinterpret_cast<uint8_t *>(memRange.GetData()), codeSize);
}

static uint8_t *GetEntryPoint(Graph *graph, [[maybe_unused]] Method *method, const std::string &methodName,
                              [[maybe_unused]] bool isOsr, CodeAllocator *codeAllocator,
                              [[maybe_unused]] ArenaAllocator *gdbDebugInfoAllocator, JITStats *jitStats)
{
#ifdef PANDA_COMPILER_DEBUG_INFO
    auto generatedData = g_options.IsCompilerEmitDebugInfo()
                             ? EmitElf(graph, codeAllocator, gdbDebugInfoAllocator, methodName)
                             : EmitCode(graph, codeAllocator);
#else
    auto generatedData = EmitCode(graph, codeAllocator);
#endif
    if (generatedData.Empty()) {
        LOG(INFO, COMPILER) << "Compilation failed due to memory allocation fail: " << methodName;
        return nullptr;
    }
    CodeInfo codeInfo(generatedData);
    LOG(INFO, COMPILER) << "Compiled code for '" << methodName << "' has been installed to "
                        << bit_cast<void *>(codeInfo.GetCode()) << ", code size " << codeInfo.GetCodeSize();

    auto entryPoint = const_cast<uint8_t *>(codeInfo.GetCode());
    EndCompilation(methodName, isOsr, method->GetCodeSize(), reinterpret_cast<uintptr_t>(entryPoint),
                   codeInfo.GetCodeSize(), codeInfo.GetInfoSize(), events::CompilationStatus::COMPILED, jitStats);
    return entryPoint;
}

template <TaskRunnerMode RUNNER_MODE>
static void RunOptimizations(CompilerTaskRunner<RUNNER_MODE> taskRunner, JITStats *jitStats)
{
    auto &taskCtx = taskRunner.GetContext();
    taskCtx.GetGraph()->SetLanguage(taskCtx.GetMethod()->GetClass()->GetSourceLang());

    taskRunner.AddCallbackOnSuccess([]([[maybe_unused]] CompilerContext<RUNNER_MODE> &compilerCtx) {
        LOG(DEBUG, COMPILER) << "The method " << compilerCtx.GetMethodName() << " is compiled";
    });
    taskRunner.AddCallbackOnFail([jitStats](CompilerContext<RUNNER_MODE> &compilerCtx) {
        if (!compiler::g_options.IsCompilerIgnoreFailures()) {
            LOG(FATAL, COMPILER) << "RunOptimizations failed!";
        }
        LOG(WARNING, COMPILER) << "RunOptimizations failed!";
        EndCompilation(compilerCtx.GetMethodName(), compilerCtx.IsOsr(), compilerCtx.GetMethod()->GetCodeSize(), 0, 0,
                       0, events::CompilationStatus::FAILED, jitStats);
    });

    // Run compiler optimizations over created graph
    RunOptimizations<RUNNER_MODE>(std::move(taskRunner));
}

template <TaskRunnerMode RUNNER_MODE>
static bool CheckCompilation(RuntimeInterface *runtime, CodeAllocator *codeAllocator,
                             ArenaAllocator *gdbDebugInfoAllocator, JITStats *jitStats, bool isDynamic, Arch arch,
                             CompilerContext<RUNNER_MODE> &compilerCtx)
{
    auto *graph = compilerCtx.GetGraph();

    ASSERT(graph != nullptr && graph->GetCode().data() != nullptr);

    auto &name = compilerCtx.GetMethodName();
    auto isOsr = compilerCtx.IsOsr();
    auto *method = compilerCtx.GetMethod();

    if (!isDynamic && !CheckSingleImplementation(graph)) {
        EndCompilation(name, isOsr, method->GetCodeSize(), 0, 0, 0, events::CompilationStatus::FAILED_SINGLE_IMPL,
                       jitStats);
        return false;
    }

    // Drop non-native code in any case
    if (arch != RUNTIME_ARCH) {
        EndCompilation(name, isOsr, method->GetCodeSize(), 0, 0, 0, events::CompilationStatus::DROPPED, jitStats);
        return false;
    }

    auto entryPoint = GetEntryPoint(graph, method, name, isOsr, codeAllocator, gdbDebugInfoAllocator, jitStats);
    if (entryPoint == nullptr) {
        return false;
    }
    if (isOsr) {
        if (!runtime->TrySetOsrCode(method, entryPoint)) {
            // Compiled code has been deoptimized, so we shouldn't install osr code.
            // NOTE(compiler): release compiled code memory, when CodeAllocator supports freeing the memory.
            return false;
        }
    } else {
        runtime->SetCompiledEntryPoint(method, entryPoint);
    }
    return true;
}

template <TaskRunnerMode RUNNER_MODE>
void JITCompileMethod(RuntimeInterface *runtime, CodeAllocator *codeAllocator, ArenaAllocator *gdbDebugInfoAllocator,
                      JITStats *jitStats, CompilerTaskRunner<RUNNER_MODE> taskRunner)
{
    auto &taskCtx = taskRunner.GetContext();
    auto *taskMethod = taskCtx.GetMethod();
    taskCtx.SetMethodName(runtime->GetMethodFullName(taskMethod, false));
    auto &methodName = taskCtx.GetMethodName();

    SCOPED_TRACE_STREAM << "JIT compiling " << methodName;

    if (!g_options.MatchesRegex(methodName)) {
        LOG(DEBUG, COMPILER) << "Skip the method due to regexp mismatch: " << methodName;
        taskCtx.SetCompilationStatus(false);
        CompilerTaskRunner<RUNNER_MODE>::EndTask(std::move(taskRunner), false);
        return;
    }

    if (jitStats != nullptr) {
        jitStats->SetCompilationStart();
    }

    taskRunner.AddFinalize([jitStats](CompilerContext<RUNNER_MODE> &compilerCtx) {
        if (jitStats != nullptr) {
            // Reset compilation start time in all cases for consistency
            jitStats->ResetCompilationStart();
        }
        auto *graph = compilerCtx.GetGraph();
        if (graph != nullptr) {
            graph->~Graph();
        }
    });

    auto arch = ChooseArch(Arch::NONE);
    bool isDynamic = ark::panda_file::IsDynamicLanguage(taskMethod->GetClass()->GetSourceLang());

    taskRunner.AddCallbackOnSuccess([runtime, codeAllocator, gdbDebugInfoAllocator, jitStats, isDynamic,
                                     arch](CompilerContext<RUNNER_MODE> &compilerCtx) {
        bool compilationStatus = CheckCompilation<RUNNER_MODE>(runtime, codeAllocator, gdbDebugInfoAllocator, jitStats,
                                                               isDynamic, arch, compilerCtx);
        compilerCtx.SetCompilationStatus(compilationStatus);
    });
    taskRunner.AddCallbackOnFail(
        [](CompilerContext<RUNNER_MODE> &compilerCtx) { compilerCtx.SetCompilationStatus(false); });

    CompileInGraph<RUNNER_MODE>(runtime, isDynamic, arch, std::move(taskRunner), jitStats);
}

template <TaskRunnerMode RUNNER_MODE>
void CompileInGraph(RuntimeInterface *runtime, bool isDynamic, Arch arch, CompilerTaskRunner<RUNNER_MODE> taskRunner,
                    JITStats *jitStats)
{
    auto &taskCtx = taskRunner.GetContext();
    auto isOsr = taskCtx.IsOsr();
    auto *method = taskCtx.GetMethod();
    auto &methodName = taskCtx.GetMethodName();

    LOG(INFO, COMPILER) << "Compile method" << (isOsr ? "(OSR)" : "") << ": " << methodName << " ("
                        << runtime->GetFileName(method) << ')';

    if (arch == Arch::NONE || !BackendSupport(arch)) {
        LOG(DEBUG, COMPILER) << "Compilation unsupported for this platform!";
        CompilerTaskRunner<RUNNER_MODE>::EndTask(std::move(taskRunner), false);
        return;
    }

    auto *allocator = taskCtx.GetAllocator();
    auto *localAllocator = taskCtx.GetLocalAllocator();
    auto *graph =
        allocator->template New<Graph>(allocator, localAllocator, arch, method, runtime, isOsr, nullptr, isDynamic);
    taskCtx.SetGraph(graph);
    if (graph == nullptr) {
        LOG(ERROR, COMPILER) << "Creating graph failed!";
        EndCompilation(methodName, isOsr, method->GetCodeSize(), 0, 0, 0, events::CompilationStatus::FAILED, jitStats);
        CompilerTaskRunner<RUNNER_MODE>::EndTask(std::move(taskRunner), false);
        return;
    }

    taskRunner.SetTaskOnSuccess([jitStats](CompilerTaskRunner<RUNNER_MODE> nextRunner) {
        RunOptimizations<RUNNER_MODE>(std::move(nextRunner), jitStats);
    });

    bool success = graph->template RunPass<IrBuilder>();
    if (!success) {
        if (!compiler::g_options.IsCompilerIgnoreFailures()) {
            LOG(FATAL, COMPILER) << "IrBuilder failed!";
        }
        LOG(WARNING, COMPILER) << "IrBuilder failed!";
        EndCompilation(methodName, isOsr, method->GetCodeSize(), 0, 0, 0, events::CompilationStatus::FAILED, jitStats);
    };
    CompilerTaskRunner<RUNNER_MODE>::EndTask(std::move(taskRunner), success);
}

template void JITCompileMethod<BACKGROUND_MODE>(RuntimeInterface *, CodeAllocator *, ArenaAllocator *, JITStats *,
                                                CompilerTaskRunner<BACKGROUND_MODE>);
template void JITCompileMethod<INPLACE_MODE>(RuntimeInterface *, CodeAllocator *, ArenaAllocator *, JITStats *,
                                             CompilerTaskRunner<INPLACE_MODE>);
template void CompileInGraph<BACKGROUND_MODE>(RuntimeInterface *, bool, Arch, CompilerTaskRunner<BACKGROUND_MODE>,
                                              JITStats *);
template void CompileInGraph<INPLACE_MODE>(RuntimeInterface *, bool, Arch, CompilerTaskRunner<INPLACE_MODE>,
                                           JITStats *);
template void RunOptimizations<BACKGROUND_MODE>(CompilerTaskRunner<BACKGROUND_MODE>, JITStats *);
template void RunOptimizations<INPLACE_MODE>(CompilerTaskRunner<INPLACE_MODE>, JITStats *);
template bool CheckCompilation<BACKGROUND_MODE>(RuntimeInterface *, CodeAllocator *, ArenaAllocator *, JITStats *, bool,
                                                Arch, CompilerContext<BACKGROUND_MODE> &);
template bool CheckCompilation<INPLACE_MODE>(RuntimeInterface *, CodeAllocator *, ArenaAllocator *, JITStats *, bool,
                                             Arch, CompilerContext<INPLACE_MODE> &);

}  // namespace ark::compiler

#ifdef PANDA_COMPILER_DEBUG_INFO

#include "optimizer/ir/aot_data.h"
#include "tools/debug/jit_writer.h"

// Next "C"-code need for enable interaction with gdb
// Please read "JIT Compilation Interface" from gdb-documentation for more information
extern "C" {
// Gdb will replace implementation of this function
// NOLINTNEXTLINE(readability-identifier-naming)
void NO_INLINE __jit_debug_register_code(void)
{
    // NOLINTNEXTLINE(hicpp-no-assembler)
    asm("");
}

// Default version for descriptor (may be checked before register code)
// NOLINTNEXTLINE(modernize-use-nullptr, readability-identifier-naming)
jit_descriptor __jit_debug_descriptor = {1, JIT_NOACTION, NULL, NULL};
}  // extern "C"

namespace ark::compiler {

// NOLINTNEXTLINE(fuchsia-statically-constructed-objects)
static os::memory::Mutex g_jitDebugLock;

// Will register jit-elf description in linked list
static void RegisterJitCode(jit_code_entry *entry)
{
    ASSERT(g_options.IsCompilerEmitDebugInfo());

    os::memory::LockHolder lock(g_jitDebugLock);
    // Re-link list
    entry->nextEntry = __jit_debug_descriptor.firstEntry;
    if (__jit_debug_descriptor.firstEntry != nullptr) {
        __jit_debug_descriptor.firstEntry->prevEntry = entry;
    }
    __jit_debug_descriptor.firstEntry = entry;

    // Fill last entry
    __jit_debug_descriptor.relevantEntry = entry;
    __jit_debug_descriptor.actionFlag = JIT_REGISTER_FN;

    // Call gdb-callback
    __jit_debug_register_code();
    __jit_debug_descriptor.actionFlag = JIT_NOACTION;
    __jit_debug_descriptor.relevantEntry = nullptr;
}

// When code allocator cleaned - also will clean entry
void CleanJitDebugCode()
{
    ASSERT(g_options.IsCompilerEmitDebugInfo());

    os::memory::LockHolder lock(g_jitDebugLock);
    __jit_debug_descriptor.actionFlag = JIT_UNREGISTER_FN;

    while (__jit_debug_descriptor.firstEntry != nullptr) {
        __jit_debug_descriptor.firstEntry->prevEntry = nullptr;
        __jit_debug_descriptor.relevantEntry = __jit_debug_descriptor.firstEntry;
        // Call gdb-callback
        __jit_debug_register_code();

        __jit_debug_descriptor.firstEntry = __jit_debug_descriptor.firstEntry->nextEntry;
    }

    __jit_debug_descriptor.actionFlag = JIT_NOACTION;
    __jit_debug_descriptor.relevantEntry = nullptr;
}

// For each jit code - will generate small elf description and put them in gdb-special linked list.
static Span<uint8_t> EmitElf(Graph *graph, CodeAllocator *codeAllocator, ArenaAllocator *gdbDebugInfoAllocator,
                             const std::string &methodName)
{
    ASSERT(g_options.IsCompilerEmitDebugInfo());

    if (graph->GetCode().Empty()) {
        return {};
    }

    JitDebugWriter jitWriter(graph->GetArch(), graph->GetRuntime(), codeAllocator, methodName);

    jitWriter.Start();

    auto method = reinterpret_cast<Method *>(graph->GetMethod());
    auto klass = reinterpret_cast<Class *>(graph->GetRuntime()->GetClass(method));
    jitWriter.StartClass(*klass);

    CompiledMethod compiledMethod(graph->GetArch(), method, 0);
    compiledMethod.SetCode(graph->GetCode().ToConst());
    compiledMethod.SetCodeInfo(graph->GetCodeInfoData());
    compiledMethod.SetCfiInfo(graph->GetCallingConvention()->GetCfiInfo());

    jitWriter.AddMethod(compiledMethod);
    jitWriter.EndClass();
    jitWriter.End();
    if (!jitWriter.Write()) {
        return {};
    }

    auto gdbEntry {gdbDebugInfoAllocator->New<jit_code_entry>()};
    if (gdbEntry == nullptr) {
        return {};
    }

    auto elfFile {jitWriter.GetElf()};
    // Pointer to Elf-file entry
    gdbEntry->symfileAddr = reinterpret_cast<const char *>(elfFile.Data());
    // Elf-in-memory file size
    gdbEntry->symfileSize = elfFile.Size();
    gdbEntry->prevEntry = nullptr;

    RegisterJitCode(gdbEntry);
    return jitWriter.GetCode();
}

}  // namespace ark::compiler
#endif