1 /*
2 * Copyright (c) 2023-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include <unordered_map>
17
18 #include "transforms/gc_utils.h"
19 #include "transforms/passes/intrinsics_lowering.h"
20 #include "transforms/runtime_calls.h"
21 #include "transforms/transform_utils.h"
22 #include "llvm_ark_interface.h"
23
24 #include <llvm/Pass.h>
25 #include <llvm/IR/IntrinsicInst.h>
26 #include <llvm/IR/Intrinsics.h>
27 #include <llvm/IR/Module.h>
28 #include <llvm/IR/Verifier.h>
29 #include <llvm/Transforms/Utils/BasicBlockUtils.h>
30
31 using ark::llvmbackend::runtime_calls::GetPandaRuntimeFunctionCallee;
32 using llvm::Function;
33 using llvm::FunctionAnalysisManager;
34
35 namespace ark::llvmbackend::passes {
36
Create(LLVMArkInterface * arkInterface,const ark::llvmbackend::LLVMCompilerOptions * options)37 IntrinsicsLowering IntrinsicsLowering::Create(LLVMArkInterface *arkInterface,
38 [[maybe_unused]] const ark::llvmbackend::LLVMCompilerOptions *options)
39 {
40 return IntrinsicsLowering(arkInterface);
41 }
42
IntrinsicsLowering(LLVMArkInterface * arkInterface)43 IntrinsicsLowering::IntrinsicsLowering(LLVMArkInterface *arkInterface) : arkInterface_ {arkInterface} {}
44
run(Function & function,FunctionAnalysisManager &)45 llvm::PreservedAnalyses IntrinsicsLowering::run(Function &function, FunctionAnalysisManager & /*analysisManager*/)
46 {
47 ASSERT(arkInterface_ != nullptr);
48 if (gc_utils::IsFunctionSupplemental(function)) {
49 return llvm::PreservedAnalyses::all();
50 }
51 bool changed = false;
52 std::unordered_map<llvm::Instruction *, llvm::Instruction *> instToReplaceWithInst;
53 for (auto &block : function) {
54 for (auto &instruction : block) {
55 auto llvmIntrinsicId = arkInterface_->GetLLVMIntrinsicId(&instruction);
56 if (llvmIntrinsicId != llvm::Intrinsic::not_intrinsic) {
57 ASSERT(llvm::isa<llvm::CallInst>(&instruction));
58 changed |= ReplaceWithLLVMIntrinsic(llvm::cast<llvm::CallInst>(&instruction), llvmIntrinsicId);
59 continue;
60 }
61 auto intrinsicId = arkInterface_->GetIntrinsicId(&instruction);
62 if (intrinsicId == LLVMArkInterface::NO_INTRINSIC_ID) {
63 continue;
64 }
65
66 auto opcode = instruction.getOpcode();
67 if (opcode == llvm::Instruction::Call) {
68 changed |= HandleCall(llvm::cast<llvm::CallInst>(&instruction), intrinsicId, &instToReplaceWithInst);
69 } else if (opcode == llvm::Instruction::FRem) {
70 changed |= HandleFRem(&instruction, intrinsicId, &instToReplaceWithInst);
71 } else {
72 llvm_unreachable("Unexpected opcode while lowering intrinsics");
73 }
74 }
75 }
76 for (auto item : instToReplaceWithInst) {
77 llvm::ReplaceInstWithInst(item.first, item.second);
78 }
79 return changed ? llvm::PreservedAnalyses::none() : llvm::PreservedAnalyses::all();
80 }
ReplaceWithLLVMIntrinsic(llvm::CallInst * call,llvm::Intrinsic::ID intrinsicId)81 bool IntrinsicsLowering::ReplaceWithLLVMIntrinsic(llvm::CallInst *call, llvm::Intrinsic::ID intrinsicId)
82 {
83 ASSERT(intrinsicId == llvm::Intrinsic::memcpy_inline || intrinsicId == llvm::Intrinsic::memset_inline);
84 std::vector<llvm::Type *> argTypes;
85 auto functionType = call->getFunctionType();
86 if (intrinsicId == llvm::Intrinsic::memcpy_inline) {
87 // Skip the 4-th `isvolatile` arg
88 argTypes.push_back(functionType->getParamType(0U)); // Dst type
89 argTypes.push_back(functionType->getParamType(1U)); // Src type
90 argTypes.push_back(functionType->getParamType(2U)); // Size type
91 } else if (intrinsicId == llvm::Intrinsic::memset_inline) {
92 argTypes.push_back(functionType->getParamType(0U)); // Dst type
93 argTypes.push_back(functionType->getParamType(2U)); // Size type
94 } else {
95 llvm_unreachable("Attempt to insert unsupported llvm intrinsic");
96 }
97 auto *module = call->getModule();
98 auto intrinsicDecl = llvm::Intrinsic::getDeclaration(module, intrinsicId, argTypes);
99 call->setCalledFunction(intrinsicDecl);
100 return true;
101 }
102
HandleMemCall(llvm::CallInst * call,llvm::FunctionCallee callee,std::unordered_map<llvm::Instruction *,llvm::Instruction * > * instToReplaceWithInst)103 void IntrinsicsLowering::HandleMemCall(
104 llvm::CallInst *call, llvm::FunctionCallee callee,
105 std::unordered_map<llvm::Instruction *, llvm::Instruction *> *instToReplaceWithInst)
106 {
107 auto builder = llvm::IRBuilder<>(call);
108 static constexpr unsigned DEST = 0U;
109 static constexpr unsigned SRC_OR_CHAR = 1U;
110 static constexpr unsigned COUNT = 2U;
111
112 llvm::Value *op0 = call->getOperand(DEST);
113 llvm::Value *op1 = call->getOperand(SRC_OR_CHAR);
114 llvm::Value *op2 = call->getOperand(COUNT);
115
116 ASSERT(op0->getType()->isPointerTy());
117 if (op0->getType() != callee.getFunctionType()->getParamType(DEST)) {
118 op0 = builder.CreateAddrSpaceCast(op0, callee.getFunctionType()->getParamType(DEST));
119 }
120 if (op1->getType() != callee.getFunctionType()->getParamType(SRC_OR_CHAR)) {
121 ASSERT(op1->getType()->isPointerTy());
122 op1 = builder.CreateAddrSpaceCast(op1, callee.getFunctionType()->getParamType(SRC_OR_CHAR));
123 }
124
125 ASSERT(callee.getFunctionType()->getParamType(COUNT)->isIntegerTy());
126 auto realCountType = llvm::cast<llvm::IntegerType>(callee.getFunctionType()->getParamType(COUNT));
127 if (llvm::cast<llvm::IntegerType>(op2->getType())->getBitWidth() < realCountType->getBitWidth()) {
128 op2 = builder.CreateCast(llvm::Instruction::ZExt, op2, realCountType);
129 }
130 // Remove is_volatile last operand
131 auto newCall = llvm::CallInst::Create(callee, {op0, op1, op2});
132 instToReplaceWithInst->insert({call, newCall});
133 }
134
HandleCall(llvm::CallInst * call,LLVMArkInterface::IntrinsicId intrinsicId,std::unordered_map<llvm::Instruction *,llvm::Instruction * > * instToReplaceWithInst)135 bool IntrinsicsLowering::HandleCall(llvm::CallInst *call, LLVMArkInterface::IntrinsicId intrinsicId,
136 std::unordered_map<llvm::Instruction *, llvm::Instruction *> *instToReplaceWithInst)
137 {
138 llvm::StringRef intrinsicName = arkInterface_->GetIntrinsicRuntimeFunctionName(intrinsicId);
139 auto intrinsicFunctionTy = arkInterface_->GetRuntimeFunctionType(intrinsicName);
140
141 ASSERT(intrinsicFunctionTy != nullptr);
142 auto builder = llvm::IRBuilder<>(call);
143 auto type = call->getType();
144
145 auto callee = GetPandaRuntimeFunctionCallee(intrinsicId, intrinsicFunctionTy, &builder, intrinsicName);
146
147 if (type->isVectorTy()) {
148 llvm::Value *vec = llvm::UndefValue::get(type);
149 auto vecLen = llvm::cast<llvm::VectorType>(type)->getElementCount().getKnownMinValue();
150 std::vector<llvm::Value *> args;
151
152 for (uint64_t i = 0; i < vecLen; i++) {
153 for (auto &arg : call->args()) {
154 args.push_back(builder.CreateExtractElement(arg, i));
155 }
156
157 auto newCall = builder.CreateCall(callee, llvm::makeArrayRef(args));
158 if (i < vecLen - 1) {
159 vec = builder.CreateInsertElement(vec, newCall, i);
160 } else {
161 vec = llvm::InsertElementInst::Create(vec, newCall, builder.getInt64(i));
162 }
163
164 args.clear();
165 }
166
167 auto result = llvm::cast<llvm::Instruction>(vec);
168 instToReplaceWithInst->insert({call, result});
169 } else {
170 if (llvm::isa<llvm::MemCpyInst>(call) || llvm::isa<llvm::MemMoveInst>(call) ||
171 llvm::isa<llvm::MemSetInst>(call)) {
172 HandleMemCall(call, callee, instToReplaceWithInst);
173 } else {
174 call->setCalledFunction(callee);
175 }
176 }
177
178 return true;
179 }
180
HandleFRem(llvm::Instruction * inst,LLVMArkInterface::IntrinsicId intrinsicId,std::unordered_map<llvm::Instruction *,llvm::Instruction * > * instToReplaceWithInst)181 bool IntrinsicsLowering::HandleFRem(llvm::Instruction *inst, LLVMArkInterface::IntrinsicId intrinsicId,
182 std::unordered_map<llvm::Instruction *, llvm::Instruction *> *instToReplaceWithInst)
183 {
184 ASSERT(inst->getOpcode() == llvm::Instruction::FRem);
185 llvm::StringRef intrinsicName = arkInterface_->GetIntrinsicRuntimeFunctionName(intrinsicId);
186 auto intrinsicFunctionTy = arkInterface_->GetRuntimeFunctionType(intrinsicName);
187
188 auto arg1 = inst->getOperand(0U);
189 auto arg2 = inst->getOperand(1U);
190 auto module = inst->getModule();
191
192 auto builder = llvm::IRBuilder<>(inst);
193 auto table = module->getGlobalVariable("__aot_got");
194 auto arrayType = llvm::ArrayType::get(builder.getInt64Ty(), 0);
195 auto pointerToEntrypointAddress = builder.CreateConstInBoundsGEP2_32(arrayType, table, 0, intrinsicId);
196 auto entrypointAddress =
197 builder.CreateLoad(builder.getInt64Ty(), pointerToEntrypointAddress, intrinsicName + "_address");
198 auto calleePointer = builder.CreateIntToPtr(entrypointAddress, builder.getPtrTy(0));
199
200 auto call = llvm::CallInst::Create(intrinsicFunctionTy, calleePointer, {arg1, arg2});
201 instToReplaceWithInst->insert({inst, call});
202
203 return true;
204 }
205
206 } // namespace ark::llvmbackend::passes
207