1 /*
2 * Copyright (c) 2023-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "optimizer/code_generator/codegen.h"
17 #include "runtime/include/coretypes/string.h"
18
19 #include "llvm_ir_constructor.h"
20
21 #include "gc_barriers.h"
22 #include "irtoc_function_utils.h"
23 #include "llvm_logger.h"
24 #include "llvm_options.h"
25 #include "metadata.h"
26 #include "utils.h"
27 #include "transforms/builtins.h"
28 #include "transforms/gc_utils.h"
29 #include "transforms/runtime_calls.h"
30
31 namespace ark::compiler {
32 #define ONLY_NEEDSAFEPOINT
33 #include <intrinsics_ir_build.inl>
34 #undef ONLY_NEEDSAFEPOINT
35 } // namespace ark::compiler
36
37 #include <llvm/IR/InlineAsm.h>
38 #include <llvm/IR/IntrinsicsAArch64.h>
39 #include <llvm/IR/MDBuilder.h>
40 #include <llvm/IR/Verifier.h>
41 #include <llvm/Transforms/Utils/BasicBlockUtils.h>
42
43 using ark::llvmbackend::DebugDataBuilder;
44 using ark::llvmbackend::LLVMArkInterface;
45 using ark::llvmbackend::builtins::BarrierReturnVoid;
46 using ark::llvmbackend::builtins::KeepThis;
47 using ark::llvmbackend::builtins::LenArray;
48 using ark::llvmbackend::builtins::LoadClass;
49 using ark::llvmbackend::builtins::LoadInitClass;
50 using ark::llvmbackend::builtins::LoadString;
51 using ark::llvmbackend::builtins::ResolveVirtual;
52 using ark::llvmbackend::irtoc_function_utils::IsNoAliasIrtocFunction;
53 #ifndef NDEBUG
54 using ark::llvmbackend::irtoc_function_utils::IsPtrIgnIrtocFunction;
55 #endif
56 using ark::llvmbackend::utils::CreateLoadClassFromObject;
57
58 static constexpr unsigned VECTOR_SIZE_2 = 2;
59 static constexpr unsigned VECTOR_SIZE_8 = 8;
60 static constexpr unsigned VECTOR_SIZE_16 = 16;
61
62 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
63 #define ASSERT_TYPE(input, expectedType) \
64 ASSERT_DO((input)->getType() == (expectedType), \
65 std::cerr << "Unexpected data type: " << GetTypeName((input)->getType()) << ". Should be a " \
66 << GetTypeName(expectedType) << "." << std::endl)
67
68 // Max integer that can be represented in float/double without losing precision
MaxIntAsExactFloat()69 constexpr float MaxIntAsExactFloat()
70 {
71 return static_cast<float>((1U << static_cast<unsigned>(std::numeric_limits<float>::digits)) - 1);
72 }
73
MaxIntAsExactDouble()74 constexpr double MaxIntAsExactDouble()
75 {
76 return static_cast<double>((1ULL << static_cast<unsigned>(std::numeric_limits<double>::digits)) - 1);
77 }
78
79 // arm64: { dispatch: 24, pc: 20, frame: 23, acc: 21, accTag: 22, moffset: 25, methodPtr: 26 },
80 static constexpr auto AARCH64_PC = 20;
81 static constexpr auto AARCH64_ACC = 21;
82 static constexpr auto AARCH64_ACC_TAG = 22;
83 static constexpr auto AARCH64_FP = 23;
84 static constexpr auto AARCH64_DISPATCH = 24;
85 static constexpr auto AARCH64_MOFFSET = 25;
86 static constexpr auto AARCH64_METHOD_PTR = 26;
87 static constexpr auto AARCH64_REAL_FP = 29;
88
89 // x86_64: { dispatch: 8, pc: 4, frame: 5, acc: 11, accTag: 3 }
90 static constexpr auto X86_64_PC = 4; // renamed r10
91 static constexpr auto X86_64_ACC = 11; // renamed r3 (rbx)
92 static constexpr auto X86_64_ACC_TAG = 3; // renamed r11
93 static constexpr auto X86_64_FP = 5; // renamed r9
94 static constexpr auto X86_64_DISPATCH = 8;
95 static constexpr auto X86_64_REAL_FP = 9; // renamed r5 (rbp)
96
97 namespace {
CreateFunctionDeclaration(llvm::FunctionType * functionType,const std::string & name,llvm::Module * module)98 inline llvm::Function *CreateFunctionDeclaration(llvm::FunctionType *functionType, const std::string &name,
99 llvm::Module *module)
100 {
101 ASSERT(functionType != nullptr);
102 ASSERT(!name.empty());
103 ASSERT(module != nullptr);
104
105 auto function = module->getFunction(name);
106 if (function != nullptr) {
107 ASSERT(function->getVisibility() == llvm::GlobalValue::ProtectedVisibility);
108 ASSERT(function->doesNotThrow());
109 return function;
110 }
111
112 function = llvm::Function::Create(functionType, llvm::Function::ExternalLinkage, name, module);
113 function->setDoesNotThrow();
114 function->setVisibility(llvm::GlobalValue::ProtectedVisibility);
115 function->setSectionPrefix(name);
116
117 return function;
118 }
119
CreateBlackBoxAsm(llvm::IRBuilder<> * builder,const std::string & inlineAsm)120 inline void CreateBlackBoxAsm(llvm::IRBuilder<> *builder, const std::string &inlineAsm)
121 {
122 auto iasmType = llvm::FunctionType::get(builder->getVoidTy(), {}, false);
123 builder->CreateCall(iasmType, llvm::InlineAsm::get(iasmType, inlineAsm, "", true), {});
124 }
125
CreateInt32ImmAsm(llvm::IRBuilder<> * builder,const std::string & inlineAsm,uint32_t imm)126 inline void CreateInt32ImmAsm(llvm::IRBuilder<> *builder, const std::string &inlineAsm, uint32_t imm)
127 {
128 auto oneInt = llvm::FunctionType::get(builder->getVoidTy(), {builder->getInt32Ty()}, false);
129 builder->CreateCall(oneInt, llvm::InlineAsm::get(oneInt, inlineAsm, "i", true), {builder->getInt32(imm)});
130 }
131
ToAtomicOrdering(bool isVolatile)132 inline llvm::AtomicOrdering ToAtomicOrdering(bool isVolatile)
133 {
134 return isVolatile ? LLVMArkInterface::VOLATILE_ORDER : LLVMArkInterface::NOT_ATOMIC_ORDER;
135 }
136
137 #ifndef NDEBUG
GetTypeName(llvm::Type * type)138 inline std::string GetTypeName(llvm::Type *type)
139 {
140 std::string name;
141 auto stream = llvm::raw_string_ostream(name);
142 type->print(stream);
143 return stream.str();
144 }
145 #endif
146 } // namespace
147
148 namespace ark::compiler {
149
150 #include <can_compile_intrinsics_gen.inl>
151
152 class MemCharSimdLowering {
153 public:
154 MemCharSimdLowering(MemCharSimdLowering &&) = delete;
155 MemCharSimdLowering(const MemCharSimdLowering &) = delete;
156 MemCharSimdLowering &operator=(const MemCharSimdLowering &) = delete;
157 MemCharSimdLowering &operator=(MemCharSimdLowering &&) = delete;
158 MemCharSimdLowering() = delete;
159 ~MemCharSimdLowering() = default;
160
161 MemCharSimdLowering(llvm::Value *ch, llvm::Value *addr, llvm::IRBuilder<> *builder, llvm::Function *func);
162
163 template <bool MEM_BLOCK_SIZE_256_BITS>
164 llvm::Value *Generate(llvm::VectorType *vecTy);
165
GetU64X2Ty() const166 llvm::VectorType *GetU64X2Ty() const
167 {
168 return llvm::VectorType::get(builder_->getInt64Ty(), VECTOR_SIZE_2, false);
169 }
GetU16X8Ty() const170 llvm::VectorType *GetU16X8Ty() const
171 {
172 return llvm::VectorType::get(builder_->getInt16Ty(), VECTOR_SIZE_8, false);
173 }
GetU8X16Ty() const174 llvm::VectorType *GetU8X16Ty() const
175 {
176 return llvm::VectorType::get(builder_->getInt8Ty(), VECTOR_SIZE_16, false);
177 }
178
179 private:
180 static const uint64_t UL64 = 64UL;
181 static const uint64_t UL128 = 128UL;
182 static const uint64_t UL192 = 192UL;
183
184 void GenLoadAndFastCheck128(llvm::VectorType *vecTy);
185 void GenLoadAndFastCheck256(llvm::VectorType *vecTy);
186 llvm::Value *GenFindChar128(llvm::IntegerType *charTy);
187 llvm::Value *GenFindChar256(llvm::IntegerType *charTy);
ShuffleMask(llvm::Type * charTy)188 static llvm::SmallVector<int> ShuffleMask(llvm::Type *charTy)
189 {
190 ASSERT(charTy->isIntegerTy(8U) || charTy->isIntegerTy(16U));
191 static constexpr std::initializer_list<int> MASK_U8 {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
192 static constexpr std::initializer_list<int> MASK_U16 {7, 6, 5, 4, 3, 2, 1, 0};
193 return {charTy->isIntegerTy(8U) ? MASK_U8 : MASK_U16};
194 }
195
196 private:
197 llvm::Value *ch_;
198 llvm::Value *addr_;
199 llvm::IRBuilder<> *builder_;
200 llvm::Function *func_;
201
202 llvm::BasicBlock *lastBb_ = nullptr;
203 llvm::BasicBlock *foundBb_ = nullptr;
204 llvm::BasicBlock *inspectV1D0Bb_ = nullptr;
205 llvm::BasicBlock *inspectV1D1Bb_ = nullptr;
206 llvm::BasicBlock *inspectV2D0Bb_ = nullptr;
207 llvm::BasicBlock *inspectV2D1Bb_ = nullptr;
208 llvm::BasicBlock *clzV1D0Bb_ = nullptr;
209 llvm::BasicBlock *clzV1D1Bb_ = nullptr;
210 llvm::BasicBlock *clzV2D0Bb_ = nullptr;
211 llvm::Value *vcmpeq1_ = nullptr;
212 llvm::Value *vcmpeq2_ = nullptr;
213 };
214
MemCharSimdLowering(llvm::Value * ch,llvm::Value * addr,llvm::IRBuilder<> * builder,llvm::Function * func)215 MemCharSimdLowering::MemCharSimdLowering(llvm::Value *ch, llvm::Value *addr, llvm::IRBuilder<> *builder,
216 llvm::Function *func)
217 : ch_(ch), addr_(addr), builder_(builder), func_(func)
218 {
219 ASSERT(addr_ != nullptr);
220 ASSERT(builder_ != nullptr);
221 ASSERT(func_ != nullptr);
222 }
223
GenLoadAndFastCheck128(llvm::VectorType * vecTy)224 void MemCharSimdLowering::GenLoadAndFastCheck128(llvm::VectorType *vecTy)
225 {
226 auto *module = func_->getParent();
227 auto addpId = llvm::Intrinsic::AARCH64Intrinsics::aarch64_neon_addp;
228 auto addp = llvm::Intrinsic::getDeclaration(module, addpId, {GetU64X2Ty()});
229 // Read 16-byte chunk of memory
230 auto vld1 = builder_->CreateLoad(vecTy, addr_);
231 // Prepare the search pattern
232 auto insert = builder_->CreateInsertElement(vecTy, ch_, 0UL);
233 auto pattern = builder_->CreateShuffleVector(insert, ShuffleMask(vecTy->getElementType()));
234 // Compare
235 vcmpeq1_ = builder_->CreateSExt(builder_->CreateICmpEQ(vld1, pattern), vecTy);
236 // Do fast check and give up if char is not there
237 auto v64x2 = builder_->CreateBitCast(vcmpeq1_, GetU64X2Ty());
238 auto vaddp = builder_->CreateCall(addp, {v64x2, v64x2});
239 auto low64 = builder_->CreateBitCast(builder_->CreateExtractElement(vaddp, 0UL), builder_->getInt64Ty());
240 auto charIsNotThere = builder_->CreateICmpEQ(low64, llvm::Constant::getNullValue(low64->getType()));
241 builder_->CreateCondBr(charIsNotThere, lastBb_, inspectV1D0Bb_);
242 }
243
GenLoadAndFastCheck256(llvm::VectorType * vecTy)244 void MemCharSimdLowering::GenLoadAndFastCheck256(llvm::VectorType *vecTy)
245 {
246 auto *module = func_->getParent();
247 auto ld1Id = llvm::Intrinsic::AARCH64Intrinsics::aarch64_neon_ld1x2;
248 auto addpId = llvm::Intrinsic::AARCH64Intrinsics::aarch64_neon_addp;
249 auto ld1 = llvm::Intrinsic::getDeclaration(module, ld1Id, {vecTy, addr_->getType()});
250 auto addp1 = llvm::Intrinsic::getDeclaration(module, addpId, {vecTy});
251 auto addp2 = llvm::Intrinsic::getDeclaration(module, addpId, {GetU64X2Ty()});
252 // Read 32-byte chunk of memory
253 auto vld1 = builder_->CreateCall(ld1, {addr_});
254 auto v1 = builder_->CreateExtractValue(vld1, {0});
255 auto v2 = builder_->CreateExtractValue(vld1, {1});
256 // Prepare the search pattern
257 auto insert = builder_->CreateInsertElement(vecTy, ch_, 0UL);
258 auto pattern = builder_->CreateShuffleVector(insert, ShuffleMask(vecTy->getElementType()));
259 // Compare
260 vcmpeq1_ = builder_->CreateSExt(builder_->CreateICmpEQ(v1, pattern), vecTy);
261 vcmpeq2_ = builder_->CreateSExt(builder_->CreateICmpEQ(v2, pattern), vecTy);
262 // Do fast check and give up if char is not there
263 auto vaddp = builder_->CreateCall(addp1, {vcmpeq1_, vcmpeq2_});
264 auto v64x2 = builder_->CreateBitCast(vaddp, GetU64X2Ty());
265 vaddp = builder_->CreateCall(addp2, {v64x2, v64x2});
266 auto low64 = builder_->CreateBitCast(builder_->CreateExtractElement(vaddp, 0UL), builder_->getInt64Ty());
267 auto charIsNotThere = builder_->CreateICmpEQ(low64, llvm::Constant::getNullValue(low64->getType()));
268 builder_->CreateCondBr(charIsNotThere, lastBb_, inspectV1D0Bb_);
269 }
270
GenFindChar128(llvm::IntegerType * charTy)271 llvm::Value *MemCharSimdLowering::GenFindChar128(llvm::IntegerType *charTy)
272 {
273 ASSERT(vcmpeq1_ != nullptr && vcmpeq2_ == nullptr);
274 auto i64Ty = builder_->getInt64Ty();
275 constexpr uint32_t DWORD_SIZE = 64U;
276 // Inspect low 64-bit part of vcmpeq1
277 builder_->SetInsertPoint(inspectV1D0Bb_);
278 auto vcmpeq1 = builder_->CreateBitCast(vcmpeq1_, GetU64X2Ty());
279 auto v1d0 = builder_->CreateBitCast(builder_->CreateExtractElement(vcmpeq1, 0UL), i64Ty);
280 auto v1d0IsZero = builder_->CreateICmpEQ(v1d0, llvm::Constant::getNullValue(v1d0->getType()));
281 builder_->CreateCondBr(v1d0IsZero, inspectV1D1Bb_, clzV1D0Bb_);
282 builder_->SetInsertPoint(clzV1D0Bb_);
283 auto rev10 = builder_->CreateUnaryIntrinsic(llvm::Intrinsic::bswap, v1d0, nullptr);
284 auto pos10 = builder_->CreateBinaryIntrinsic(llvm::Intrinsic::ctlz, rev10, builder_->getFalse(), nullptr);
285 builder_->CreateBr(foundBb_);
286 // Inspect high 64-bit part of vcmpeq1
287 builder_->SetInsertPoint(inspectV1D1Bb_);
288 auto v1d1 = builder_->CreateBitCast(builder_->CreateExtractElement(vcmpeq1, 1UL), i64Ty);
289 auto rev11 = builder_->CreateUnaryIntrinsic(llvm::Intrinsic::bswap, v1d1, nullptr);
290 auto clz11 = builder_->CreateBinaryIntrinsic(llvm::Intrinsic::ctlz, rev11, builder_->getFalse(), nullptr);
291 auto pos11 = builder_->CreateAdd(clz11, llvm::Constant::getIntegerValue(i64Ty, llvm::APInt(DWORD_SIZE, UL64)));
292 builder_->CreateBr(foundBb_);
293 // Compute a pointer to the char
294 builder_->SetInsertPoint(foundBb_);
295 auto nbits = builder_->CreatePHI(i64Ty, 2U);
296 nbits->addIncoming(pos10, clzV1D0Bb_);
297 nbits->addIncoming(pos11, inspectV1D1Bb_);
298 auto nbytes = builder_->CreateLShr(nbits, charTy->isIntegerTy(8U) ? 3UL : 4UL);
299 auto foundCharPtr = builder_->CreateInBoundsGEP(charTy, addr_, nbytes);
300 builder_->CreateBr(lastBb_);
301 return foundCharPtr;
302 }
303
GenFindChar256(llvm::IntegerType * charTy)304 llvm::Value *MemCharSimdLowering::GenFindChar256(llvm::IntegerType *charTy)
305 {
306 ASSERT(vcmpeq1_ != nullptr);
307 ASSERT(vcmpeq2_ != nullptr);
308 auto i64Ty = builder_->getInt64Ty();
309 constexpr uint32_t DWORD_SIZE = 64U;
310 // Inspect low 64-bit part of vcmpeq1
311 builder_->SetInsertPoint(inspectV1D0Bb_);
312 auto vcmpeq1 = builder_->CreateBitCast(vcmpeq1_, GetU64X2Ty());
313 auto v1d0 = builder_->CreateBitCast(builder_->CreateExtractElement(vcmpeq1, 0UL), i64Ty);
314 auto v1d0IsZero = builder_->CreateICmpEQ(v1d0, llvm::Constant::getNullValue(v1d0->getType()));
315 builder_->CreateCondBr(v1d0IsZero, inspectV1D1Bb_, clzV1D0Bb_);
316 builder_->SetInsertPoint(clzV1D0Bb_);
317 auto rev10 = builder_->CreateUnaryIntrinsic(llvm::Intrinsic::bswap, v1d0, nullptr);
318 auto pos10 = builder_->CreateBinaryIntrinsic(llvm::Intrinsic::ctlz, rev10, builder_->getFalse(), nullptr);
319 builder_->CreateBr(foundBb_);
320 // Inspect high 64-bit part of vcmpeq1
321 builder_->SetInsertPoint(inspectV1D1Bb_);
322 auto v1d1 = builder_->CreateBitCast(builder_->CreateExtractElement(vcmpeq1, 1UL), i64Ty);
323 auto v1d1IsZero = builder_->CreateICmpEQ(v1d1, llvm::Constant::getNullValue(v1d1->getType()));
324 builder_->CreateCondBr(v1d1IsZero, inspectV2D0Bb_, clzV1D1Bb_);
325 builder_->SetInsertPoint(clzV1D1Bb_);
326 auto rev11 = builder_->CreateUnaryIntrinsic(llvm::Intrinsic::bswap, v1d1, nullptr);
327 auto clz11 = builder_->CreateBinaryIntrinsic(llvm::Intrinsic::ctlz, rev11, builder_->getFalse(), nullptr);
328 auto pos11 = builder_->CreateAdd(clz11, llvm::Constant::getIntegerValue(i64Ty, llvm::APInt(DWORD_SIZE, UL64)));
329 builder_->CreateBr(foundBb_);
330 // Inspect low 64-bit part of vcmpeq2
331 builder_->SetInsertPoint(inspectV2D0Bb_);
332 auto vcmpeq2 = builder_->CreateBitCast(vcmpeq2_, GetU64X2Ty());
333 auto v2d0 = builder_->CreateBitCast(builder_->CreateExtractElement(vcmpeq2, 0UL), i64Ty);
334 auto v2d0IsZero = builder_->CreateICmpEQ(v2d0, llvm::Constant::getNullValue(v2d0->getType()));
335 builder_->CreateCondBr(v2d0IsZero, inspectV2D1Bb_, clzV2D0Bb_);
336 builder_->SetInsertPoint(clzV2D0Bb_);
337 auto rev20 = builder_->CreateUnaryIntrinsic(llvm::Intrinsic::bswap, v2d0, nullptr);
338 auto clz20 = builder_->CreateBinaryIntrinsic(llvm::Intrinsic::ctlz, rev20, builder_->getFalse(), nullptr);
339 auto pos20 = builder_->CreateAdd(clz20, llvm::Constant::getIntegerValue(i64Ty, llvm::APInt(DWORD_SIZE, UL128)));
340 builder_->CreateBr(foundBb_);
341 // Inspect high 64-bit part of vcmpeq2
342 builder_->SetInsertPoint(inspectV2D1Bb_);
343 auto v2d1 = builder_->CreateBitCast(builder_->CreateExtractElement(vcmpeq2, 1UL), i64Ty);
344 auto rev21 = builder_->CreateUnaryIntrinsic(llvm::Intrinsic::bswap, v2d1, nullptr);
345 auto clz21 = builder_->CreateBinaryIntrinsic(llvm::Intrinsic::ctlz, rev21, builder_->getFalse(), nullptr);
346 auto pos21 = builder_->CreateAdd(clz21, llvm::Constant::getIntegerValue(i64Ty, llvm::APInt(DWORD_SIZE, UL192)));
347 builder_->CreateBr(foundBb_);
348 // Compute a pointer to the char
349 builder_->SetInsertPoint(foundBb_);
350 auto nbits = builder_->CreatePHI(i64Ty, 4U);
351 nbits->addIncoming(pos10, clzV1D0Bb_);
352 nbits->addIncoming(pos11, clzV1D1Bb_);
353 nbits->addIncoming(pos20, clzV2D0Bb_);
354 nbits->addIncoming(pos21, inspectV2D1Bb_);
355 auto nbytes = builder_->CreateLShr(nbits, charTy->isIntegerTy(8U) ? 3UL : 4UL);
356 auto foundCharPtr = builder_->CreateInBoundsGEP(charTy, addr_, nbytes);
357 builder_->CreateBr(lastBb_);
358 return foundCharPtr;
359 }
360
361 template <bool MEM_BLOCK_SIZE_256_BITS>
Generate(llvm::VectorType * vecTy)362 llvm::Value *MemCharSimdLowering::Generate(llvm::VectorType *vecTy)
363 {
364 auto *charTy = llvm::cast<llvm::IntegerType>(vecTy->getElementType());
365 ASSERT(vecTy == GetU8X16Ty() || vecTy == GetU16X8Ty());
366 auto &context = func_->getContext();
367 auto firstBb = builder_->GetInsertBlock();
368 lastBb_ = llvm::BasicBlock::Create(context, "mem_char_using_simd_last", func_);
369 foundBb_ = llvm::BasicBlock::Create(context, "mem_char_using_simd_found", func_);
370 inspectV1D0Bb_ = llvm::BasicBlock::Create(context, "mem_char_using_simd_inspect_v1d0", func_);
371 inspectV1D1Bb_ = llvm::BasicBlock::Create(context, "mem_char_using_simd_inspect_v1d1", func_);
372 clzV1D0Bb_ = llvm::BasicBlock::Create(context, "mem_char_using_simd_inspect_v1d1", func_);
373 llvm::Value *foundCharPtr = nullptr;
374 if constexpr (MEM_BLOCK_SIZE_256_BITS) {
375 inspectV2D0Bb_ = llvm::BasicBlock::Create(context, "mem_char_using_simd_inspect_v2d0", func_);
376 inspectV2D1Bb_ = llvm::BasicBlock::Create(context, "mem_char_using_simd_inspect_v2d1", func_);
377 clzV1D1Bb_ = llvm::BasicBlock::Create(context, "mem_char_using_simd_inspect_v1d1", func_);
378 clzV2D0Bb_ = llvm::BasicBlock::Create(context, "mem_char_using_simd_inspect_v1d1", func_);
379 GenLoadAndFastCheck256(vecTy);
380 foundCharPtr = GenFindChar256(charTy);
381 } else {
382 GenLoadAndFastCheck128(vecTy);
383 foundCharPtr = GenFindChar128(charTy);
384 }
385 ASSERT(foundCharPtr != nullptr);
386 // The result is either a pointer to the char or null
387 builder_->SetInsertPoint(lastBb_);
388 auto result = builder_->CreatePHI(builder_->getPtrTy(), 2U);
389 result->addIncoming(llvm::Constant::getNullValue(builder_->getPtrTy()), firstBb);
390 result->addIncoming(foundCharPtr, foundBb_);
391 // Cleanup and return
392 lastBb_ = nullptr;
393 foundBb_ = nullptr;
394 inspectV1D0Bb_ = nullptr;
395 inspectV1D1Bb_ = nullptr;
396 inspectV2D0Bb_ = nullptr;
397 inspectV2D1Bb_ = nullptr;
398 clzV1D0Bb_ = nullptr;
399 clzV1D1Bb_ = nullptr;
400 clzV2D0Bb_ = nullptr;
401 vcmpeq1_ = nullptr;
402 vcmpeq2_ = nullptr;
403 return result;
404 }
405
MarkNormalBlocksRecursive(BasicBlock * block,Marker normal)406 static void MarkNormalBlocksRecursive(BasicBlock *block, Marker normal)
407 {
408 [[maybe_unused]] size_t expected = 0;
409 bool processSucc = true;
410 auto last = block->GetLastInst();
411 if (last != nullptr) {
412 // Any successors of blocks with terminators are either TryEnd or Catch blocks
413 if (last->GetFlag(inst_flags::TERMINATOR)) {
414 processSucc = false;
415 }
416 if (last->GetOpcode() == Opcode::IfImm || last->GetOpcode() == Opcode::If) {
417 expected = 1;
418 }
419 }
420 for (size_t i = 0; i < block->GetSuccsBlocks().size(); i++) {
421 auto succ = block->GetSuccessor(i);
422 if (succ->IsCatch()) {
423 ASSERT_DO(i > expected,
424 (std::cerr << "Catch block found too early in successors: at index " << i << std::endl));
425 continue;
426 }
427 ASSERT_DO(i <= expected, (std::cerr << "Unexpected non-catch successor block at index " << i << std::endl));
428 if (processSucc && !succ->SetMarker(normal)) {
429 MarkNormalBlocksRecursive(succ, normal);
430 }
431 }
432 }
433
434 // Use that only to pass it into method like rvalue
CreateBasicBlockName(Inst * inst,const std::string & bbName)435 static inline std::string CreateBasicBlockName(Inst *inst, const std::string &bbName)
436 {
437 std::stringstream name;
438 name << "bb" << std::to_string(inst->GetBasicBlock()->GetId()) << "_i" << std::to_string(inst->GetId()) << ".."
439 << bbName << "..";
440 return name.str();
441 }
442
CreateNameForInst(Inst * inst)443 static inline std::string CreateNameForInst(Inst *inst)
444 {
445 return std::string("v") + std::to_string(inst->GetId());
446 }
447
IsInteger(DataType::Type type)448 static inline bool IsInteger(DataType::Type type)
449 {
450 return DataType::IsTypeNumeric(type) && !DataType::IsFloatType(type) && type != DataType::POINTER;
451 }
452
IsSignedInteger(const DataType::Type & type)453 static inline bool IsSignedInteger(const DataType::Type &type)
454 {
455 return IsInteger(type) && DataType::IsTypeSigned(type);
456 }
457
IsUnsignedInteger(DataType::Type type)458 static inline bool IsUnsignedInteger(DataType::Type type)
459 {
460 return IsInteger(type) && !DataType::IsTypeSigned(type);
461 }
462
IsAlwaysThrowBasicBlock(Inst * inst)463 static inline bool IsAlwaysThrowBasicBlock(Inst *inst)
464 {
465 if (!g_options.IsCompilerInliningSkipThrowBlocks()) {
466 return false;
467 }
468
469 auto bbLastInst = inst->GetBasicBlock()->GetLastInst();
470 return bbLastInst->GetOpcode() == Opcode::Throw || bbLastInst->GetOpcode() == Opcode::Deoptimize;
471 }
472
ICmpCodeConvert(ConditionCode cc)473 static llvm::ICmpInst::Predicate ICmpCodeConvert(ConditionCode cc)
474 {
475 switch (cc) {
476 case ConditionCode::CC_EQ:
477 return llvm::CmpInst::Predicate::ICMP_EQ;
478 case ConditionCode::CC_NE:
479 return llvm::CmpInst::Predicate::ICMP_NE;
480 case ConditionCode::CC_LT:
481 return llvm::CmpInst::Predicate::ICMP_SLT;
482 case ConditionCode::CC_GT:
483 return llvm::CmpInst::Predicate::ICMP_SGT;
484 case ConditionCode::CC_LE:
485 return llvm::CmpInst::Predicate::ICMP_SLE;
486 case ConditionCode::CC_GE:
487 return llvm::CmpInst::Predicate::ICMP_SGE;
488 case ConditionCode::CC_B:
489 return llvm::CmpInst::Predicate::ICMP_ULT;
490 case ConditionCode::CC_A:
491 return llvm::CmpInst::Predicate::ICMP_UGT;
492 case ConditionCode::CC_BE:
493 return llvm::CmpInst::Predicate::ICMP_ULE;
494 case ConditionCode::CC_AE:
495 return llvm::CmpInst::Predicate::ICMP_UGE;
496 default:
497 UNREACHABLE();
498 return llvm::CmpInst::Predicate::ICMP_NE;
499 }
500 }
501
FCmpCodeConvert(ConditionCode conditionCode)502 static llvm::FCmpInst::Predicate FCmpCodeConvert(ConditionCode conditionCode)
503 {
504 switch (conditionCode) {
505 case ConditionCode::CC_EQ:
506 return llvm::FCmpInst::Predicate::FCMP_UEQ;
507 case ConditionCode::CC_NE:
508 return llvm::FCmpInst::Predicate::FCMP_UNE;
509 case ConditionCode::CC_LT:
510 return llvm::FCmpInst::Predicate::FCMP_ULT;
511 case ConditionCode::CC_GT:
512 return llvm::FCmpInst::Predicate::FCMP_UGT;
513 case ConditionCode::CC_LE:
514 return llvm::FCmpInst::Predicate::FCMP_ULE;
515 case ConditionCode::CC_GE:
516 return llvm::FCmpInst::Predicate::FCMP_UGE;
517 case ConditionCode::CC_B:
518 return llvm::FCmpInst::Predicate::FCMP_ULT;
519 case ConditionCode::CC_A:
520 return llvm::FCmpInst::Predicate::FCMP_UGT;
521 case ConditionCode::CC_BE:
522 return llvm::FCmpInst::Predicate::FCMP_ULE;
523 case ConditionCode::CC_AE:
524 return llvm::FCmpInst::Predicate::FCMP_UGE;
525 default:
526 ASSERT_DO(false, (std::cerr << "Unexpected condition_code = " << conditionCode << std::endl));
527 UNREACHABLE();
528 }
529 }
530
GetDeoptimizationType(Inst * inst)531 static DeoptimizeType GetDeoptimizationType(Inst *inst)
532 {
533 switch (inst->GetOpcode()) {
534 case Opcode::NullCheck:
535 return DeoptimizeType::NULL_CHECK;
536 case Opcode::DeoptimizeIf:
537 return inst->CastToDeoptimizeIf()->GetDeoptimizeType();
538 case Opcode::BoundsCheck:
539 return DeoptimizeType::BOUNDS_CHECK_WITH_DEOPT;
540 case Opcode::NegativeCheck:
541 return DeoptimizeType::NEGATIVE_CHECK;
542 case Opcode::ZeroCheck:
543 return DeoptimizeType::ZERO_CHECK;
544 case Opcode::SubOverflowCheck:
545 return DeoptimizeType::OVERFLOW;
546 case Opcode::CheckCast:
547 return DeoptimizeType::CHECK_CAST;
548 case Opcode::RefTypeCheck:
549 default:
550 ASSERT_DO(false, (std::cerr << "Unexpected inst to GetDeoptimizationType, inst:" << std::endl,
551 inst->Dump(&std::cerr, true)));
552 UNREACHABLE();
553 }
554 }
555
GetFastPathCallingConv(uint32_t numArgs)556 static llvm::CallingConv::ID GetFastPathCallingConv(uint32_t numArgs)
557 {
558 switch (numArgs) {
559 case 0U:
560 return llvm::CallingConv::ArkFast0;
561 case 1U:
562 return llvm::CallingConv::ArkFast1;
563 case 2U:
564 return llvm::CallingConv::ArkFast2;
565 case 3U:
566 return llvm::CallingConv::ArkFast3;
567 case 4U:
568 return llvm::CallingConv::ArkFast4;
569 case 5U:
570 return llvm::CallingConv::ArkFast5;
571 case 6U:
572 return llvm::CallingConv::ArkFast6;
573 default:
574 UNREACHABLE();
575 }
576 }
577
GetAllocateArrayTlabEntrypoint(size_t elementSize)578 static RuntimeInterface::EntrypointId GetAllocateArrayTlabEntrypoint(size_t elementSize)
579 {
580 switch (elementSize) {
581 case sizeof(uint8_t):
582 return RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB8;
583 case sizeof(uint16_t):
584 return RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB16;
585 case sizeof(uint32_t):
586 return RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB32;
587 case sizeof(uint64_t):
588 return RuntimeInterface::EntrypointId::ALLOCATE_ARRAY_TLAB64;
589 default:
590 UNREACHABLE();
591 }
592 }
593
GetRealFrameReg(Arch arch)594 static size_t GetRealFrameReg(Arch arch)
595 {
596 switch (arch) {
597 case Arch::AARCH64:
598 return AARCH64_REAL_FP;
599 case Arch::X86_64:
600 return X86_64_REAL_FP;
601 default:
602 UNREACHABLE();
603 }
604 }
605
606 /**
607 * Call when we are sure that instruction shouldn't appear for translating but
608 * eventually we've tried to translate it.
609 */
UnexpectedLowering(Inst * inst)610 static void UnexpectedLowering([[maybe_unused]] Inst *inst)
611 {
612 ASSERT_DO(false, (std::cerr << "Unexpected attempt to lower: ", inst->Dump(&std::cerr, true)));
613 UNREACHABLE();
614 }
615
IsSafeCast(Inst * inst,unsigned int index)616 bool LLVMIrConstructor::IsSafeCast(Inst *inst, unsigned int index)
617 {
618 auto trueType = inst->GetInput(index).GetInst()->GetType();
619 auto instType = inst->GetInputType(index);
620 bool signTheSame = IsSignedInteger(trueType) == IsSignedInteger(instType);
621 bool extending = DataType::GetTypeSize(trueType, GetGraph()->GetArch()) <=
622 DataType::GetTypeSize(instType, GetGraph()->GetArch());
623 return signTheSame || extending;
624 }
625
TryEmitIntrinsic(Inst * inst,RuntimeInterface::IntrinsicId arkId)626 bool LLVMIrConstructor::TryEmitIntrinsic(Inst *inst, RuntimeInterface::IntrinsicId arkId)
627 {
628 auto module = func_->getParent();
629 auto f32Ty = builder_.getFloatTy();
630 auto f64Ty = builder_.getDoubleTy();
631 llvm::Function *llvmId = nullptr;
632
633 switch (arkId) {
634 #include "intrinsics_llvm_codegen.inl"
635 #ifndef NDEBUG
636 // Must be lowered earlier in IrBuilder, impossible to meet
637 case RuntimeInterface::IntrinsicId::INTRINSIC_OBJECT_MONITOR_ENTER:
638 case RuntimeInterface::IntrinsicId::INTRINSIC_OBJECT_MONITOR_EXIT:
639 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_ABS_I32:
640 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_ABS_I64:
641 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_ABS_F32:
642 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_ABS_F64:
643 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_SQRT_F32:
644 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_SQRT_F64:
645 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_MIN_I32:
646 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_MIN_I64:
647 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_MIN_F32:
648 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_MIN_F64:
649 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_MAX_I32:
650 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_MAX_I64:
651 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_MAX_F32:
652 case RuntimeInterface::IntrinsicId::INTRINSIC_MATH_MAX_F64:
653 UNREACHABLE();
654 // Can appear only after LLVM optimizations
655 case RuntimeInterface::IntrinsicId::LIB_CALL_MEM_COPY:
656 case RuntimeInterface::IntrinsicId::LIB_CALL_MEM_SET:
657 case RuntimeInterface::IntrinsicId::LIB_CALL_MEM_MOVE:
658 UNREACHABLE();
659 #include "emit_intrinsic_llvm_ir_constructor_gen.inl"
660 #endif
661 default:
662 return false;
663 }
664
665 ASSERT(llvmId != nullptr);
666 ASSERT(!inst->CanThrow());
667
668 arkInterface_->GetOrCreateRuntimeFunctionType(func_->getContext(), func_->getParent(),
669 LLVMArkInterface::RuntimeCallType::INTRINSIC,
670 static_cast<LLVMArkInterface::EntrypointId>(arkId));
671
672 auto arguments = GetIntrinsicArguments(llvmId->getFunctionType(), inst->CastToIntrinsic());
673 auto result = llvm::CallInst::Create(llvmId, arguments, "", GetCurrentBasicBlock());
674 SetIntrinsicParamAttrs(result, inst->CastToIntrinsic(), arguments);
675 ValueMapAdd(inst, result);
676 return true;
677 }
678
679 // Specific intrinsic Emitters
680
EmitFastPath(Inst * inst,RuntimeInterface::EntrypointId eid,uint32_t numArgs)681 bool LLVMIrConstructor::EmitFastPath(Inst *inst, RuntimeInterface::EntrypointId eid, uint32_t numArgs)
682 {
683 ArenaVector<llvm::Value *> args(GetGraph()->GetLocalAllocator()->Adapter());
684 for (uint32_t i = 0; i < numArgs; i++) {
685 args.push_back(GetInputValue(inst, i));
686 }
687 auto call = CreateFastPathCall(inst, eid, args);
688
689 auto retType = GetType(inst->GetType());
690 if (!retType->isVoidTy()) {
691 ValueMapAdd(inst, call);
692 }
693 return true;
694 }
695
EmitStringEquals(Inst * inst)696 bool LLVMIrConstructor::EmitStringEquals(Inst *inst)
697 {
698 return EmitFastPath(inst, RuntimeInterface::EntrypointId::STRING_EQUALS_COMPRESSED, 2U);
699 }
700
EmitStringBuilderBool(Inst * inst)701 bool LLVMIrConstructor::EmitStringBuilderBool(Inst *inst)
702 {
703 return EmitFastPath(inst, RuntimeInterface::EntrypointId::STRING_BUILDER_BOOL, 2U);
704 }
705
EmitStringBuilderChar(Inst * inst)706 bool LLVMIrConstructor::EmitStringBuilderChar(Inst *inst)
707 {
708 return EmitFastPath(inst, RuntimeInterface::EntrypointId::STRING_BUILDER_CHAR, 2U);
709 }
710
EmitStringBuilderString(Inst * inst)711 bool LLVMIrConstructor::EmitStringBuilderString(Inst *inst)
712 {
713 return EmitFastPath(inst, RuntimeInterface::EntrypointId::STRING_BUILDER_STRING_COMPRESSED, 2U);
714 }
715
EmitStringConcat2(Inst * inst)716 bool LLVMIrConstructor::EmitStringConcat2(Inst *inst)
717 {
718 return EmitFastPath(inst, RuntimeInterface::EntrypointId::STRING_CONCAT2_TLAB, 2U);
719 }
720
EmitStringConcat3(Inst * inst)721 bool LLVMIrConstructor::EmitStringConcat3(Inst *inst)
722 {
723 return EmitFastPath(inst, RuntimeInterface::EntrypointId::STRING_CONCAT3_TLAB, 3U);
724 }
725
EmitStringConcat4(Inst * inst)726 bool LLVMIrConstructor::EmitStringConcat4(Inst *inst)
727 {
728 return EmitFastPath(inst, RuntimeInterface::EntrypointId::STRING_CONCAT4_TLAB, 4U);
729 }
730
EmitStringCompareTo(Inst * inst)731 bool LLVMIrConstructor::EmitStringCompareTo(Inst *inst)
732 {
733 return EmitFastPath(inst, RuntimeInterface::EntrypointId::STRING_COMPARE_TO, 2U);
734 }
735
EmitIsInf(Inst * inst)736 bool LLVMIrConstructor::EmitIsInf(Inst *inst)
737 {
738 auto result = CreateIsInf(GetInputValue(inst, 0));
739 ValueMapAdd(inst, result);
740 return true;
741 }
742
EmitMemmoveUnchecked(Inst * inst)743 bool LLVMIrConstructor::EmitMemmoveUnchecked(Inst *inst)
744 {
745 switch (inst->CastToIntrinsic()->GetIntrinsicId()) {
746 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPILER_MEMMOVE_UNCHECKED_1_BYTE:
747 return EmitFastPath(inst, RuntimeInterface::EntrypointId::ARRAY_COPY_TO_UNCHECKED_1_BYTE, 5U);
748 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPILER_MEMMOVE_UNCHECKED_2_BYTE:
749 return EmitFastPath(inst, RuntimeInterface::EntrypointId::ARRAY_COPY_TO_UNCHECKED_2_BYTE, 5U);
750 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPILER_MEMMOVE_UNCHECKED_4_BYTE:
751 return EmitFastPath(inst, RuntimeInterface::EntrypointId::ARRAY_COPY_TO_UNCHECKED_4_BYTE, 5U);
752 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPILER_MEMMOVE_UNCHECKED_8_BYTE:
753 return EmitFastPath(inst, RuntimeInterface::EntrypointId::ARRAY_COPY_TO_UNCHECKED_8_BYTE, 5U);
754 default:
755 UNREACHABLE();
756 }
757 }
758
EmitUnreachable(Inst * inst)759 bool LLVMIrConstructor::EmitUnreachable([[maybe_unused]] Inst *inst)
760 {
761 auto bb = GetCurrentBasicBlock();
762 if (bb->empty() || !llvm::isa<llvm::ReturnInst>(*(bb->rbegin()))) {
763 auto trap = llvm::Intrinsic::getDeclaration(func_->getParent(), llvm::Intrinsic::trap, {});
764 builder_.CreateCall(trap, {});
765 builder_.CreateUnreachable();
766 }
767 return true;
768 }
769
EmitNothing(Inst * inst)770 bool LLVMIrConstructor::EmitNothing([[maybe_unused]] Inst *inst)
771 {
772 return true;
773 }
774
775 #ifndef NDEBUG
CheckSlowPathName(const std::string & name,size_t funcArgsNum,size_t callArgsNum)776 static void CheckSlowPathName(const std::string &name, size_t funcArgsNum, size_t callArgsNum)
777 {
778 ASSERT_DO(std::string_view {name}.find("SlowPath") == std::string_view::npos,
779 std::cerr << "Bad bridge: SlowPath bridge not allowed in LLVM FastPath: " << name << std::endl);
780 ASSERT(callArgsNum <= funcArgsNum);
781 if (callArgsNum < funcArgsNum) {
782 funcArgsNum -= 2U; // exclude fake arguments for these asserts
783 ASSERT(funcArgsNum <= 4U);
784 ASSERT_DO((std::string_view {name}.find("1ArgBridge") != std::string_view::npos) == (funcArgsNum == 1U),
785 std::cerr << "Bad bridge: OddSaved1 for FastPath with 1 arguments "
786 << "and SlowPath with zero arguments: " << name << std::endl);
787 ASSERT_DO((std::string_view {name}.find("2ArgBridge") != std::string_view::npos) == (funcArgsNum == 2U),
788 std::cerr << "Bad bridge: OddSaved2 for FastPath with 2 arguments "
789 << "and SlowPath with 0-1 arguments: " << name << std::endl);
790 ASSERT_DO((std::string_view {name}.find("3ArgBridge") != std::string_view::npos) == (funcArgsNum == 3U),
791 std::cerr << "Bad bridge: OddSaved3 for FastPath with 3 arguments "
792 << "and SlowPath with 0-2 arguments: " << name << std::endl);
793 ASSERT_DO((std::string_view {name}.find("4ArgBridge") != std::string_view::npos) == (funcArgsNum == 4U),
794 std::cerr << "Bad bridge: OddSaved4 for FastPath with 4 arguments "
795 << "and SlowPath with 0-3 arguments: " << name << std::endl);
796 } else { // callArgsNum == funcArgsNum
797 ASSERT_DO((std::string_view {name}.find("OddSaved") != std::string_view::npos) == (funcArgsNum % 2U == 1U),
798 std::cerr << "Bad bridge: OddSaved <=> amount of arguments is odd: " << name << std::endl);
799 }
800 }
801 #endif
802
EmitSlowPathEntry(Inst * inst)803 bool LLVMIrConstructor::EmitSlowPathEntry(Inst *inst)
804 {
805 ASSERT(GetGraph()->GetMode().IsFastPath());
806 ASSERT(func_->getCallingConv() == llvm::CallingConv::ArkFast0 ||
807 func_->getCallingConv() == llvm::CallingConv::ArkFast1 ||
808 func_->getCallingConv() == llvm::CallingConv::ArkFast2 ||
809 func_->getCallingConv() == llvm::CallingConv::ArkFast3 ||
810 func_->getCallingConv() == llvm::CallingConv::ArkFast4 ||
811 func_->getCallingConv() == llvm::CallingConv::ArkFast5 ||
812 func_->getCallingConv() == llvm::CallingConv::ArkFast6);
813
814 // Arguments
815 ArenaVector<llvm::Value *> args(GetGraph()->GetLocalAllocator()->Adapter());
816 for (size_t i = 0; i < inst->GetInputs().Size(); i++) {
817 args.push_back(GetInputValue(inst, i));
818 }
819 auto threadRegPtr = builder_.CreateIntToPtr(GetThreadRegValue(), builder_.getPtrTy());
820 auto frameRegPtr = builder_.CreateIntToPtr(GetRealFrameRegValue(), builder_.getPtrTy());
821 args.push_back(threadRegPtr);
822 args.push_back(frameRegPtr);
823
824 ASSERT(inst->CastToIntrinsic()->HasImms() && inst->CastToIntrinsic()->GetImms().size() == 2U);
825 uint32_t externalId = inst->CastToIntrinsic()->GetImms()[1];
826 auto externalName = GetGraph()->GetRuntime()->GetExternalMethodName(GetGraph()->GetMethod(), externalId);
827 #ifndef NDEBUG
828 CheckSlowPathName(externalName, func_->arg_size(), args.size());
829 #endif
830 auto callee = func_->getParent()->getFunction(externalName);
831 if (callee == nullptr) {
832 ArenaVector<llvm::Type *> argTypes(GetGraph()->GetLocalAllocator()->Adapter());
833 for (const auto &input : inst->GetInputs()) {
834 argTypes.push_back(GetExactType(input.GetInst()->GetType()));
835 }
836 argTypes.push_back(builder_.getPtrTy());
837 argTypes.push_back(builder_.getPtrTy());
838 auto ftype = llvm::FunctionType::get(GetType(inst->GetType()), argTypes, false);
839 callee = llvm::Function::Create(ftype, llvm::Function::ExternalLinkage, externalName, func_->getParent());
840 callee->setCallingConv(GetFastPathCallingConv(inst->GetInputs().Size()));
841 }
842
843 auto call = builder_.CreateCall(callee->getFunctionType(), callee, args);
844 call->setCallingConv(callee->getCallingConv());
845 call->setTailCallKind(llvm::CallInst::TailCallKind::TCK_Tail);
846 call->addFnAttr(llvm::Attribute::get(call->getContext(), "ark-tail-call"));
847 CreateReturn(call);
848 return true;
849 }
850
EmitExclusiveLoadWithAcquire(Inst * inst)851 bool LLVMIrConstructor::EmitExclusiveLoadWithAcquire(Inst *inst)
852 {
853 ASSERT(GetGraph()->GetArch() == Arch::AARCH64);
854 ASSERT(inst->GetInputType(0) == DataType::POINTER);
855 auto &ctx = func_->getContext();
856 auto addr = GetInputValue(inst, 0);
857 auto dstType = GetExactType(inst->GetType());
858 auto intrinsicId = llvm::Intrinsic::AARCH64Intrinsics::aarch64_ldaxr;
859 auto load = builder_.CreateUnaryIntrinsic(intrinsicId, addr);
860 load->addParamAttr(0, llvm::Attribute::get(ctx, llvm::Attribute::ElementType, dstType));
861 ValueMapAdd(inst, load);
862 return true;
863 }
864
EmitExclusiveStoreWithRelease(Inst * inst)865 bool LLVMIrConstructor::EmitExclusiveStoreWithRelease(Inst *inst)
866 {
867 ASSERT(GetGraph()->GetArch() == Arch::AARCH64);
868 ASSERT(inst->GetInputType(0) == DataType::POINTER);
869 auto &ctx = func_->getContext();
870 auto addr = GetInputValue(inst, 0);
871 auto value = GetInputValue(inst, 1);
872 auto type = value->getType();
873 auto intrinsicId = llvm::Intrinsic::AARCH64Intrinsics::aarch64_stlxr;
874 auto stlxr = llvm::Intrinsic::getDeclaration(func_->getParent(), intrinsicId, builder_.getPtrTy());
875 value = builder_.CreateZExtOrBitCast(value, stlxr->getFunctionType()->getParamType(0));
876 auto store = builder_.CreateCall(stlxr, {value, addr});
877 store->addParamAttr(1, llvm::Attribute::get(ctx, llvm::Attribute::ElementType, type));
878 ValueMapAdd(inst, store);
879 return true;
880 }
881
EmitInterpreterReturn(Inst * inst)882 bool LLVMIrConstructor::EmitInterpreterReturn([[maybe_unused]] Inst *inst)
883 {
884 // We only support it for Irtoc interpreters on AArch64
885 ASSERT(GetGraph()->GetMode().IsInterpreter());
886
887 // This constant is hardcoded in codegen_interpreter.h and in interpreter.irt
888 constexpr size_t SPILL_SLOTS = 32;
889 CFrameLayout fl(GetGraph()->GetArch(), SPILL_SLOTS);
890 constexpr bool SAVE_UNUSED_CALLEE_REGS = true;
891
892 // Restore callee-registers
893 auto calleeRegsMask = GetCalleeRegsMask(GetGraph()->GetArch(), false, SAVE_UNUSED_CALLEE_REGS);
894 auto calleeVregsMask = GetCalleeRegsMask(GetGraph()->GetArch(), true, SAVE_UNUSED_CALLEE_REGS);
895 if (GetGraph()->GetArch() == Arch::AARCH64) {
896 constexpr bool SAVE_FRAME_AND_LINK_REGS = true;
897
898 size_t slotSize = fl.GetSlotSize();
899 size_t dslotSize = slotSize * 2U;
900
901 auto lastCalleeReg = fl.GetRegsSlotsCount() - calleeRegsMask.Count();
902 auto lastCalleeVreg = fl.GetRegsSlotsCount() - fl.GetCalleeRegistersCount(false) - calleeVregsMask.Count();
903 CreateInterpreterReturnRestoreRegs(calleeRegsMask, lastCalleeReg, false);
904 CreateInterpreterReturnRestoreRegs(calleeVregsMask, lastCalleeVreg, true);
905
906 // Adjust SP
907 auto spToFrameTopSlots = fl.GetRegsSlotsCount() + CFrameRegs::Start() - CFrameReturnAddr::Start();
908 if (SAVE_FRAME_AND_LINK_REGS) {
909 spToFrameTopSlots -= CFrameLayout::GetFpLrSlotsCount();
910 }
911
912 CreateInt32ImmAsm(&builder_,
913 std::string("add sp, sp, $0").append(LLVMArkInterface::PATCH_STACK_ADJUSTMENT_COMMENT),
914 spToFrameTopSlots * slotSize);
915 CreateInt32ImmAsm(&builder_, "ldp x29, x30, [sp], $0", dslotSize);
916 CreateBlackBoxAsm(&builder_, "ret");
917 } else {
918 // Currently there is no vector regs usage at x86_64 handlers
919 ASSERT(calleeVregsMask.count() == 0);
920 auto regShift = DOUBLE_WORD_SIZE_BYTES *
921 (fl.GetSpillsCount() + fl.GetCallerRegistersCount(false) + fl.GetCallerRegistersCount(true));
922 auto fpShift = DOUBLE_WORD_SIZE_BYTES * (2 + CFrameSlots::Start() - CFrameData::Start());
923
924 std::string iasmStr =
925 std::string("leaq ${0:c}(%rsp), %rsp").append(LLVMArkInterface::PATCH_STACK_ADJUSTMENT_COMMENT);
926 CreateInt32ImmAsm(&builder_, iasmStr, regShift);
927 Target target {GetGraph()->GetArch()};
928 while (calleeRegsMask.count() > 0) {
929 auto reg = calleeRegsMask.GetMinRegister();
930 calleeRegsMask ^= 1U << reg;
931 iasmStr = "pop %" + target.GetRegName(reg, false);
932 CreateBlackBoxAsm(&builder_, iasmStr);
933 }
934 iasmStr = "leaq " + std::to_string(fpShift) + "(%rsp), %rsp";
935 CreateBlackBoxAsm(&builder_, iasmStr);
936 CreateBlackBoxAsm(&builder_, "pop %rbp");
937 CreateBlackBoxAsm(&builder_, "retq");
938 }
939 builder_.CreateUnreachable();
940
941 return true;
942 }
943
EmitTailCall(Inst * inst)944 bool LLVMIrConstructor::EmitTailCall(Inst *inst)
945 {
946 ASSERT(func_->getCallingConv() == llvm::CallingConv::ArkFast0 ||
947 func_->getCallingConv() == llvm::CallingConv::ArkFast1 ||
948 func_->getCallingConv() == llvm::CallingConv::ArkFast2 ||
949 func_->getCallingConv() == llvm::CallingConv::ArkFast3 ||
950 func_->getCallingConv() == llvm::CallingConv::ArkFast4 ||
951 func_->getCallingConv() == llvm::CallingConv::ArkFast5 ||
952 func_->getCallingConv() == llvm::CallingConv::ArkFast6 ||
953 func_->getCallingConv() == llvm::CallingConv::ArkInt);
954 llvm::CallInst *call;
955
956 if (GetGraph()->GetMode().IsFastPath()) {
957 call = CreateTailCallFastPath(inst);
958 } else if (GetGraph()->GetMode().IsInterpreter()) {
959 call = CreateTailCallInterpreter(inst);
960 } else {
961 UNREACHABLE();
962 }
963 call->setTailCallKind(llvm::CallInst::TailCallKind::TCK_Tail);
964 call->addFnAttr(llvm::Attribute::get(call->getContext(), "ark-tail-call"));
965 CreateReturn(call);
966 std::fill(ccValues_.begin(), ccValues_.end(), nullptr);
967 return true;
968 }
969
EmitCompressEightUtf16ToUtf8CharsUsingSimd(Inst * inst)970 bool LLVMIrConstructor::EmitCompressEightUtf16ToUtf8CharsUsingSimd(Inst *inst)
971 {
972 CreateCompressUtf16ToUtf8CharsUsingSimd<VECTOR_SIZE_8>(inst);
973 return true;
974 }
975
EmitCompressSixteenUtf16ToUtf8CharsUsingSimd(Inst * inst)976 bool LLVMIrConstructor::EmitCompressSixteenUtf16ToUtf8CharsUsingSimd(Inst *inst)
977 {
978 CreateCompressUtf16ToUtf8CharsUsingSimd<VECTOR_SIZE_16>(inst);
979 return true;
980 }
981
EmitMemCharU8X16UsingSimd(Inst * inst)982 bool LLVMIrConstructor::EmitMemCharU8X16UsingSimd(Inst *inst)
983 {
984 ASSERT(GetGraph()->GetArch() == Arch::AARCH64);
985 ASSERT(GetGraph()->GetMode().IsFastPath());
986 ASSERT(inst->GetInputType(0) == DataType::UINT8);
987 ASSERT(inst->GetInputType(1) == DataType::POINTER);
988
989 MemCharSimdLowering memCharLowering(GetInputValue(inst, 0), GetInputValue(inst, 1), GetBuilder(), GetFunc());
990 ValueMapAdd(inst, memCharLowering.Generate<false>(memCharLowering.GetU8X16Ty()));
991 return true;
992 }
993
EmitMemCharU8X32UsingSimd(Inst * inst)994 bool LLVMIrConstructor::EmitMemCharU8X32UsingSimd(Inst *inst)
995 {
996 ASSERT(GetGraph()->GetArch() == Arch::AARCH64);
997 ASSERT(GetGraph()->GetMode().IsFastPath());
998 ASSERT(inst->GetInputType(0) == DataType::UINT8);
999 ASSERT(inst->GetInputType(1) == DataType::POINTER);
1000
1001 MemCharSimdLowering memCharLowering(GetInputValue(inst, 0), GetInputValue(inst, 1), GetBuilder(), GetFunc());
1002 ValueMapAdd(inst, memCharLowering.Generate<true>(memCharLowering.GetU8X16Ty()));
1003 return true;
1004 }
1005
EmitMemCharU16X8UsingSimd(Inst * inst)1006 bool LLVMIrConstructor::EmitMemCharU16X8UsingSimd(Inst *inst)
1007 {
1008 ASSERT(GetGraph()->GetArch() == Arch::AARCH64);
1009 ASSERT(GetGraph()->GetMode().IsFastPath());
1010 ASSERT(inst->GetInputType(0) == DataType::UINT16);
1011 ASSERT(inst->GetInputType(1) == DataType::POINTER);
1012
1013 MemCharSimdLowering memCharLowering(GetInputValue(inst, 0), GetInputValue(inst, 1), GetBuilder(), GetFunc());
1014 ValueMapAdd(inst, memCharLowering.Generate<false>(memCharLowering.GetU16X8Ty()));
1015 return true;
1016 }
1017
EmitMemCharU16X16UsingSimd(Inst * inst)1018 bool LLVMIrConstructor::EmitMemCharU16X16UsingSimd(Inst *inst)
1019 {
1020 ASSERT(GetGraph()->GetArch() == Arch::AARCH64);
1021 ASSERT(GetGraph()->GetMode().IsFastPath());
1022 ASSERT(inst->GetInputType(0) == DataType::UINT16);
1023 ASSERT(inst->GetInputType(1) == DataType::POINTER);
1024
1025 MemCharSimdLowering memCharLowering(GetInputValue(inst, 0), GetInputValue(inst, 1), GetBuilder(), GetFunc());
1026 ValueMapAdd(inst, memCharLowering.Generate<true>(memCharLowering.GetU16X8Ty()));
1027 return true;
1028 }
1029
EmitReverseBytes(Inst * inst)1030 bool LLVMIrConstructor::EmitReverseBytes(Inst *inst)
1031 {
1032 ASSERT(IsSafeCast(inst, 0));
1033 auto result = builder_.CreateUnaryIntrinsic(llvm::Intrinsic::bswap, GetInputValue(inst, 0), nullptr);
1034 ValueMapAdd(inst, result);
1035 return true;
1036 }
1037
EmitMemoryFenceFull(Inst * inst)1038 bool LLVMIrConstructor::EmitMemoryFenceFull([[maybe_unused]] Inst *inst)
1039 {
1040 CreateMemoryFence(memory_order::FULL);
1041 return true;
1042 }
1043
EmitMemoryFenceRelease(Inst * inst)1044 bool LLVMIrConstructor::EmitMemoryFenceRelease([[maybe_unused]] Inst *inst)
1045 {
1046 CreateMemoryFence(memory_order::RELEASE);
1047 return true;
1048 }
1049
EmitMemoryFenceAcquire(Inst * inst)1050 bool LLVMIrConstructor::EmitMemoryFenceAcquire([[maybe_unused]] Inst *inst)
1051 {
1052 CreateMemoryFence(memory_order::ACQUIRE);
1053 return true;
1054 }
1055
EmitFround(Inst * inst)1056 bool LLVMIrConstructor::EmitFround(Inst *inst)
1057 {
1058 llvm::Value *input = GetInputValue(inst, 0);
1059 ASSERT_TYPE(input, builder_.getDoubleTy());
1060 auto isNan = CreateIsNan(input);
1061 auto floatCasted = builder_.CreateCast(llvm::Instruction::FPTrunc, input, builder_.getFloatTy());
1062 auto casted = builder_.CreateCast(llvm::Instruction::FPExt, floatCasted, builder_.getDoubleTy());
1063 llvm::Value *nan = llvm::ConstantFP::getQNaN(builder_.getDoubleTy());
1064 auto result = builder_.CreateSelect(isNan, nan, casted);
1065 ValueMapAdd(inst, result);
1066 return true;
1067 }
1068
EmitCtlz(Inst * inst)1069 bool LLVMIrConstructor::EmitCtlz(Inst *inst)
1070 {
1071 auto result = CreateZerosCount(inst, llvm::Intrinsic::ctlz);
1072 ValueMapAdd(inst, result);
1073 return true;
1074 }
1075
EmitCttz(Inst * inst)1076 bool LLVMIrConstructor::EmitCttz(Inst *inst)
1077 {
1078 auto result = CreateZerosCount(inst, llvm::Intrinsic::cttz);
1079 ValueMapAdd(inst, result);
1080 return true;
1081 }
1082
EmitSignbit(Inst * inst)1083 bool LLVMIrConstructor::EmitSignbit(Inst *inst)
1084 {
1085 auto num = GetInputValue(inst, 0);
1086 auto bitcast = builder_.CreateBitCast(num, builder_.getInt64Ty());
1087 auto cmp = builder_.CreateICmpSLT(bitcast, builder_.getInt64(0));
1088 ValueMapAdd(inst, cmp);
1089 return true;
1090 }
1091
EmitIsInteger(Inst * inst)1092 bool LLVMIrConstructor::EmitIsInteger(Inst *inst)
1093 {
1094 auto result = CreateIsInteger(inst, GetInputValue(inst, 0));
1095 ValueMapAdd(inst, result);
1096 return true;
1097 }
1098
EmitIsSafeInteger(Inst * inst)1099 bool LLVMIrConstructor::EmitIsSafeInteger(Inst *inst)
1100 {
1101 auto &ctx = func_->getContext();
1102 auto input = GetInputValue(inst, 0);
1103 ASSERT(input->getType()->isDoubleTy() || input->getType()->isFloatTy());
1104 auto isInteger = CreateIsInteger(inst, input);
1105
1106 auto maxSafe = input->getType()->isDoubleTy() ? llvm::ConstantFP::get(builder_.getDoubleTy(), MaxIntAsExactDouble())
1107 : llvm::ConstantFP::get(builder_.getFloatTy(), MaxIntAsExactFloat());
1108
1109 auto initialBb = GetCurrentBasicBlock();
1110 auto isSafeIntegerBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "is_safe_integer"), func_);
1111 auto continueBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "is_safe_integer_continue"), func_);
1112
1113 builder_.CreateCondBr(isInteger, isSafeIntegerBb, continueBb);
1114
1115 SetCurrentBasicBlock(isSafeIntegerBb);
1116 // fabs(v) <= MaxSafeInteger
1117 auto inputAbs = builder_.CreateUnaryIntrinsic(llvm::Intrinsic::fabs, input);
1118 auto cmp = builder_.CreateFCmp(llvm::CmpInst::FCMP_OLE, inputAbs, maxSafe);
1119 builder_.CreateBr(continueBb);
1120
1121 SetCurrentBasicBlock(continueBb);
1122 auto result = builder_.CreatePHI(builder_.getInt1Ty(), 2U);
1123 result->addIncoming(builder_.getInt1(false), initialBb);
1124 result->addIncoming(cmp, isSafeIntegerBb);
1125
1126 ValueMapAdd(inst, result);
1127 return true;
1128 }
1129
EmitRawBitcastToInt(Inst * inst)1130 bool LLVMIrConstructor::EmitRawBitcastToInt(Inst *inst)
1131 {
1132 llvm::Value *input = GetInputValue(inst, 0);
1133 ASSERT_TYPE(input, builder_.getFloatTy());
1134 auto result = builder_.CreateBitCast(input, builder_.getInt32Ty());
1135 ValueMapAdd(inst, result);
1136 return true;
1137 }
1138
EmitRawBitcastToLong(Inst * inst)1139 bool LLVMIrConstructor::EmitRawBitcastToLong(Inst *inst)
1140 {
1141 llvm::Value *input = GetInputValue(inst, 0);
1142 ASSERT_TYPE(input, builder_.getDoubleTy());
1143 auto result = builder_.CreateBitCast(input, builder_.getInt64Ty());
1144 ValueMapAdd(inst, result);
1145 return true;
1146 }
1147
EmitRawBitcastFromInt(Inst * inst)1148 bool LLVMIrConstructor::EmitRawBitcastFromInt(Inst *inst)
1149 {
1150 llvm::Value *input = GetInputValue(inst, 0);
1151 ASSERT_TYPE(input, builder_.getInt32Ty());
1152 auto result = builder_.CreateBitCast(input, builder_.getFloatTy());
1153 ValueMapAdd(inst, result);
1154 return true;
1155 }
1156
EmitRawBitcastFromLong(Inst * inst)1157 bool LLVMIrConstructor::EmitRawBitcastFromLong(Inst *inst)
1158 {
1159 llvm::Value *input = GetInputValue(inst, 0);
1160 ASSERT_TYPE(input, builder_.getInt64Ty());
1161 auto result = builder_.CreateBitCast(input, builder_.getDoubleTy());
1162 ValueMapAdd(inst, result);
1163 return true;
1164 }
1165
EmitStringGetCharsTlab(Inst * inst)1166 bool LLVMIrConstructor::EmitStringGetCharsTlab(Inst *inst)
1167 {
1168 auto offset = GetGraph()->GetRuntime()->GetArrayU16ClassPointerTlsOffset(GetGraph()->GetArch());
1169 auto klass = llvmbackend::runtime_calls::LoadTLSValue(&builder_, arkInterface_, offset, builder_.getPtrTy());
1170 auto eid = RuntimeInterface::EntrypointId::STRING_GET_CHARS_TLAB_COMPRESSED;
1171 auto result = CreateEntrypointCall(eid, inst,
1172 {GetInputValue(inst, 0), GetInputValue(inst, 1), GetInputValue(inst, 2), klass});
1173 ASSERT(result->getCallingConv() == llvm::CallingConv::C);
1174 result->setCallingConv(llvm::CallingConv::ArkFast4);
1175 result->addRetAttr(llvm::Attribute::NonNull);
1176 result->addRetAttr(llvm::Attribute::NoAlias);
1177 ValueMapAdd(inst, result);
1178 return true;
1179 }
1180
EmitStringHashCode(Inst * inst)1181 bool LLVMIrConstructor::EmitStringHashCode(Inst *inst)
1182 {
1183 ASSERT(GetGraph()->GetRuntime()->IsCompressedStringsEnabled());
1184 auto string = GetInputValue(inst, 0);
1185 auto offset = coretypes::String::GetHashcodeOffset();
1186 auto gep = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), string, offset);
1187 auto hashCode = builder_.CreateLoad(builder_.getInt32Ty(), gep);
1188 auto isZero = builder_.CreateICmpEQ(hashCode, llvm::Constant::getNullValue(hashCode->getType()));
1189 auto fastPath = GetCurrentBasicBlock();
1190 auto slowPath = llvm::BasicBlock::Create(func_->getContext(), "hash_code_slow_path", func_);
1191 auto continuation = llvm::BasicBlock::Create(func_->getContext(), "hash_code_continuation", func_);
1192 auto branchWeights = llvm::MDBuilder(func_->getContext())
1193 .createBranchWeights(llvmbackend::Metadata::BranchWeights::UNLIKELY_BRANCH_WEIGHT,
1194 llvmbackend::Metadata::BranchWeights::LIKELY_BRANCH_WEIGHT);
1195 builder_.CreateCondBr(isZero, slowPath, continuation, branchWeights);
1196 SetCurrentBasicBlock(slowPath);
1197
1198 auto newHash = CreateEntrypointCall(RuntimeInterface::EntrypointId::STRING_HASH_CODE_COMPRESSED, inst, {string});
1199 ASSERT(newHash->getCallingConv() == llvm::CallingConv::C);
1200 newHash->setCallingConv(llvm::CallingConv::ArkFast1);
1201 builder_.CreateBr(continuation);
1202 SetCurrentBasicBlock(continuation);
1203
1204 auto result = builder_.CreatePHI(hashCode->getType(), 2U);
1205 result->addIncoming(hashCode, fastPath);
1206 result->addIncoming(newHash, slowPath);
1207 ValueMapAdd(inst, result);
1208
1209 return true;
1210 }
1211
EmitWriteTlabStatsSafe(Inst * inst)1212 bool LLVMIrConstructor::EmitWriteTlabStatsSafe(Inst *inst)
1213 {
1214 auto addr = GetInputValue(inst, 0);
1215 auto size = GetInputValue(inst, 1);
1216 CreateEntrypointCall(RuntimeInterface::EntrypointId::WRITE_TLAB_STATS_NO_BRIDGE, inst, {addr, size});
1217
1218 return true;
1219 }
1220
EmitExpandU8U16(Inst * inst)1221 bool LLVMIrConstructor::EmitExpandU8U16(Inst *inst)
1222 {
1223 auto input = GetInputValue(inst, 0);
1224 ASSERT(input->getType()->getScalarSizeInBits() == 32U); // has to be f32
1225
1226 auto srcTy = llvm::VectorType::get(builder_.getInt8Ty(), 4U, false);
1227 auto dstTy = llvm::VectorType::get(builder_.getInt16Ty(), 4U, false);
1228
1229 auto val = builder_.CreateBitCast(input, srcTy);
1230 auto result = builder_.CreateZExt(val, dstTy);
1231 ValueMapAdd(inst, result);
1232
1233 return true;
1234 }
1235
EmitReverseHalfWords(Inst * inst)1236 bool LLVMIrConstructor::EmitReverseHalfWords(Inst *inst)
1237 {
1238 auto input = GetInputValue(inst, 0);
1239 ASSERT(input->getType()->getScalarSizeInBits() == 64U); // has to be f64
1240 auto srcTy = llvm::VectorType::get(builder_.getInt16Ty(), 4U, false);
1241 auto val = builder_.CreateBitCast(input, srcTy);
1242
1243 const llvm::SmallVector<int, 4> indices = {3, 2, 1, 0};
1244 auto result = builder_.CreateShuffleVector(val, indices);
1245 ValueMapAdd(inst, result);
1246
1247 return true;
1248 }
1249
EmitAtomicByteOr(Inst * inst)1250 bool LLVMIrConstructor::EmitAtomicByteOr(Inst *inst)
1251 {
1252 auto addr = GetInputValue(inst, 0);
1253 auto value = GetInputValue(inst, 1);
1254 auto byteVal = builder_.CreateTrunc(value, builder_.getInt8Ty());
1255 auto op = llvm::AtomicRMWInst::BinOp::Or;
1256 builder_.CreateAtomicRMW(op, addr, byteVal, llvm::MaybeAlign(0), llvm::AtomicOrdering::Monotonic);
1257
1258 return true;
1259 }
1260
GetMappedValue(Inst * inst,DataType::Type type)1261 llvm::Value *LLVMIrConstructor::GetMappedValue(Inst *inst, DataType::Type type)
1262 {
1263 ASSERT(inputMap_.count(inst) == 1);
1264 auto &typeMap = inputMap_.at(inst);
1265 ASSERT(typeMap.count(type) == 1);
1266 auto result = typeMap.at(type);
1267 ASSERT(result != nullptr);
1268 return result;
1269 }
1270
GetInputValue(Inst * inst,size_t index,bool skipCoerce)1271 llvm::Value *LLVMIrConstructor::GetInputValue(Inst *inst, size_t index, bool skipCoerce)
1272 {
1273 auto input = inst->GetInput(index).GetInst();
1274 auto type = inst->GetInputType(index);
1275 ASSERT(type != DataType::NO_TYPE);
1276
1277 if (skipCoerce) {
1278 ASSERT(input->GetType() == DataType::UINT64 || input->GetType() == DataType::INT64);
1279 type = input->GetType();
1280 }
1281
1282 if (input->IsConst()) {
1283 return GetInputValueFromConstant(input->CastToConstant(), type);
1284 }
1285 if (input->GetOpcode() == Opcode::NullPtr) {
1286 auto llvmType = GetExactType(DataType::REFERENCE);
1287 ASSERT(llvmType == builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
1288 return llvm::Constant::getNullValue(llvmType);
1289 }
1290 return GetMappedValue(input, type);
1291 }
1292
GetInputValueFromConstant(ConstantInst * constant,DataType::Type pandaType)1293 llvm::Value *LLVMIrConstructor::GetInputValueFromConstant(ConstantInst *constant, DataType::Type pandaType)
1294 {
1295 auto llvmType = GetExactType(pandaType);
1296 if (pandaType == DataType::FLOAT64) {
1297 double value = constant->GetDoubleValue();
1298 return llvm::ConstantFP::get(llvmType, value);
1299 }
1300 if (pandaType == DataType::FLOAT32) {
1301 float value = constant->GetFloatValue();
1302 return llvm::ConstantFP::get(llvmType, value);
1303 }
1304 if (pandaType == DataType::POINTER) {
1305 auto cval = static_cast<int64_t>(constant->GetIntValue());
1306 auto integer = builder_.getInt64(cval);
1307 return builder_.CreateIntToPtr(integer, builder_.getPtrTy());
1308 }
1309 if (DataType::IsTypeNumeric(pandaType)) {
1310 auto isSigned = DataType::IsTypeSigned(pandaType);
1311 auto cval = static_cast<int64_t>(constant->GetIntValue());
1312 return llvm::ConstantInt::get(llvmType, cval, isSigned);
1313 }
1314 if (DataType::IsReference(pandaType) && constant->GetRawValue() == 0) {
1315 return llvm::Constant::getNullValue(llvmType);
1316 }
1317 UNREACHABLE();
1318 }
1319
1320 // Initializers. BuildIr calls them
1321
BuildBasicBlocks(Marker normal)1322 void LLVMIrConstructor::BuildBasicBlocks(Marker normal)
1323 {
1324 auto &context = func_->getContext();
1325 for (auto block : graph_->GetBlocksRPO()) {
1326 if (block->IsEndBlock()) {
1327 continue;
1328 }
1329 if (!block->IsMarked(normal)) {
1330 continue;
1331 }
1332 auto bb = llvm::BasicBlock::Create(context, llvm::StringRef("bb") + llvm::Twine(block->GetId()), func_);
1333 AddBlock(block, bb);
1334 // Checking that irtoc handler contains a return instruction
1335 if (!graph_->GetMode().IsInterpreter()) {
1336 continue;
1337 }
1338 for (auto inst : block->AllInsts()) {
1339 if (inst->IsIntrinsic() && inst->CastToIntrinsic()->GetIntrinsicId() ==
1340 RuntimeInterface::IntrinsicId::INTRINSIC_INTERPRETER_RETURN) {
1341 arkInterface_->AppendIrtocReturnHandler(func_->getName());
1342 }
1343 }
1344 }
1345 }
1346
BuildInstructions(Marker normal)1347 void LLVMIrConstructor::BuildInstructions(Marker normal)
1348 {
1349 for (auto block : graph_->GetBlocksRPO()) {
1350 if (block->IsEndBlock() || !block->IsMarked(normal)) {
1351 continue;
1352 }
1353 SetCurrentBasicBlock(GetTailBlock(block));
1354 for (auto inst : block->AllInsts()) {
1355 auto bb = GetCurrentBasicBlock();
1356 if (!bb->empty() && llvm::isa<llvm::UnreachableInst>(*(bb->rbegin()))) {
1357 break;
1358 }
1359 VisitInstruction(inst);
1360 }
1361
1362 if (block->IsTryBegin()) {
1363 ASSERT(block->GetSuccsBlocks().size() > 1);
1364 ASSERT(block->GetSuccessor(0)->IsMarked(normal) && !block->GetSuccessor(1)->IsMarked(normal));
1365 ASSERT(!block->GetLastInst()->IsControlFlow());
1366 builder_.CreateBr(GetHeadBlock(block->GetSuccessor(0)));
1367 }
1368 if (((block->GetSuccsBlocks().size() == 1 && !block->GetSuccessor(0)->IsEndBlock()) || block->IsTryEnd()) &&
1369 block->GetSuccessor(0)->IsMarked(normal)) {
1370 ASSERT(block->IsTryEnd() ? !block->GetSuccessor(1)->IsMarked(normal) : true);
1371 builder_.CreateBr(GetHeadBlock(block->GetSuccessor(0)));
1372 }
1373 ReplaceTailBlock(block, GetCurrentBasicBlock());
1374 }
1375 }
1376
FillPhiInputs(BasicBlock * block,Marker normal)1377 void LLVMIrConstructor::FillPhiInputs(BasicBlock *block, Marker normal)
1378 {
1379 if (block->IsStartBlock() || block->IsEndBlock() || !block->IsMarked(normal)) {
1380 return;
1381 }
1382 for (auto inst : block->PhiInsts()) {
1383 auto phi = llvm::cast<llvm::PHINode>(GetMappedValue(inst, inst->GetType()));
1384 for (size_t i = 0; i < inst->GetInputsCount(); i++) {
1385 auto inputBlock = inst->CastToPhi()->GetPhiInputBb(i);
1386 if (!inputBlock->IsMarked(normal)) {
1387 continue;
1388 }
1389
1390 auto input = GetInputValue(inst, i);
1391 phi->addIncoming(input, GetTailBlock(inputBlock));
1392 }
1393 }
1394 }
1395
1396 // Creator functions for internal usage
1397
CreateEntrypointCall(RuntimeInterface::EntrypointId eid,Inst * inst,llvm::ArrayRef<llvm::Value * > args)1398 llvm::CallInst *LLVMIrConstructor::CreateEntrypointCall(RuntimeInterface::EntrypointId eid, Inst *inst,
1399 llvm::ArrayRef<llvm::Value *> args)
1400 {
1401 arkInterface_->GetOrCreateRuntimeFunctionType(func_->getContext(), func_->getParent(),
1402 LLVMArkInterface::RuntimeCallType::ENTRYPOINT,
1403 static_cast<LLVMArkInterface::EntrypointId>(eid));
1404
1405 // Sanity assert to not misuse this scenario
1406 ASSERT(inst != nullptr);
1407
1408 llvm::CallInst *call;
1409 auto threadReg = GetThreadRegValue();
1410 if (GetGraph()->SupportManagedCode() && (inst->CanThrow() || inst->CanDeoptimize())) {
1411 bool noReturn = GetGraph()->GetRuntime()->IsEntrypointNoreturn(eid);
1412 call = llvmbackend::runtime_calls::CreateEntrypointCallCommon(
1413 &builder_, threadReg, arkInterface_, static_cast<llvmbackend::runtime_calls::EntrypointId>(eid), args,
1414 CreateSaveStateBundle(inst, noReturn));
1415 } else {
1416 call = llvmbackend::runtime_calls::CreateEntrypointCallCommon(
1417 &builder_, threadReg, arkInterface_, static_cast<llvmbackend::runtime_calls::EntrypointId>(eid), args);
1418 }
1419 if (inst->RequireState()) {
1420 WrapArkCall(inst, call);
1421 }
1422 return call;
1423 }
1424
CreateIntrinsicCall(Inst * inst)1425 llvm::CallInst *LLVMIrConstructor::CreateIntrinsicCall(Inst *inst)
1426 {
1427 auto entryId = inst->CastToIntrinsic()->GetIntrinsicId();
1428 auto rtFunctionTy = arkInterface_->GetOrCreateRuntimeFunctionType(
1429 func_->getContext(), func_->getParent(), LLVMArkInterface::RuntimeCallType::INTRINSIC,
1430 static_cast<LLVMArkInterface::EntrypointId>(entryId));
1431 auto arguments = GetIntrinsicArguments(rtFunctionTy, inst->CastToIntrinsic());
1432 return CreateIntrinsicCall(inst, entryId, arguments);
1433 }
1434
CreateIntrinsicCall(Inst * inst,RuntimeInterface::IntrinsicId entryId,llvm::ArrayRef<llvm::Value * > arguments)1435 llvm::CallInst *LLVMIrConstructor::CreateIntrinsicCall(Inst *inst, RuntimeInterface::IntrinsicId entryId,
1436 llvm::ArrayRef<llvm::Value *> arguments)
1437 {
1438 auto rtFunctionTy = arkInterface_->GetOrCreateRuntimeFunctionType(
1439 func_->getContext(), func_->getParent(), LLVMArkInterface::RuntimeCallType::INTRINSIC,
1440 static_cast<LLVMArkInterface::EntrypointId>(entryId));
1441 auto rtFunctionName = arkInterface_->GetRuntimeFunctionName(LLVMArkInterface::RuntimeCallType::INTRINSIC,
1442 static_cast<LLVMArkInterface::EntrypointId>(entryId));
1443 auto intrinsicOffset = static_cast<int>(entryId);
1444 auto callee = llvmbackend::runtime_calls::GetPandaRuntimeFunctionCallee(intrinsicOffset, rtFunctionTy, &builder_,
1445 rtFunctionName);
1446 llvm::CallInst *result;
1447 if (inst->CanThrow()) {
1448 ASSERT_PRINT(inst->GetSaveState() != nullptr, "Intrinsic with can_throw does not have a savestate");
1449 result = builder_.CreateCall(callee, arguments, CreateSaveStateBundle(inst));
1450 } else {
1451 result = builder_.CreateCall(callee, arguments);
1452 }
1453 SetIntrinsicParamAttrs(result, inst->CastToIntrinsic(), arguments);
1454
1455 if (inst->RequireState()) {
1456 WrapArkCall(inst, result);
1457 }
1458 if (NeedSafePointAfterIntrinsic(entryId) && g_options.IsCompilerUseSafepoint()) {
1459 result->addFnAttr(llvm::Attribute::get(result->getContext(), "needs-extra-safepoint"));
1460 result->getFunction()->addFnAttr("needs-extra-safepoint");
1461 }
1462
1463 return result;
1464 }
1465
1466 // Helper function. Regardless of where we use `alloca` to pass args, we want to do all of them in the
1467 // first basic block. This should allow LLVM to combine allocas into prologue
CreateAllocaForArgs(llvm::Type * type,uint32_t arraySize)1468 llvm::Value *LLVMIrConstructor::CreateAllocaForArgs(llvm::Type *type, uint32_t arraySize)
1469 {
1470 auto currentBb = GetCurrentBasicBlock();
1471 auto &firstBb = func_->getEntryBlock();
1472 auto inst = firstBb.getFirstNonPHI();
1473 builder_.SetInsertPoint(inst);
1474 llvm::AllocaInst *result;
1475
1476 if (llvm::isa<llvm::AllocaInst>(inst)) {
1477 auto alloca = llvm::cast<llvm::AllocaInst>(inst);
1478 ASSERT(alloca->getAllocatedType() == type);
1479 ASSERT(llvm::isa<llvm::ConstantInt>(alloca->getArraySize()));
1480 auto allocaSize = llvm::cast<llvm::ConstantInt>(alloca->getArraySize())->getZExtValue();
1481 if (allocaSize < arraySize) {
1482 alloca->setOperand(0, builder_.getInt32(arraySize));
1483 }
1484 result = alloca;
1485 } else {
1486 result = builder_.CreateAlloca(type, builder_.getInt32(arraySize), "call_arg_buffer");
1487 }
1488
1489 SetCurrentBasicBlock(currentBb);
1490 return result;
1491 }
1492
CreateFastPathCall(Inst * inst,RuntimeInterface::EntrypointId eid,llvm::ArrayRef<llvm::Value * > args)1493 llvm::CallInst *LLVMIrConstructor::CreateFastPathCall(Inst *inst, RuntimeInterface::EntrypointId eid,
1494 llvm::ArrayRef<llvm::Value *> args)
1495 {
1496 auto call = CreateEntrypointCall(eid, inst, args);
1497 ASSERT(call->getCallingConv() == llvm::CallingConv::C);
1498 call->setCallingConv(GetFastPathCallingConv(args.size()));
1499 return call;
1500 }
1501
1502 // IsInstance Helpers
1503
CreateIsInstanceEntrypointCall(Inst * inst)1504 llvm::Value *LLVMIrConstructor::CreateIsInstanceEntrypointCall(Inst *inst)
1505 {
1506 auto object = GetInputValue(inst, 0);
1507 auto klass = GetInputValue(inst, 1);
1508 return CreateEntrypointCall(RuntimeInterface::EntrypointId::IS_INSTANCE, inst, {object, klass});
1509 }
1510
CreateIsInstanceObject(llvm::Value * klassObj)1511 llvm::Value *LLVMIrConstructor::CreateIsInstanceObject(llvm::Value *klassObj)
1512 {
1513 auto typeOffset = GetGraph()->GetRuntime()->GetClassTypeOffset(GetGraph()->GetArch());
1514 auto typeMask = GetGraph()->GetRuntime()->GetReferenceTypeMask();
1515 auto typePtr = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), klassObj, typeOffset);
1516 auto typeLdr = builder_.CreateLoad(builder_.getInt8Ty(), typePtr);
1517 auto cmpLocal =
1518 builder_.CreateICmpEQ(builder_.getInt32(typeMask), builder_.CreateZExt(typeLdr, builder_.getInt32Ty()));
1519 return builder_.CreateZExt(cmpLocal, builder_.getInt8Ty(), "isinstance_object_out");
1520 }
1521
CreateIsInstanceOther(Inst * inst,llvm::Value * klassObj,llvm::Value * klassId)1522 llvm::Value *LLVMIrConstructor::CreateIsInstanceOther(Inst *inst, llvm::Value *klassObj, llvm::Value *klassId)
1523 {
1524 auto initialBb = GetCurrentBasicBlock();
1525 auto &ctx = func_->getContext();
1526 auto loopHeaderBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "other_loop_h"), func_);
1527 auto loopBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "other_loop"), func_);
1528 auto outBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "other_out"), func_);
1529 builder_.CreateBr(loopHeaderBb);
1530
1531 SetCurrentBasicBlock(loopHeaderBb);
1532 auto typeOffset = GetGraph()->GetRuntime()->GetClassBaseOffset(GetGraph()->GetArch());
1533 auto loopPhi = builder_.CreatePHI(builder_.getPtrTy(), 2U, "loop_in");
1534 auto typePtr = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), loopPhi, typeOffset);
1535 auto typeLdr = builder_.CreateLoad(builder_.getPtrTy(), typePtr);
1536 auto cmpLocal = builder_.CreateIsNotNull(typeLdr);
1537 loopPhi->addIncoming(klassObj, initialBb);
1538 loopPhi->addIncoming(typeLdr, loopBb);
1539 builder_.CreateCondBr(cmpLocal, loopBb, outBb);
1540
1541 SetCurrentBasicBlock(loopBb);
1542 cmpLocal = builder_.CreateICmpEQ(typeLdr, klassId);
1543 builder_.CreateCondBr(cmpLocal, outBb, loopHeaderBb);
1544
1545 SetCurrentBasicBlock(outBb);
1546 auto outPhi = builder_.CreatePHI(builder_.getInt8Ty(), 2U, "isinstance_other_out");
1547 outPhi->addIncoming(builder_.getInt8(1), loopBb);
1548 outPhi->addIncoming(builder_.getInt8(0), loopHeaderBb);
1549 return outPhi;
1550 }
1551
CreateIsInstanceArray(Inst * inst,llvm::Value * klassObj,llvm::Value * klassId)1552 llvm::Value *LLVMIrConstructor::CreateIsInstanceArray(Inst *inst, llvm::Value *klassObj, llvm::Value *klassId)
1553 {
1554 auto &ctx = func_->getContext();
1555 auto initialBb = GetCurrentBasicBlock();
1556 auto secondLoadBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "array_second_load"), func_);
1557 auto slowPath = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "array_slow_path"), func_);
1558 auto outBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "array_out"), func_);
1559
1560 auto componentOffset = GetGraph()->GetRuntime()->GetClassComponentTypeOffset(GetGraph()->GetArch());
1561 auto typePtrObj = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), klassObj, componentOffset);
1562 auto typeLdrObj = builder_.CreateLoad(builder_.getPtrTy(), typePtrObj);
1563 auto cmpLocal = builder_.CreateIsNotNull(typeLdrObj);
1564 builder_.CreateCondBr(cmpLocal, secondLoadBb, outBb);
1565
1566 SetCurrentBasicBlock(secondLoadBb);
1567 auto typePtrKlass = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), klassId, componentOffset);
1568 auto typeLdrKlass = builder_.CreateLoad(builder_.getPtrTy(), typePtrKlass);
1569 cmpLocal = builder_.CreateICmpEQ(typeLdrObj, typeLdrKlass);
1570 auto branchWeights = llvm::MDBuilder(ctx).createBranchWeights(
1571 llvmbackend::Metadata::BranchWeights::LIKELY_BRANCH_WEIGHT, // if other comparisons are enough
1572 llvmbackend::Metadata::BranchWeights::UNLIKELY_BRANCH_WEIGHT); // else
1573 builder_.CreateCondBr(cmpLocal, outBb, slowPath, branchWeights);
1574
1575 SetCurrentBasicBlock(slowPath);
1576 auto slowPathResult = CreateIsInstanceEntrypointCall(inst);
1577 builder_.CreateBr(outBb);
1578
1579 SetCurrentBasicBlock(outBb);
1580 auto outPhi = builder_.CreatePHI(builder_.getInt8Ty(), 3U, "isinstance_array_out");
1581 outPhi->addIncoming(builder_.getInt8(0), initialBb);
1582 outPhi->addIncoming(builder_.getInt8(1), secondLoadBb);
1583 outPhi->addIncoming(slowPathResult, slowPath);
1584 return outPhi;
1585 }
1586
CreateIsInstanceArrayObject(Inst * inst,llvm::Value * klassObj)1587 llvm::Value *LLVMIrConstructor::CreateIsInstanceArrayObject(Inst *inst, llvm::Value *klassObj)
1588 {
1589 auto &ctx = func_->getContext();
1590 auto initialBb = GetCurrentBasicBlock();
1591 auto checkMaskBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "array_object_check_mask"), func_);
1592 auto outBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "array_object_out"), func_);
1593
1594 auto componentOffset = GetGraph()->GetRuntime()->GetClassComponentTypeOffset(GetGraph()->GetArch());
1595 auto typePtr = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), klassObj, componentOffset);
1596 auto typeLdr = builder_.CreateLoad(builder_.getPtrTy(), typePtr);
1597 auto cmpLocal = builder_.CreateIsNotNull(typeLdr);
1598 builder_.CreateCondBr(cmpLocal, checkMaskBb, outBb);
1599
1600 SetCurrentBasicBlock(checkMaskBb);
1601 auto typeOffset = GetGraph()->GetRuntime()->GetClassTypeOffset(GetGraph()->GetArch());
1602 auto typeMask = GetGraph()->GetRuntime()->GetReferenceTypeMask();
1603 auto typePtrElem = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), typeLdr, typeOffset);
1604 auto typeLdrElem = builder_.CreateLoad(builder_.getInt8Ty(), typePtrElem);
1605 cmpLocal =
1606 builder_.CreateICmpEQ(builder_.getInt32(typeMask), builder_.CreateZExt(typeLdrElem, builder_.getInt32Ty()));
1607 auto cmpExt = builder_.CreateZExt(cmpLocal, builder_.getInt8Ty());
1608 builder_.CreateBr(outBb);
1609
1610 SetCurrentBasicBlock(outBb);
1611 auto outPhi = builder_.CreatePHI(builder_.getInt8Ty(), 2U, "isinstance_array_object_out");
1612 outPhi->addIncoming(builder_.getInt8(0), initialBb);
1613 outPhi->addIncoming(cmpExt, checkMaskBb);
1614 return outPhi;
1615 }
1616
CreateIsInstanceInnerBlock(Inst * inst,llvm::Value * klassObj,llvm::Value * klassId)1617 llvm::Value *LLVMIrConstructor::CreateIsInstanceInnerBlock(Inst *inst, llvm::Value *klassObj, llvm::Value *klassId)
1618 {
1619 auto klassType = inst->CastToIsInstance()->GetClassType();
1620 switch (klassType) {
1621 case ClassType::OBJECT_CLASS:
1622 return CreateIsInstanceObject(klassObj);
1623 case ClassType::OTHER_CLASS:
1624 return CreateIsInstanceOther(inst, klassObj, klassId);
1625 case ClassType::ARRAY_CLASS:
1626 return CreateIsInstanceArray(inst, klassObj, klassId);
1627 case ClassType::ARRAY_OBJECT_CLASS:
1628 return CreateIsInstanceArrayObject(inst, klassObj);
1629 case ClassType::INTERFACE_CLASS:
1630 return CreateIsInstanceEntrypointCall(inst);
1631 default:
1632 UNREACHABLE();
1633 }
1634 }
1635
1636 // IsInstance Helpers End
1637
1638 // CheckCast Helpers
1639
CreateCheckCastEntrypointCall(Inst * inst)1640 void LLVMIrConstructor::CreateCheckCastEntrypointCall(Inst *inst)
1641 {
1642 auto object = GetInputValue(inst, 0);
1643 auto klass = GetInputValue(inst, 1);
1644 if (inst->CanDeoptimize()) {
1645 auto call = CreateEntrypointCall(RuntimeInterface::EntrypointId::CHECK_CAST_DEOPTIMIZE, inst, {object, klass});
1646 call->addFnAttr(llvm::Attribute::get(call->getContext(), "may-deoptimize"));
1647 } else {
1648 CreateEntrypointCall(RuntimeInterface::EntrypointId::CHECK_CAST, inst, {object, klass});
1649 }
1650 }
1651
CreateCheckCastObject(Inst * inst,llvm::Value * klassObj,llvm::Value * klassId)1652 void LLVMIrConstructor::CreateCheckCastObject(Inst *inst, llvm::Value *klassObj, llvm::Value *klassId)
1653 {
1654 auto typeOffset = GetGraph()->GetRuntime()->GetClassTypeOffset(GetGraph()->GetArch());
1655 auto typeMask = GetGraph()->GetRuntime()->GetReferenceTypeMask();
1656 auto typePtr = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), klassObj, typeOffset);
1657 auto typeLdr = builder_.CreateLoad(builder_.getInt8Ty(), typePtr);
1658 auto src = GetInputValue(inst, 0);
1659 auto zext = builder_.CreateZExt(typeLdr, builder_.getInt32Ty());
1660 auto deoptimize = builder_.CreateICmpNE(builder_.getInt32(typeMask), zext);
1661
1662 auto exception = RuntimeInterface::EntrypointId::CLASS_CAST_EXCEPTION;
1663 CreateDeoptimizationBranch(inst, deoptimize, exception, {klassId, src});
1664 }
1665
CreateCheckCastOther(Inst * inst,llvm::Value * klassObj,llvm::Value * klassId)1666 void LLVMIrConstructor::CreateCheckCastOther(Inst *inst, llvm::Value *klassObj, llvm::Value *klassId)
1667 {
1668 auto initialBb = GetCurrentBasicBlock();
1669 auto src = GetInputValue(inst, 0);
1670
1671 auto &ctx = func_->getContext();
1672 auto loopHeaderBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "other_loop_h"), func_);
1673 auto loopBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "other_loop"), func_);
1674 auto outBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "other_out"), func_);
1675 builder_.CreateBr(loopHeaderBb);
1676
1677 SetCurrentBasicBlock(loopHeaderBb);
1678 auto typeOffset = GetGraph()->GetRuntime()->GetClassBaseOffset(GetGraph()->GetArch());
1679 auto loopPhi = builder_.CreatePHI(builder_.getPtrTy(), 2U, "loop_in");
1680 auto typePtr = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), loopPhi, typeOffset);
1681 auto typeLdr = builder_.CreateLoad(builder_.getPtrTy(), typePtr);
1682 auto deoptimize = builder_.CreateIsNull(typeLdr);
1683 loopPhi->addIncoming(klassObj, initialBb);
1684 loopPhi->addIncoming(typeLdr, loopBb);
1685
1686 auto exception = RuntimeInterface::EntrypointId::CLASS_CAST_EXCEPTION;
1687 CreateDeoptimizationBranch(inst, deoptimize, exception, {klassId, src});
1688 builder_.CreateBr(loopBb);
1689
1690 SetCurrentBasicBlock(loopBb);
1691 auto cmp = builder_.CreateICmpEQ(typeLdr, klassId);
1692 builder_.CreateCondBr(cmp, outBb, loopHeaderBb);
1693
1694 SetCurrentBasicBlock(outBb);
1695 }
1696
CreateCheckCastArray(Inst * inst,llvm::Value * klassObj,llvm::Value * klassId)1697 void LLVMIrConstructor::CreateCheckCastArray(Inst *inst, llvm::Value *klassObj, llvm::Value *klassId)
1698 {
1699 auto src = GetInputValue(inst, 0);
1700
1701 auto &ctx = func_->getContext();
1702 auto slowPath = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "array_slow_path"), func_);
1703 auto outBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "array_out"), func_);
1704
1705 auto componentOffset = GetGraph()->GetRuntime()->GetClassComponentTypeOffset(GetGraph()->GetArch());
1706 auto typePtrObj = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), klassObj, componentOffset);
1707 auto typeLdrObj = builder_.CreateLoad(builder_.getPtrTy(), typePtrObj);
1708
1709 auto deoptimize = builder_.CreateIsNull(typeLdrObj);
1710 auto exception = RuntimeInterface::EntrypointId::CLASS_CAST_EXCEPTION;
1711 CreateDeoptimizationBranch(inst, deoptimize, exception, {klassId, src});
1712
1713 auto typePtrKlass = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), klassId, componentOffset);
1714 auto typeLdrKlass = builder_.CreateLoad(builder_.getPtrTy(), typePtrKlass);
1715 auto cmpLocal = builder_.CreateICmpEQ(typeLdrObj, typeLdrKlass);
1716 auto branchWeights = llvm::MDBuilder(ctx).createBranchWeights(
1717 llvmbackend::Metadata::BranchWeights::LIKELY_BRANCH_WEIGHT, // if other comparisons are enough
1718 llvmbackend::Metadata::BranchWeights::UNLIKELY_BRANCH_WEIGHT); // else
1719 builder_.CreateCondBr(cmpLocal, outBb, slowPath, branchWeights);
1720
1721 SetCurrentBasicBlock(slowPath);
1722 CreateCheckCastEntrypointCall(inst);
1723 builder_.CreateBr(outBb);
1724
1725 SetCurrentBasicBlock(outBb);
1726 }
1727
CreateCheckCastArrayObject(Inst * inst,llvm::Value * klassObj,llvm::Value * klassId)1728 void LLVMIrConstructor::CreateCheckCastArrayObject(Inst *inst, llvm::Value *klassObj, llvm::Value *klassId)
1729 {
1730 auto src = GetInputValue(inst, 0);
1731
1732 auto componentOffset = GetGraph()->GetRuntime()->GetClassComponentTypeOffset(GetGraph()->GetArch());
1733 auto typePtr = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), klassObj, componentOffset);
1734 auto typeLdr = builder_.CreateLoad(builder_.getPtrTy(), typePtr);
1735
1736 auto deoptimize = builder_.CreateIsNull(typeLdr);
1737 auto exception = RuntimeInterface::EntrypointId::CLASS_CAST_EXCEPTION;
1738 CreateDeoptimizationBranch(inst, deoptimize, exception, {klassId, src});
1739
1740 auto typeOffset = GetGraph()->GetRuntime()->GetClassTypeOffset(GetGraph()->GetArch());
1741 auto typeMask = GetGraph()->GetRuntime()->GetReferenceTypeMask();
1742 auto typePtrElem = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), typeLdr, typeOffset);
1743 auto typeLdrElem = builder_.CreateLoad(builder_.getInt8Ty(), typePtrElem);
1744 deoptimize =
1745 builder_.CreateICmpNE(builder_.getInt32(typeMask), builder_.CreateZExt(typeLdrElem, builder_.getInt32Ty()));
1746 CreateDeoptimizationBranch(inst, deoptimize, exception, {klassId, src});
1747 }
1748
CreateCheckCastInner(Inst * inst,llvm::Value * klassObj,llvm::Value * klassId)1749 void LLVMIrConstructor::CreateCheckCastInner(Inst *inst, llvm::Value *klassObj, llvm::Value *klassId)
1750 {
1751 auto klassType = inst->CastToCheckCast()->GetClassType();
1752 switch (klassType) {
1753 case ClassType::OBJECT_CLASS:
1754 CreateCheckCastObject(inst, klassObj, klassId);
1755 break;
1756 case ClassType::OTHER_CLASS:
1757 CreateCheckCastOther(inst, klassObj, klassId);
1758 break;
1759 case ClassType::ARRAY_CLASS:
1760 CreateCheckCastArray(inst, klassObj, klassId);
1761 break;
1762 case ClassType::ARRAY_OBJECT_CLASS:
1763 CreateCheckCastArrayObject(inst, klassObj, klassId);
1764 break;
1765 case ClassType::INTERFACE_CLASS:
1766 default:
1767 UNREACHABLE();
1768 }
1769 }
1770
1771 // CheckCast Helpers End
1772
CreateInterpreterReturnRestoreRegs(RegMask & regMask,size_t offset,bool fp)1773 void LLVMIrConstructor::CreateInterpreterReturnRestoreRegs(RegMask ®Mask, size_t offset, bool fp)
1774 {
1775 int32_t slotSize = PointerSize(GetGraph()->GetArch());
1776 int32_t dslotSize = slotSize * 2U;
1777 int32_t totalSize = regMask.count() * slotSize;
1778 auto startRegOffset = offset * DOUBLE_WORD_SIZE_BYTES;
1779 auto endRegOffset = startRegOffset + std::max(0, totalSize - dslotSize);
1780
1781 constexpr uint32_t MAX_REPR_VAL = 504U;
1782 bool representable = startRegOffset <= MAX_REPR_VAL && (startRegOffset & 0x7U) == 0 &&
1783 endRegOffset <= MAX_REPR_VAL && (endRegOffset & 0x7U) == 0;
1784
1785 std::string baseReg = representable ? "sp" : "x16";
1786 if (!representable) {
1787 CreateInt32ImmAsm(&builder_,
1788 std::string("add x16, sp, $0").append(LLVMArkInterface::PATCH_STACK_ADJUSTMENT_COMMENT),
1789 startRegOffset);
1790 startRegOffset = 0;
1791 }
1792
1793 while (regMask.count() > 0) {
1794 std::string asmString = regMask.count() / 2U > 0 ? "ldp " : "ldr ";
1795 auto first = regMask.GetMinRegister();
1796 asmString += (fp ? "d" : "x") + std::to_string(first);
1797 regMask ^= 1U << first;
1798 if (regMask.count() > 0) {
1799 asmString += ", ";
1800 auto second = regMask.GetMinRegister();
1801 asmString += (fp ? "d" : "x") + std::to_string(second);
1802 regMask ^= 1U << second;
1803 }
1804 asmString += ", [";
1805 asmString += baseReg;
1806 asmString += ", $0]";
1807 if (representable) {
1808 asmString += LLVMArkInterface::PATCH_STACK_ADJUSTMENT_COMMENT;
1809 }
1810 CreateInt32ImmAsm(&builder_, asmString, startRegOffset);
1811 startRegOffset += dslotSize;
1812 }
1813 }
1814
CreateLoadClassById(Inst * inst,uint32_t typeId,bool init)1815 llvm::Value *LLVMIrConstructor::CreateLoadClassById(Inst *inst, uint32_t typeId, bool init)
1816 {
1817 auto builtin = init ? LoadInitClass(func_->getParent()) : LoadClass(func_->getParent());
1818 auto slotIdVal = builder_.getInt32(arkInterface_->GetClassIndexInAotGot(GetGraph()->GetAotData(), typeId, init));
1819
1820 // remember two functions, later we will use it in panda_runtime_lowering pass
1821 arkInterface_->GetOrCreateRuntimeFunctionType(
1822 func_->getContext(), func_->getParent(), LLVMArkInterface::RuntimeCallType::ENTRYPOINT,
1823 static_cast<LLVMArkInterface::EntrypointId>(RuntimeInterface::EntrypointId::CLASS_RESOLVER));
1824 arkInterface_->GetOrCreateRuntimeFunctionType(
1825 func_->getContext(), func_->getParent(), LLVMArkInterface::RuntimeCallType::ENTRYPOINT,
1826 static_cast<LLVMArkInterface::EntrypointId>(RuntimeInterface::EntrypointId::CLASS_INIT_RESOLVER));
1827
1828 auto callInst = builder_.CreateCall(builtin, {builder_.getInt32(typeId), slotIdVal}, CreateSaveStateBundle(inst));
1829 WrapArkCall(inst, callInst);
1830 return callInst;
1831 }
1832
CreateBinaryOp(Inst * inst,llvm::Instruction::BinaryOps opcode)1833 llvm::Value *LLVMIrConstructor::CreateBinaryOp(Inst *inst, llvm::Instruction::BinaryOps opcode)
1834 {
1835 llvm::Value *x = GetInputValue(inst, 0);
1836 llvm::Value *y = GetInputValue(inst, 1);
1837
1838 if (x->getType()->isPointerTy()) {
1839 if (y->getType()->isPointerTy()) {
1840 ASSERT(opcode == llvm::Instruction::Sub);
1841 x = builder_.CreatePtrToInt(x, builder_.getInt64Ty());
1842 y = builder_.CreatePtrToInt(y, builder_.getInt64Ty());
1843 return builder_.CreateBinOp(opcode, x, y);
1844 }
1845 if (y->getType()->isIntegerTy()) {
1846 ASSERT(opcode == llvm::Instruction::Add);
1847 ASSERT(x->getType()->isPointerTy());
1848 return builder_.CreateInBoundsGEP(builder_.getInt8Ty(), x, y);
1849 }
1850 UNREACHABLE();
1851 }
1852 if (IsTypeNumeric(inst->GetType())) {
1853 // Peephole can remove casts and instead put a constant with the wrong type
1854 // so we need to create them here.
1855 x = CoerceValue(x, inst->GetInputType(0), inst->GetType());
1856 y = CoerceValue(y, inst->GetInputType(1), inst->GetType());
1857 }
1858 return builder_.CreateBinOp(opcode, x, y);
1859 }
1860
CreateBinaryImmOp(Inst * inst,llvm::Instruction::BinaryOps opcode,uint64_t c)1861 llvm::Value *LLVMIrConstructor::CreateBinaryImmOp(Inst *inst, llvm::Instruction::BinaryOps opcode, uint64_t c)
1862 {
1863 ASSERT(IsTypeNumeric(inst->GetType()));
1864 llvm::Value *x = GetInputValue(inst, 0);
1865 if (x->getType()->isPointerTy()) {
1866 ASSERT(x->getType()->isPointerTy());
1867 ASSERT(opcode == llvm::Instruction::Add || opcode == llvm::Instruction::Sub);
1868 if (opcode == llvm::Instruction::Sub) {
1869 c = -c;
1870 }
1871 return builder_.CreateConstInBoundsGEP1_64(builder_.getInt8Ty(), x, c);
1872 }
1873 llvm::Value *y = CoerceValue(builder_.getInt64(c), DataType::INT64, inst->GetType());
1874 return builder_.CreateBinOp(opcode, x, y);
1875 }
1876
CreateShiftOp(Inst * inst,llvm::Instruction::BinaryOps opcode)1877 llvm::Value *LLVMIrConstructor::CreateShiftOp(Inst *inst, llvm::Instruction::BinaryOps opcode)
1878 {
1879 llvm::Value *x = GetInputValue(inst, 0);
1880 llvm::Value *y = GetInputValue(inst, 1);
1881 auto targetType = inst->GetType();
1882 bool target64 = (targetType == DataType::UINT64) || (targetType == DataType::INT64);
1883 auto constexpr SHIFT32_RANGE = 0x1f;
1884 auto constexpr SHIFT64_RANGE = 0x3f;
1885
1886 y = builder_.CreateBinOp(llvm::Instruction::And, y,
1887 llvm::ConstantInt::get(y->getType(), target64 ? SHIFT64_RANGE : SHIFT32_RANGE));
1888
1889 return builder_.CreateBinOp(opcode, x, y);
1890 }
1891
CreateSignDivMod(Inst * inst,llvm::Instruction::BinaryOps opcode)1892 llvm::Value *LLVMIrConstructor::CreateSignDivMod(Inst *inst, llvm::Instruction::BinaryOps opcode)
1893 {
1894 ASSERT(opcode == llvm::Instruction::SDiv || opcode == llvm::Instruction::SRem);
1895 llvm::Value *x = GetInputValue(inst, 0);
1896 llvm::Value *y = GetInputValue(inst, 1);
1897 auto &ctx = func_->getContext();
1898 auto eqM1 = builder_.CreateICmpEQ(y, llvm::ConstantInt::get(y->getType(), -1));
1899 auto m1Result = opcode == llvm::Instruction::SDiv ? builder_.CreateNeg(x) : llvm::ConstantInt::get(y->getType(), 0);
1900
1901 // Select for AArch64, as 'sdiv' correctly handles the INT_MIN / -1 case
1902 if (GetGraph()->GetArch() == Arch::AARCH64) {
1903 auto result = builder_.CreateBinOp(opcode, x, y);
1904 auto selectVal = builder_.CreateSelect(eqM1, m1Result, result);
1905 if (auto selectInst = llvm::dyn_cast<llvm::SelectInst>(selectVal)) {
1906 auto *metadata = llvm::MDNode::get(ctx, {});
1907 auto sdiv = ark::llvmbackend::LLVMArkInterface::AARCH64_SDIV_INST;
1908 selectInst->setMetadata(sdiv, metadata);
1909 }
1910 return selectVal;
1911 }
1912
1913 // X86_64 solution with control flow
1914 auto currBb = GetCurrentBasicBlock();
1915 auto notM1Bb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "divmod_normal"), func_);
1916 auto contBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "divmod_cont"), func_);
1917 builder_.CreateCondBr(eqM1, contBb, notM1Bb);
1918
1919 SetCurrentBasicBlock(notM1Bb);
1920 auto result = builder_.CreateBinOp(opcode, x, y);
1921 builder_.CreateBr(contBb);
1922
1923 SetCurrentBasicBlock(contBb);
1924 auto resultPhi = builder_.CreatePHI(y->getType(), 2U);
1925 resultPhi->addIncoming(m1Result, currBb);
1926 resultPhi->addIncoming(result, notM1Bb);
1927 return resultPhi;
1928 }
1929
CreateFloatComparison(CmpInst * cmpInst,llvm::Value * x,llvm::Value * y)1930 llvm::Value *LLVMIrConstructor::CreateFloatComparison(CmpInst *cmpInst, llvm::Value *x, llvm::Value *y)
1931 {
1932 // if x is less than y then return -1
1933 // else return zero extend of (x > y)
1934 llvm::CmpInst::Predicate greaterThanPredicate;
1935 llvm::CmpInst::Predicate lessThanPredicate;
1936 if (cmpInst->IsFcmpg()) {
1937 // if x or y is nan then greaterThanPredicate yields true
1938 greaterThanPredicate = llvm::CmpInst::FCMP_UGT;
1939 lessThanPredicate = llvm::CmpInst::FCMP_OLT;
1940 } else if (cmpInst->IsFcmpl()) {
1941 greaterThanPredicate = llvm::CmpInst::FCMP_OGT;
1942 // if x or y is nan then lessThanPredicate yields true
1943 lessThanPredicate = llvm::CmpInst::FCMP_ULT;
1944 } else {
1945 ASSERT_PRINT(false, "cmpInst must be either Fcmpg, or Fcmpl");
1946 UNREACHABLE();
1947 }
1948 // x > y || (inst == Fcmpg && (x == NaN || y == NaN))
1949 auto greaterThan = builder_.CreateFCmp(greaterThanPredicate, x, y);
1950 // x < y || (inst == Fcmpl && (x == NaN || y == NaN))
1951 auto lessThan = builder_.CreateFCmp(lessThanPredicate, x, y);
1952 auto comparison = builder_.CreateZExt(greaterThan, builder_.getInt32Ty());
1953 auto negativeOne = builder_.getInt32(-1);
1954 return builder_.CreateSelect(lessThan, negativeOne, comparison);
1955 }
1956
CreateIntegerComparison(CmpInst * inst,llvm::Value * x,llvm::Value * y)1957 llvm::Value *LLVMIrConstructor::CreateIntegerComparison(CmpInst *inst, llvm::Value *x, llvm::Value *y)
1958 {
1959 ASSERT(x->getType() == y->getType());
1960 llvm::Value *greaterThan;
1961 llvm::Value *lessThan;
1962
1963 if (DataType::IsTypeSigned(inst->GetOperandsType())) {
1964 greaterThan = builder_.CreateICmpSGT(x, y);
1965 lessThan = builder_.CreateICmpSLT(x, y);
1966 } else {
1967 greaterThan = builder_.CreateICmpUGT(x, y);
1968 lessThan = builder_.CreateICmpULT(x, y);
1969 }
1970 auto castComparisonResult = builder_.CreateZExt(greaterThan, builder_.getInt32Ty());
1971 auto negativeOne = builder_.getInt32(-1);
1972 return builder_.CreateSelect(lessThan, negativeOne, castComparisonResult);
1973 }
1974
CreateNewArrayWithRuntime(Inst * inst)1975 llvm::Value *LLVMIrConstructor::CreateNewArrayWithRuntime(Inst *inst)
1976 {
1977 auto type = GetInputValue(inst, 0);
1978 auto size = ToSizeT(GetInputValue(inst, 1));
1979 auto eid = RuntimeInterface::EntrypointId::CREATE_ARRAY;
1980 auto result = CreateEntrypointCall(eid, inst, {type, size});
1981 MarkAsAllocation(result);
1982 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
1983 result->addFnAttr(llvm::Attribute::get(result->getContext(), "needs-mem-barrier"));
1984 }
1985 return result;
1986 }
1987
CreateNewObjectWithRuntime(Inst * inst)1988 llvm::Value *LLVMIrConstructor::CreateNewObjectWithRuntime(Inst *inst)
1989 {
1990 auto initClass = GetInputValue(inst, 0);
1991 auto eid = RuntimeInterface::EntrypointId::CREATE_OBJECT_BY_CLASS;
1992 auto result = CreateEntrypointCall(eid, inst, {initClass});
1993 auto srcInst = inst->GetInput(0).GetInst();
1994 if (srcInst->GetOpcode() != Opcode::LoadAndInitClass ||
1995 GetGraph()->GetRuntime()->CanUseTlabForClass(srcInst->CastToLoadAndInitClass()->GetClass())) {
1996 MarkAsAllocation(result);
1997 }
1998 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
1999 result->addFnAttr(llvm::Attribute::get(result->getContext(), "needs-mem-barrier"));
2000 }
2001 return result;
2002 }
2003
CreateResolveVirtualCallBuiltin(Inst * inst,llvm::Value * thiz,uint32_t methodId)2004 llvm::Value *LLVMIrConstructor::CreateResolveVirtualCallBuiltin(Inst *inst, llvm::Value *thiz, uint32_t methodId)
2005 {
2006 ASSERT(thiz->getType()->isPointerTy());
2007
2008 auto builtin = ResolveVirtual(func_->getParent());
2009 arkInterface_->GetOrCreateRuntimeFunctionType(
2010 func_->getContext(), func_->getParent(), LLVMArkInterface::RuntimeCallType::ENTRYPOINT,
2011 static_cast<LLVMArkInterface::EntrypointId>(RuntimeInterface::EntrypointId::RESOLVE_VIRTUAL_CALL_AOT));
2012 arkInterface_->GetOrCreateRuntimeFunctionType(
2013 func_->getContext(), func_->getParent(), LLVMArkInterface::RuntimeCallType::ENTRYPOINT,
2014 static_cast<LLVMArkInterface::EntrypointId>(RuntimeInterface::EntrypointId::INTF_INLINE_CACHE));
2015
2016 auto zero = builder_.getInt64(0);
2017 auto arrayType = llvm::ArrayType::get(builder_.getInt64Ty(), 0);
2018 auto offset = builder_.CreateIntToPtr(zero, arrayType->getPointerTo());
2019 auto callInst =
2020 builder_.CreateCall(builtin, {thiz, ToSizeT(builder_.getInt32(methodId)), offset}, CreateSaveStateBundle(inst));
2021 WrapArkCall(inst, callInst);
2022 return builder_.CreateIntToPtr(callInst, builder_.getPtrTy());
2023 }
2024
CreateLoadManagedClassFromClass(llvm::Value * klass)2025 llvm::Value *LLVMIrConstructor::CreateLoadManagedClassFromClass(llvm::Value *klass)
2026 {
2027 ASSERT(klass->getType()->isPointerTy());
2028 auto dataOff = GetGraph()->GetRuntime()->GetManagedClassOffset(GetGraph()->GetArch());
2029 auto ptrData = builder_.CreateConstInBoundsGEP1_32(builder_.getInt8Ty(), klass, dataOff);
2030 return builder_.CreateLoad(builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE), ptrData);
2031 }
2032
CreateIsInf(llvm::Value * input)2033 llvm::Value *LLVMIrConstructor::CreateIsInf(llvm::Value *input)
2034 {
2035 llvm::Type *type = nullptr;
2036 uint64_t infMaskInt;
2037 if (input->getType()->isFloatTy()) {
2038 constexpr uint32_t INF_MASK_FLOAT = 0xff000000;
2039 infMaskInt = INF_MASK_FLOAT;
2040 type = builder_.getInt32Ty();
2041 } else {
2042 ASSERT_TYPE(input, builder_.getDoubleTy());
2043 constexpr uint64_t INF_MASK_DOUBLE = 0xffe0000000000000;
2044 infMaskInt = INF_MASK_DOUBLE;
2045 type = builder_.getInt64Ty();
2046 }
2047
2048 auto infMask = llvm::ConstantInt::get(type, infMaskInt);
2049 auto one = llvm::ConstantInt::get(type, 1);
2050 auto castedInput = builder_.CreateBitCast(input, type);
2051 auto shiftedInput = builder_.CreateShl(castedInput, one);
2052 auto result = builder_.CreateICmpEQ(shiftedInput, infMask);
2053 return result;
2054 }
2055
CreateIsInteger(Inst * inst,llvm::Value * input)2056 llvm::Value *LLVMIrConstructor::CreateIsInteger(Inst *inst, llvm::Value *input)
2057 {
2058 auto &ctx = func_->getContext();
2059 ASSERT(input->getType()->isDoubleTy() || input->getType()->isFloatTy());
2060
2061 auto isInf = CreateIsInf(input);
2062 auto epsilon = input->getType()->isDoubleTy()
2063 ? llvm::ConstantFP::get(builder_.getDoubleTy(), std::numeric_limits<double>::epsilon())
2064 : llvm::ConstantFP::get(builder_.getFloatTy(), std::numeric_limits<float>::epsilon());
2065
2066 auto initialBb = GetCurrentBasicBlock();
2067 auto notInfBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "is_integer_not_inf"), func_);
2068 auto continueBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "is_integer_continue"), func_);
2069
2070 builder_.CreateCondBr(isInf, continueBb, notInfBb);
2071
2072 SetCurrentBasicBlock(notInfBb);
2073 // fabs(v - trunc(v)) <= epsilon
2074 auto truncated = builder_.CreateUnaryIntrinsic(llvm::Intrinsic::trunc, input);
2075 auto diff = builder_.CreateFSub(input, truncated);
2076 auto diffAbs = builder_.CreateUnaryIntrinsic(llvm::Intrinsic::fabs, diff);
2077 auto cmp = builder_.CreateFCmp(llvm::CmpInst::FCMP_OLE, diffAbs, epsilon);
2078 builder_.CreateBr(continueBb);
2079
2080 SetCurrentBasicBlock(continueBb);
2081 auto result = builder_.CreatePHI(builder_.getInt1Ty(), 2U);
2082 result->addIncoming(builder_.getInt1(false), initialBb);
2083 result->addIncoming(cmp, notInfBb);
2084
2085 return result;
2086 }
2087
CreateCastToInt(Inst * inst)2088 llvm::Value *LLVMIrConstructor::CreateCastToInt(Inst *inst)
2089 {
2090 llvm::Value *input = GetInputValue(inst, 0);
2091 auto sourceType = input->getType();
2092 auto targetType = inst->GetType();
2093
2094 ASSERT_DO(sourceType->isFloatTy() || sourceType->isDoubleTy(),
2095 std::cerr << "Unexpected source type: " << GetTypeName(sourceType) << ". Should be a float or double."
2096 << std::endl);
2097
2098 auto llvmId = DataType::IsTypeSigned(targetType) ? llvm::Intrinsic::fptosi_sat : llvm::Intrinsic::fptoui_sat;
2099 ArenaVector<llvm::Type *> intrinsicTypes(GetGraph()->GetLocalAllocator()->Adapter());
2100 intrinsicTypes.assign({GetExactType(targetType), sourceType});
2101 return builder_.CreateIntrinsic(llvmId, intrinsicTypes, {input}, nullptr);
2102 }
2103
CreateLoadWithOrdering(Inst * inst,llvm::Value * value,llvm::AtomicOrdering ordering,const llvm::Twine & name)2104 llvm::Value *LLVMIrConstructor::CreateLoadWithOrdering(Inst *inst, llvm::Value *value, llvm::AtomicOrdering ordering,
2105 const llvm::Twine &name)
2106 {
2107 auto pandaType = inst->GetType();
2108 llvm::Type *type = GetExactType(pandaType);
2109
2110 auto load = builder_.CreateLoad(type, value, false, name); // C-like volatile is not applied
2111 if (ordering != LLVMArkInterface::NOT_ATOMIC_ORDER) {
2112 auto alignment = func_->getParent()->getDataLayout().getPrefTypeAlignment(type);
2113 load->setOrdering(ordering);
2114 load->setAlignment(llvm::Align(alignment));
2115 }
2116
2117 return load;
2118 }
2119
CreateStoreWithOrdering(llvm::Value * value,llvm::Value * ptr,llvm::AtomicOrdering ordering)2120 llvm::Value *LLVMIrConstructor::CreateStoreWithOrdering(llvm::Value *value, llvm::Value *ptr,
2121 llvm::AtomicOrdering ordering)
2122 {
2123 auto store = builder_.CreateStore(value, ptr, false); // C-like volatile is not applied
2124 if (ordering != LLVMArkInterface::NOT_ATOMIC_ORDER) {
2125 auto alignment = func_->getParent()->getDataLayout().getPrefTypeAlignment(value->getType());
2126 store->setAlignment(llvm::Align(alignment));
2127 store->setOrdering(ordering);
2128 }
2129 return store;
2130 }
2131
CreateZerosCount(Inst * inst,llvm::Intrinsic::ID llvmId)2132 llvm::Value *LLVMIrConstructor::CreateZerosCount(Inst *inst, llvm::Intrinsic::ID llvmId)
2133 {
2134 ASSERT(IsSafeCast(inst, 0));
2135 auto zeroDefined = llvm::ConstantInt::getFalse(func_->getContext());
2136 return builder_.CreateBinaryIntrinsic(llvmId, GetInputValue(inst, 0), zeroDefined, nullptr);
2137 }
2138
CreateRoundArm64(Inst * inst,bool is64)2139 llvm::Value *LLVMIrConstructor::CreateRoundArm64(Inst *inst, bool is64)
2140 {
2141 auto input = GetInputValue(inst, 0);
2142
2143 auto sourceType = is64 ? builder_.getDoubleTy() : builder_.getFloatTy();
2144 auto targetType = is64 ? builder_.getInt64Ty() : builder_.getInt32Ty();
2145
2146 double constexpr HALF = 0.5;
2147 auto half = llvm::ConstantFP::get(sourceType, HALF);
2148 auto zero = is64 ? builder_.getInt64(0) : builder_.getInt32(0);
2149
2150 auto initialBb = GetCurrentBasicBlock();
2151 auto &ctx = func_->getContext();
2152 auto module = func_->getParent();
2153
2154 // lround - fcvtas instruction (positive solved fine, NaN mapped to 0, but negatives ties wrong way)
2155 auto decl = llvm::Intrinsic::getDeclaration(module, llvm::Intrinsic::lround, {targetType, sourceType});
2156 llvm::Value *round = llvm::CallInst::Create(decl, input, "", initialBb);
2157
2158 // Check if rounded value less than zero (if not negative rounding is done)
2159 auto negative = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "neg"), func_);
2160 auto done = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "cont"), func_);
2161 auto lessThan = builder_.CreateICmpSLT(round, zero);
2162 builder_.CreateCondBr(lessThan, negative, done);
2163
2164 // CC-OFFNXT(C_RULE_ID_HORIZON_SPACE_SHIELD) false-positive
2165 // Negative input case, add 1 iff "input - round(input) == 0.5"
2166 SetCurrentBasicBlock(negative);
2167 // frinta instruction
2168 auto floatRound = builder_.CreateUnaryIntrinsic(llvm::Intrinsic::round, input, nullptr);
2169 auto sub = builder_.CreateBinOp(llvm::Instruction::FSub, input, floatRound);
2170 auto one = is64 ? builder_.getInt64(1) : builder_.getInt32(1);
2171 auto add = builder_.CreateBinOp(llvm::Instruction::Add, round, one);
2172 auto equal = builder_.CreateFCmp(llvm::CmpInst::FCMP_OEQ, sub, half);
2173 auto roundMayInc = builder_.CreateSelect(equal, add, round);
2174 builder_.CreateBr(done);
2175
2176 // Continue block
2177 SetCurrentBasicBlock(done);
2178 auto roundPhi = builder_.CreatePHI(targetType, 2U);
2179 roundPhi->addIncoming(round, initialBb);
2180 roundPhi->addIncoming(roundMayInc, negative);
2181 return roundPhi;
2182 }
2183
CreateNewStringFromCharsTlab(Inst * inst,llvm::Value * offset,llvm::Value * length,llvm::Value * array)2184 llvm::Value *LLVMIrConstructor::CreateNewStringFromCharsTlab(Inst *inst, llvm::Value *offset, llvm::Value *length,
2185 llvm::Value *array)
2186 {
2187 auto entryId = RuntimeInterface::EntrypointId::CREATE_STRING_FROM_CHAR_ARRAY_TLAB_COMPRESSED;
2188 ArenaVector<llvm::Value *> arguments(GetGraph()->GetLocalAllocator()->Adapter());
2189 auto callConv = llvm::CallingConv::ArkFast3;
2190 if (llvm::isa<llvm::Constant>(offset) && llvm::cast<llvm::Constant>(offset)->isNullValue()) {
2191 entryId = RuntimeInterface::EntrypointId::CREATE_STRING_FROM_ZERO_BASED_CHAR_ARRAY_TLAB_COMPRESSED;
2192 } else {
2193 arguments.push_back(offset);
2194 callConv = llvm::CallingConv::ArkFast4;
2195 }
2196 arguments.push_back(length);
2197 arguments.push_back(array);
2198 auto klassOffset = GetGraph()->GetRuntime()->GetStringClassPointerTlsOffset(GetGraph()->GetArch());
2199 auto klass = llvmbackend::runtime_calls::LoadTLSValue(&builder_, arkInterface_, klassOffset, builder_.getPtrTy());
2200 arguments.push_back(klass);
2201 auto result = CreateEntrypointCall(entryId, inst, arguments);
2202 ASSERT(result->getCallingConv() == llvm::CallingConv::C);
2203 result->setCallingConv(callConv);
2204 MarkAsAllocation(result);
2205 return result;
2206 }
2207
CreateNewStringFromStringTlab(Inst * inst,llvm::Value * stringVal)2208 llvm::Value *LLVMIrConstructor::CreateNewStringFromStringTlab(Inst *inst, llvm::Value *stringVal)
2209 {
2210 auto entryId = RuntimeInterface::EntrypointId::CREATE_STRING_FROM_STRING_TLAB_COMPRESSED;
2211 auto result = CreateEntrypointCall(entryId, inst, {stringVal});
2212 ASSERT(result->getCallingConv() == llvm::CallingConv::C);
2213 result->setCallingConv(llvm::CallingConv::ArkFast1);
2214 MarkAsAllocation(result);
2215 return result;
2216 }
2217
CreateLaunchArgsArray(CallInst * callInst,uint32_t argStart)2218 llvm::Value *LLVMIrConstructor::CreateLaunchArgsArray(CallInst *callInst, uint32_t argStart)
2219 {
2220 auto callArgsCount = callInst->GetInputsCount() - argStart - 1U; // last arg is a SaveState
2221 auto callArgs = CreateAllocaForArgs(builder_.getInt64Ty(), callArgsCount);
2222
2223 // Store actual call arguments
2224 for (size_t i = 0; i < callArgsCount; i++) {
2225 auto arg = GetInputValue(callInst, argStart + i);
2226
2227 auto type = callInst->GetInputType(argStart + i);
2228 auto typeSize = DataType::GetTypeSize(type, GetGraph()->GetArch());
2229 if (typeSize < DataType::GetTypeSize(DataType::INT32, GetGraph()->GetArch())) {
2230 arg = CoerceValue(arg, type, DataType::INT32);
2231 }
2232
2233 auto gep = builder_.CreateConstInBoundsGEP1_32(builder_.getInt64Ty(), callArgs, i);
2234 builder_.CreateStore(arg, gep);
2235 }
2236 return callArgs;
2237 }
2238
CreateLaunchCall(CallInst * callInst)2239 void LLVMIrConstructor::CreateLaunchCall([[maybe_unused]] CallInst *callInst)
2240 {
2241 #ifdef PANDA_WITH_ETS
2242 ArenaVector<llvm::Value *> args(GetGraph()->GetLocalAllocator()->Adapter());
2243
2244 if (callInst->GetOpcode() == Opcode::CallResolvedLaunchStatic ||
2245 callInst->GetOpcode() == Opcode::CallResolvedLaunchVirtual) {
2246 args.push_back(GetInputValue(callInst, 0));
2247 args.push_back(GetInputValue(callInst, 1));
2248
2249 auto argStart = callInst->GetOpcode() == Opcode::CallResolvedLaunchVirtual ? 3U : 2U;
2250 auto callArgs = CreateLaunchArgsArray(callInst, argStart);
2251
2252 args.push_back(callArgs);
2253
2254 if (callInst->GetOpcode() == Opcode::CallResolvedLaunchVirtual) {
2255 args.push_back(GetInputValue(callInst, 2U));
2256 }
2257 } else {
2258 if (callInst->GetOpcode() != Opcode::CallLaunchVirtual) {
2259 ASSERT_DO(false, (std::cerr << "Unexpected Launch Call: \n", callInst->Dump(&std::cerr, true)));
2260 UNREACHABLE();
2261 }
2262
2263 ASSERT(GetGraph()->GetAotData()->GetUseCha());
2264
2265 auto method = ark::llvmbackend::utils::CreateLoadMethodUsingVTable(
2266 GetInputValue(callInst, 1), func_, callInst->GetCallMethodId(), &builder_, arkInterface_);
2267 args.push_back(method);
2268 args.push_back(GetInputValue(callInst, 0));
2269
2270 auto callArgs = CreateLaunchArgsArray(callInst, 2U);
2271 args.push_back(callArgs);
2272 args.push_back(GetInputValue(callInst, 1));
2273 }
2274
2275 auto eid = callInst->IsStaticLaunchCall() ? RuntimeInterface::EntrypointId::CREATE_LAUNCH_STATIC_COROUTINE
2276 : RuntimeInterface::EntrypointId::CREATE_LAUNCH_VIRTUAL_COROUTINE;
2277 auto entryCall = CreateEntrypointCall(eid, callInst, args);
2278 if (callInst->GetOpcode() == Opcode::CallResolvedLaunchVirtual) {
2279 entryCall->addFnAttr(llvm::Attribute::get(entryCall->getContext(), "original-method-id",
2280 std::to_string(callInst->GetCallMethodId())));
2281 entryCall->addFnAttr(llvm::Attribute::get(entryCall->getContext(), "is-launch-call"));
2282 }
2283 if (callInst->GetFlag(inst_flags::MEM_BARRIER)) {
2284 entryCall->addFnAttr(llvm::Attribute::get(entryCall->getContext(), "needs-mem-barrier"));
2285 }
2286 #else
2287 UNREACHABLE();
2288 #endif
2289 }
2290
CreateDeoptimizationBranch(Inst * inst,llvm::Value * deoptimize,RuntimeInterface::EntrypointId exception,llvm::ArrayRef<llvm::Value * > arguments)2291 void LLVMIrConstructor::CreateDeoptimizationBranch(Inst *inst, llvm::Value *deoptimize,
2292 RuntimeInterface::EntrypointId exception,
2293 llvm::ArrayRef<llvm::Value *> arguments)
2294 {
2295 ASSERT_TYPE(deoptimize, builder_.getInt1Ty());
2296 ASSERT(exception != RuntimeInterface::EntrypointId::DEOPTIMIZE || inst->CanDeoptimize());
2297 auto &ctx = func_->getContext();
2298
2299 /* Create basic blocks for continuation and throw */
2300 auto continuation = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "cont"), func_);
2301 auto throwPath = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "deopt"), func_);
2302
2303 /* Creating branch */
2304 auto branchWeights = llvm::MDBuilder(ctx).createBranchWeights(
2305 llvmbackend::Metadata::BranchWeights::UNLIKELY_BRANCH_WEIGHT, // if unlikely(deoptimize) then throw
2306 llvmbackend::Metadata::BranchWeights::LIKELY_BRANCH_WEIGHT); // else continue
2307 auto branch = builder_.CreateCondBr(deoptimize, throwPath, continuation, branchWeights);
2308
2309 /* Creating throw block */
2310 SetCurrentBasicBlock(throwPath);
2311
2312 auto call = CreateDeoptimizeCall(inst, arguments, exception);
2313
2314 /* Set metadata for implicit null check */
2315 if (!inst->CanDeoptimize() && exception == RuntimeInterface::EntrypointId::NULL_POINTER_EXCEPTION &&
2316 g_options.IsCompilerImplicitNullCheck()) {
2317 ASSERT(inst->IsNullCheck());
2318 auto *metadata = llvm::MDNode::get(ctx, {});
2319 branch->setMetadata(llvm::LLVMContext::MD_make_implicit, metadata);
2320 }
2321
2322 /* Create 'ret' after llvm.experimental.deoptimize call */
2323 CreateReturn(call);
2324 WrapArkCall(inst, call);
2325
2326 /* Continue */
2327 SetCurrentBasicBlock(continuation);
2328 }
2329
CreateDeoptimizeCall(Inst * inst,llvm::ArrayRef<llvm::Value * > arguments,RuntimeInterface::EntrypointId exception)2330 llvm::CallInst *LLVMIrConstructor::CreateDeoptimizeCall(Inst *inst, llvm::ArrayRef<llvm::Value *> arguments,
2331 RuntimeInterface::EntrypointId exception)
2332 {
2333 auto deoptimizeDeclaration = llvm::Intrinsic::getDeclaration(
2334 func_->getParent(), llvm::Intrinsic::experimental_deoptimize, {func_->getReturnType()});
2335 llvm::CallInst *call;
2336 if (inst->CanDeoptimize()) {
2337 // If inst CanDeoptimize then call Deoptimize to bail out into interpreter, do not throw exception
2338 exception = RuntimeInterface::EntrypointId::DEOPTIMIZE;
2339 auto type = helpers::ToUnderlying(GetDeoptimizationType(inst)) |
2340 (inst->GetId() << MinimumBitsToStore(DeoptimizeType::COUNT));
2341 ASSERT(GetGraph()->GetRuntime()->IsEntrypointNoreturn(exception));
2342 call = builder_.CreateCall(
2343 {deoptimizeDeclaration}, {builder_.getInt64(type), builder_.getInt32(static_cast<uint32_t>(exception))},
2344 CreateSaveStateBundle(inst, GetGraph()->GetRuntime()->IsEntrypointNoreturn(exception)));
2345 call->addFnAttr(llvm::Attribute::get(call->getContext(), "may-deoptimize"));
2346 } else {
2347 std::vector<llvm::Value *> args = arguments;
2348 args.push_back(builder_.getInt32(static_cast<uint32_t>(exception)));
2349 call =
2350 builder_.CreateCall({deoptimizeDeclaration}, args,
2351 CreateSaveStateBundle(inst, GetGraph()->GetRuntime()->IsEntrypointNoreturn(exception)));
2352 }
2353 arkInterface_->GetOrCreateRuntimeFunctionType(func_->getContext(), func_->getParent(),
2354 LLVMArkInterface::RuntimeCallType::ENTRYPOINT,
2355 static_cast<LLVMArkInterface::EntrypointId>(exception));
2356 return call;
2357 }
2358
CreateSaveStateBundle(Inst * inst,bool noReturn)2359 ArenaVector<llvm::OperandBundleDef> LLVMIrConstructor::CreateSaveStateBundle(Inst *inst, bool noReturn)
2360 {
2361 ASSERT_PRINT(inst->CanThrow() || inst->CanDeoptimize(),
2362 "Attempt to create a regmap for instruction that doesn't throw (or deoptimize)");
2363 ArenaVector<llvm::OperandBundleDef> bundle(GetGraph()->GetLocalAllocator()->Adapter());
2364 if (!arkInterface_->DeoptsEnabled()) {
2365 return bundle;
2366 }
2367 ArenaVector<llvm::Value *> vals(GetGraph()->GetLocalAllocator()->Adapter());
2368 ArenaVector<SaveStateInst *> saveStates(GetGraph()->GetLocalAllocator()->Adapter());
2369
2370 auto saveState = inst->GetSaveState();
2371 while (saveState != nullptr) {
2372 saveStates.push_back(saveState);
2373 auto caller = saveState->GetCallerInst();
2374 saveState = caller == nullptr ? nullptr : caller->GetSaveState();
2375 }
2376
2377 std::reverse(saveStates.begin(), saveStates.end());
2378 for (auto ss : saveStates) {
2379 auto method = ss->GetMethod();
2380 auto caller = ss->GetCallerInst();
2381 if (caller != nullptr) {
2382 method = caller->GetCallMethod();
2383 }
2384 ASSERT(method != nullptr);
2385 // Put a function as a delimiter in inlining chain
2386 auto function = GetOrCreateFunctionForCall(caller, method);
2387 ASSERT(function != nullptr);
2388 vals.push_back(function);
2389 // Put methodId needed for inline info
2390 vals.push_back(builder_.getInt32(GetGraph()->GetRuntime()->GetMethodId(method)));
2391 // Put bytecode pc for inlining chain as well
2392 vals.push_back(builder_.getInt32(ss->GetPc()));
2393 // Put a marker if catch has been met
2394 uint32_t flags = (inst->RequireRegMap() ? 1U : 0U) | (noReturn ? 2U : 0U);
2395 vals.push_back(builder_.getInt32(flags));
2396 // Put a number of interpreter registers for the method
2397 auto vregCount = arkInterface_->GetVirtualRegistersCount(method);
2398 vals.push_back(builder_.getInt32(vregCount));
2399
2400 EncodeSaveStateInputs(&vals, ss);
2401 }
2402 bundle.assign({llvm::OperandBundleDef {"deopt", vals}});
2403 return bundle;
2404 }
2405
EncodeSaveStateInputs(ArenaVector<llvm::Value * > * vals,SaveStateInst * ss)2406 void LLVMIrConstructor::EncodeSaveStateInputs(ArenaVector<llvm::Value *> *vals, SaveStateInst *ss)
2407 {
2408 for (size_t i = 0; i < ss->GetInputsCount(); ++i) {
2409 if (ss->GetVirtualRegister(i).Value() == VirtualRegister::BRIDGE) {
2410 continue;
2411 }
2412 // Put a virtual register index
2413 vals->push_back(builder_.getInt32(ss->GetVirtualRegister(i).Value()));
2414 // Put a virtual register type
2415 auto metatype = IrTypeToMetainfoType(ss->GetInputType(i));
2416 uint32_t undertype = static_cast<std::underlying_type_t<VRegInfo::Type>>(metatype);
2417 vals->push_back(builder_.getInt32(undertype));
2418 // Put a virtual register value
2419 auto value = GetInputValue(ss, i);
2420 if (!value->getType()->isPointerTy()) {
2421 ASSERT(value->getType()->getScalarSizeInBits() <= 64U);
2422 auto intVal = builder_.CreateBitCast(value, builder_.getIntNTy(value->getType()->getScalarSizeInBits()));
2423 if (metatype == VRegInfo::Type::INT32) {
2424 intVal = CoerceValue(intVal, ss->GetInputType(i), DataType::INT32);
2425 }
2426 vals->push_back(builder_.CreateZExt(intVal, builder_.getInt64Ty()));
2427 } else {
2428 vals->push_back(value);
2429 }
2430 }
2431 }
2432
EncodeInlineInfo(Inst * inst,llvm::Instruction * instruction)2433 void LLVMIrConstructor::EncodeInlineInfo(Inst *inst, llvm::Instruction *instruction)
2434 {
2435 SaveStateInst *saveState = inst->GetSaveState();
2436 llvm::SmallVector<SaveStateInst *> saveStates;
2437 bool first = true;
2438 while (saveState != nullptr) {
2439 if (!first) {
2440 saveStates.push_back(saveState);
2441 }
2442 first = false;
2443 saveState = saveState->GetCallerInst() == nullptr ? nullptr : saveState->GetCallerInst()->GetSaveState();
2444 }
2445 llvm::reverse(saveStates);
2446 for (auto ss : saveStates) {
2447 auto method = ss->GetMethod();
2448 auto methodName = arkInterface_->GetUniqMethodName(method);
2449 auto function = func_->getParent()->getFunction(methodName);
2450 auto caller = ss->GetCallerInst();
2451 if (caller != nullptr) {
2452 method = ss->GetCallerInst()->GetCallMethod();
2453 function = GetOrCreateFunctionForCall(caller, method);
2454 }
2455 ASSERT(function != nullptr);
2456 debugData_->AppendInlinedAt(instruction, function, ss->GetPc());
2457 }
2458 }
2459
CreatePreWRB(Inst * inst,llvm::Value * mem)2460 void LLVMIrConstructor::CreatePreWRB(Inst *inst, llvm::Value *mem)
2461 {
2462 auto barrierType = GetGraph()->GetRuntime()->GetPreType();
2463 auto isVolatile = IsVolatileMemInst(inst);
2464 if (barrierType == mem::BarrierType::PRE_WRB_NONE) {
2465 ASSERT(GetGraph()->SupportManagedCode());
2466 return;
2467 }
2468 ASSERT(barrierType == mem::BarrierType::PRE_SATB_BARRIER);
2469
2470 if (llvmbackend::g_options.IsLlvmBuiltinWrb() && !arkInterface_->IsIrtocMode()) {
2471 auto builtin = llvmbackend::builtins::PreWRB(func_->getParent(), mem->getType()->getPointerAddressSpace());
2472 builder_.CreateCall(builtin, {mem, builder_.getInt1(isVolatile)});
2473 return;
2474 }
2475 auto &ctx = func_->getContext();
2476 auto outBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "pre_wrb_out"), func_);
2477 llvmbackend::gc_barriers::EmitPreWRB(&builder_, mem, isVolatile, outBb, arkInterface_, GetThreadRegValue());
2478 }
2479
CreatePostWRB(Inst * inst,llvm::Value * mem,llvm::Value * offset,llvm::Value * value)2480 void LLVMIrConstructor::CreatePostWRB(Inst *inst, llvm::Value *mem, llvm::Value *offset, llvm::Value *value)
2481 {
2482 auto barrierType = GetGraph()->GetRuntime()->GetPostType();
2483 if (barrierType == mem::BarrierType::POST_WRB_NONE) {
2484 ASSERT(GetGraph()->SupportManagedCode());
2485 return;
2486 }
2487 ASSERT(barrierType == mem::BarrierType::POST_INTERGENERATIONAL_BARRIER ||
2488 barrierType == mem::BarrierType::POST_INTERREGION_BARRIER);
2489
2490 Inst *secondValue;
2491 Inst *val = InstStoredValue(inst, &secondValue);
2492 ASSERT(secondValue == nullptr);
2493
2494 if (val->GetOpcode() == Opcode::NullPtr) {
2495 return;
2496 }
2497
2498 bool irtoc = arkInterface_->IsIrtocMode();
2499 if (!irtoc && llvmbackend::g_options.IsLlvmBuiltinWrb()) {
2500 auto builtin = llvmbackend::builtins::PostWRB(func_->getParent(), mem->getType()->getPointerAddressSpace());
2501 builder_.CreateCall(builtin, {mem, offset, value});
2502 return;
2503 }
2504 auto frame = (irtoc && GetGraph()->GetArch() == Arch::X86_64) ? GetRealFrameRegValue() : nullptr;
2505 llvmbackend::gc_barriers::EmitPostWRB(&builder_, mem, offset, value, arkInterface_, GetThreadRegValue(), frame);
2506 }
2507
CreateMemoryFence(memory_order::Order order)2508 llvm::Value *LLVMIrConstructor::CreateMemoryFence(memory_order::Order order)
2509 {
2510 llvm::AtomicOrdering ordering;
2511 switch (order) {
2512 case memory_order::RELEASE:
2513 ordering = llvm::AtomicOrdering::Release;
2514 break;
2515 case memory_order::ACQUIRE:
2516 ordering = llvm::AtomicOrdering::Acquire;
2517 break;
2518 case memory_order::FULL:
2519 ordering = llvm::AtomicOrdering::SequentiallyConsistent;
2520 break;
2521 default:
2522 UNREACHABLE();
2523 }
2524 return builder_.CreateFence(ordering);
2525 }
2526
CreateCondition(ConditionCode cc,llvm::Value * x,llvm::Value * y)2527 llvm::Value *LLVMIrConstructor::CreateCondition(ConditionCode cc, llvm::Value *x, llvm::Value *y)
2528 {
2529 if (cc == CC_TST_EQ || cc == CC_TST_NE) {
2530 auto tst = builder_.CreateBinOp(llvm::Instruction::And, x, y);
2531 return (cc == CC_TST_EQ) ? builder_.CreateIsNull(tst) : builder_.CreateIsNotNull(tst);
2532 }
2533 return builder_.CreateICmp(ICmpCodeConvert(cc), x, y);
2534 }
2535
CreateIf(Inst * inst,llvm::Value * cond,bool likely,bool unlikely)2536 void LLVMIrConstructor::CreateIf(Inst *inst, llvm::Value *cond, bool likely, bool unlikely)
2537 {
2538 llvm::MDNode *weights = nullptr;
2539 auto constexpr LIKELY = llvmbackend::Metadata::BranchWeights::LIKELY_BRANCH_WEIGHT;
2540 auto constexpr UNLIKELY = llvmbackend::Metadata::BranchWeights::UNLIKELY_BRANCH_WEIGHT;
2541 if (likely) {
2542 weights = llvm::MDBuilder(func_->getContext()).createBranchWeights(LIKELY, UNLIKELY);
2543 } else if (unlikely) {
2544 weights = llvm::MDBuilder(func_->getContext()).createBranchWeights(UNLIKELY, LIKELY);
2545 }
2546 builder_.CreateCondBr(cond, GetHeadBlock(inst->GetBasicBlock()->GetTrueSuccessor()),
2547 GetHeadBlock(inst->GetBasicBlock()->GetFalseSuccessor()), weights);
2548 }
2549
CreateReturn(llvm::Value * value)2550 llvm::Value *LLVMIrConstructor::CreateReturn(llvm::Value *value)
2551 {
2552 ASSERT(value != nullptr);
2553 if (value->getType()->isVoidTy()) {
2554 return builder_.CreateRetVoid();
2555 }
2556 return builder_.CreateRet(value);
2557 }
2558
CreateTailCallFastPath(Inst * inst)2559 llvm::CallInst *LLVMIrConstructor::CreateTailCallFastPath(Inst *inst)
2560 {
2561 ASSERT(inst->GetInputs().Size() == 0);
2562 ASSERT(inst->CastToIntrinsic()->HasImms() && inst->CastToIntrinsic()->GetImms().size() == 2U);
2563 ASSERT(ccValues_.size() == func_->arg_size());
2564
2565 ArenaVector<llvm::Value *> args(GetGraph()->GetLocalAllocator()->Adapter());
2566 uint32_t externalId = inst->CastToIntrinsic()->GetImms()[1];
2567 auto externalName = GetGraph()->GetRuntime()->GetExternalMethodName(GetGraph()->GetMethod(), externalId);
2568 auto callee = func_->getParent()->getFunction(externalName);
2569 llvm::CallingConv::ID cc = 0;
2570 if (callee == nullptr) {
2571 ArenaVector<llvm::Type *> argTypes(GetGraph()->GetLocalAllocator()->Adapter());
2572 for (size_t i = 0; i < func_->arg_size(); i++) {
2573 args.push_back(i < ccValues_.size() && ccValues_.at(i) != nullptr ? ccValues_.at(i) : func_->getArg(i));
2574 argTypes.push_back(args.at(i)->getType());
2575 }
2576 auto ftype = llvm::FunctionType::get(GetType(inst->GetType()), argTypes, false);
2577 callee = llvm::Function::Create(ftype, llvm::Function::ExternalLinkage, externalName, func_->getParent());
2578 cc = func_->getCallingConv();
2579 } else {
2580 size_t size = func_->arg_size();
2581 ASSERT(callee->arg_size() <= size);
2582 for (size_t i = 0; i < callee->arg_size() - 2U; i++) {
2583 args.push_back(i < ccValues_.size() && ccValues_.at(i) != nullptr ? ccValues_.at(i) : func_->getArg(i));
2584 }
2585 args.push_back(func_->getArg(size - 2U));
2586 args.push_back(func_->getArg(size - 1U));
2587 cc = callee->getCallingConv();
2588 }
2589 auto call = builder_.CreateCall(callee->getFunctionType(), callee, args);
2590 call->setCallingConv(cc);
2591 return call;
2592 }
2593
CreateTailCallInterpreter(Inst * inst)2594 llvm::CallInst *LLVMIrConstructor::CreateTailCallInterpreter(Inst *inst)
2595 {
2596 auto ptr = GetInputValue(inst, 0);
2597 ASSERT_TYPE(ptr, builder_.getPtrTy());
2598 ASSERT(ccValues_.size() == (GetGraph()->GetArch() == Arch::AARCH64 ? 8U : 7U));
2599 ASSERT(ccValues_.at(0) != nullptr); // pc
2600 static constexpr unsigned ACC = 1U;
2601 static constexpr unsigned ACC_TAG = 2U;
2602 ArenaVector<llvm::Type *> argTypes(GetGraph()->GetLocalAllocator()->Adapter());
2603 for (size_t i = 0; i < cc_.size(); i++) {
2604 if (ccValues_.at(i) != nullptr) {
2605 argTypes.push_back(ccValues_.at(i)->getType());
2606 } else {
2607 argTypes.push_back(func_->getFunctionType()->getParamType(i));
2608 }
2609 }
2610 if (ccValues_.at(ACC) == nullptr) {
2611 ccValues_[ACC] = llvm::Constant::getNullValue(argTypes[ACC]);
2612 }
2613 if (ccValues_.at(ACC_TAG) == nullptr) {
2614 ccValues_[ACC_TAG] = llvm::Constant::getNullValue(argTypes[ACC_TAG]);
2615 }
2616 ASSERT(ccValues_.at(3U) != nullptr); // frame
2617 ASSERT(ccValues_.at(4U) != nullptr); // dispatch
2618 if (GetGraph()->GetArch() == Arch::AARCH64) {
2619 ASSERT(ccValues_.at(5U) != nullptr); // moffset
2620 ASSERT(ccValues_.at(6U) != nullptr); // methodPtr
2621 ASSERT(ccValues_.at(7U) != nullptr); // thread
2622 } else {
2623 static constexpr unsigned REAL_FRAME_POINER = 6U;
2624 ASSERT(ccValues_.at(5U) != nullptr); // thread
2625 ASSERT(ccValues_.at(REAL_FRAME_POINER) == nullptr); // real frame pointer
2626 ccValues_[REAL_FRAME_POINER] = func_->getArg(REAL_FRAME_POINER);
2627 }
2628
2629 auto functionType = llvm::FunctionType::get(func_->getReturnType(), argTypes, false);
2630 auto call = builder_.CreateCall(functionType, ptr, ccValues_);
2631 call->setCallingConv(func_->getCallingConv());
2632 return call;
2633 }
2634
2635 template <uint32_t VECTOR_SIZE>
CreateCompressUtf16ToUtf8CharsUsingSimd(Inst * inst)2636 void LLVMIrConstructor::CreateCompressUtf16ToUtf8CharsUsingSimd(Inst *inst)
2637 {
2638 ASSERT(GetGraph()->GetArch() == Arch::AARCH64);
2639 ASSERT(GetGraph()->GetMode().IsFastPath());
2640 ASSERT(inst->GetInputType(0) == DataType::POINTER);
2641 ASSERT(inst->GetInputType(1) == DataType::POINTER);
2642 static_assert(VECTOR_SIZE == VECTOR_SIZE_8 || VECTOR_SIZE == VECTOR_SIZE_16, "Unexpected vector size");
2643 auto vecInTy = llvm::VectorType::get(builder_.getInt16Ty(), VECTOR_SIZE, false);
2644 auto vecOutTy = llvm::VectorType::get(builder_.getInt8Ty(), VECTOR_SIZE, false);
2645
2646 auto u16Ptr = GetInputValue(inst, 0); // ptr to src array of utf16 chars
2647 auto u8Ptr = GetInputValue(inst, 1); // ptr to dst array of utf8 chars
2648 auto inVec = builder_.CreateLoad(vecInTy, u16Ptr);
2649 auto outVec = builder_.CreateTrunc(inVec, vecOutTy);
2650 builder_.CreateStore(outVec, u8Ptr);
2651 }
2652
2653 // Getters
2654
GetEntryFunctionType()2655 llvm::FunctionType *LLVMIrConstructor::GetEntryFunctionType()
2656 {
2657 ArenaVector<llvm::Type *> argTypes(graph_->GetLocalAllocator()->Adapter());
2658
2659 // Method*
2660 if (graph_->SupportManagedCode()) {
2661 argTypes.push_back(builder_.getPtrTy());
2662 }
2663
2664 // ArkInt have fake parameters
2665 if (graph_->GetMode().IsInterpreter()) {
2666 for (size_t i = 0; i < cc_.size(); ++i) {
2667 argTypes.push_back(builder_.getPtrTy());
2668 }
2669 }
2670
2671 // Actual function arguments
2672 auto method = graph_->GetMethod();
2673 for (size_t i = 0; i < graph_->GetRuntime()->GetMethodTotalArgumentsCount(method); i++) {
2674 ASSERT(!graph_->GetMode().IsInterpreter());
2675 auto type = graph_->GetRuntime()->GetMethodTotalArgumentType(method, i);
2676 if (graph_->GetMode().IsFastPath()) {
2677 argTypes.push_back(GetExactType(type));
2678 } else {
2679 argTypes.push_back(GetType(type));
2680 }
2681 }
2682
2683 // ThreadReg and RealFP for FastPaths
2684 if (graph_->GetMode().IsFastPath()) {
2685 argTypes.push_back(builder_.getPtrTy());
2686 argTypes.push_back(builder_.getPtrTy());
2687 }
2688
2689 auto retType = graph_->GetRuntime()->GetMethodReturnType(method);
2690 ASSERT(graph_->GetMode().IsInterpreter() || retType != DataType::NO_TYPE);
2691 retType = retType == DataType::NO_TYPE ? DataType::VOID : retType;
2692 return llvm::FunctionType::get(GetType(retType), makeArrayRef(argTypes.data(), argTypes.size()), false);
2693 }
2694
ToSizeT(llvm::Value * value)2695 llvm::Value *LLVMIrConstructor::ToSizeT(llvm::Value *value)
2696 {
2697 auto entrypointSizeType = GetEntrypointSizeType();
2698 if (value->getType() == entrypointSizeType) {
2699 return value;
2700 }
2701 ASSERT(value->getType()->getIntegerBitWidth() < entrypointSizeType->getBitWidth());
2702 return builder_.CreateZExt(value, entrypointSizeType);
2703 }
2704
ToSSizeT(llvm::Value * value)2705 llvm::Value *LLVMIrConstructor::ToSSizeT(llvm::Value *value)
2706 {
2707 auto entrypointSizeType = GetEntrypointSizeType();
2708 if (value->getType() == entrypointSizeType) {
2709 return value;
2710 }
2711 ASSERT(value->getType()->getIntegerBitWidth() < entrypointSizeType->getBitWidth());
2712 return builder_.CreateSExt(value, entrypointSizeType);
2713 }
2714
GetArgumentsForCall(llvm::Value * callee,CallInst * call,bool skipFirst)2715 ArenaVector<llvm::Value *> LLVMIrConstructor::GetArgumentsForCall(llvm::Value *callee, CallInst *call, bool skipFirst)
2716 {
2717 ASSERT(callee->getType()->isPointerTy());
2718 ArenaVector<llvm::Value *> args(GetGraph()->GetLocalAllocator()->Adapter());
2719 args.push_back(callee);
2720
2721 // SaveState skipping - last arg
2722 for (size_t i = skipFirst ? 1 : 0; i < call->GetInputsCount() - 1; i++) {
2723 auto arg = GetInputValue(call, i);
2724 auto type = call->GetInputType(i);
2725 if (DataType::IsLessInt32(type)) {
2726 arg = CoerceValue(arg, type, DataType::INT32);
2727 }
2728 args.push_back(arg);
2729 }
2730
2731 return args;
2732 }
2733
GetIntrinsicArguments(llvm::FunctionType * intrinsicFunctionType,IntrinsicInst * inst)2734 ArenaVector<llvm::Value *> LLVMIrConstructor::GetIntrinsicArguments(llvm::FunctionType *intrinsicFunctionType,
2735 IntrinsicInst *inst)
2736 {
2737 ASSERT(intrinsicFunctionType != nullptr);
2738 ASSERT(inst != nullptr);
2739
2740 ArenaVector<llvm::Value *> args(GetGraph()->GetLocalAllocator()->Adapter());
2741
2742 if (inst->IsMethodFirstInput()) {
2743 args.push_back(GetMethodArgument());
2744 }
2745 if (inst->HasImms()) {
2746 for (uint64_t imm : inst->GetImms()) {
2747 size_t index = args.size();
2748 auto type = intrinsicFunctionType->getParamType(index);
2749 args.push_back(llvm::ConstantInt::get(type, imm));
2750 }
2751 }
2752 for (size_t i = 0; i < inst->GetInputsCount(); i++) {
2753 // Skip SaveState
2754 if (inst->GetInput(i).GetInst()->IsSaveState()) {
2755 continue;
2756 }
2757 args.push_back(GetInputValue(inst, i));
2758 }
2759 ASSERT(intrinsicFunctionType->getNumParams() == args.size());
2760 return args;
2761 }
2762
SetIntrinsicParamAttrs(llvm::CallInst * call,IntrinsicInst * inst,llvm::ArrayRef<llvm::Value * > args)2763 void LLVMIrConstructor::SetIntrinsicParamAttrs(llvm::CallInst *call, IntrinsicInst *inst,
2764 [[maybe_unused]] llvm::ArrayRef<llvm::Value *> args)
2765 {
2766 size_t i = inst->IsMethodFirstInput() ? 1U : 0;
2767 if (inst->HasImms()) {
2768 i += inst->GetImms().size();
2769 }
2770 #ifndef NDEBUG
2771 for (size_t j = 0; j < i; j++) {
2772 ASSERT(!args[j]->getType()->isIntegerTy() || args[j]->getType()->getIntegerBitWidth() > VECTOR_SIZE_16);
2773 }
2774 #endif
2775 for (size_t arkIndex = 0; arkIndex < inst->GetInputsCount(); arkIndex++) {
2776 // Skip SaveState
2777 if (inst->GetInput(arkIndex).GetInst()->IsSaveState()) {
2778 continue;
2779 }
2780 auto arkType = inst->GetInputType(arkIndex);
2781 switch (arkType) {
2782 case DataType::UINT8:
2783 ASSERT(args[i]->getType()->isIntegerTy() && args[i]->getType()->getIntegerBitWidth() == VECTOR_SIZE_8);
2784 call->addParamAttr(i, llvm::Attribute::ZExt);
2785 break;
2786 case DataType::UINT16:
2787 ASSERT(args[i]->getType()->isIntegerTy() && args[i]->getType()->getIntegerBitWidth() == VECTOR_SIZE_16);
2788 call->addParamAttr(i, llvm::Attribute::ZExt);
2789 break;
2790 case DataType::INT8:
2791 ASSERT(args[i]->getType()->isIntegerTy() && args[i]->getType()->getIntegerBitWidth() == VECTOR_SIZE_8);
2792 call->addParamAttr(i, llvm::Attribute::SExt);
2793 break;
2794 case DataType::INT16:
2795 ASSERT(args[i]->getType()->isIntegerTy() && args[i]->getType()->getIntegerBitWidth() == VECTOR_SIZE_16);
2796 call->addParamAttr(i, llvm::Attribute::SExt);
2797 break;
2798 case DataType::BOOL:
2799 break;
2800 default:
2801 ASSERT(!args[i]->getType()->isIntegerTy() || args[i]->getType()->getIntegerBitWidth() > VECTOR_SIZE_16);
2802 break;
2803 }
2804 i++;
2805 }
2806 ASSERT(i == args.size());
2807 }
2808
2809 template <typename T>
GetFunctionTypeForCall(T * inst)2810 llvm::FunctionType *LLVMIrConstructor::GetFunctionTypeForCall(T *inst)
2811 {
2812 ArenaVector<llvm::Type *> argTypes(GetGraph()->GetLocalAllocator()->Adapter());
2813
2814 if (GetGraph()->SupportManagedCode()) {
2815 // Callee
2816 argTypes.push_back(builder_.getPtrTy());
2817 }
2818
2819 auto runtime = GetGraph()->GetRuntime();
2820 auto methodPtr = GetGraph()->GetMethod();
2821 auto methodId = inst->GetCallMethodId();
2822 // For instance methods pass implicit object argument
2823 if (!runtime->IsMethodStatic(methodPtr, methodId)) {
2824 argTypes.push_back(builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
2825 }
2826
2827 for (size_t i = 0; i < runtime->GetMethodArgumentsCount(methodPtr, methodId); i++) {
2828 auto ptype = runtime->GetMethodArgumentType(methodPtr, methodId, i);
2829 argTypes.push_back(GetType(ptype));
2830 }
2831
2832 auto retType = runtime->GetMethodReturnType(methodPtr, methodId);
2833 // Ugly fix CallVirtual opcode for SaveState-excluded run codegen statistics
2834 if (methodPtr == nullptr) {
2835 retType = inst->GetType();
2836 }
2837
2838 if constexpr (std::is_same_v<T, CallInst>) {
2839 ASSERT(inst->IsInlined() || inst->GetType() == retType);
2840 }
2841
2842 return llvm::FunctionType::get(GetType(retType), argTypes, false);
2843 }
2844
GetThreadRegValue()2845 llvm::Value *LLVMIrConstructor::GetThreadRegValue()
2846 {
2847 if (GetGraph()->SupportManagedCode()) {
2848 return llvmbackend::runtime_calls::GetThreadRegValue(&builder_, arkInterface_);
2849 }
2850 auto regInput = std::find(cc_.begin(), cc_.end(), GetThreadReg(GetGraph()->GetArch()));
2851 ASSERT(regInput != cc_.end());
2852 auto threadRegValue = func_->arg_begin() + std::distance(cc_.begin(), regInput);
2853 return threadRegValue;
2854 }
2855
GetRealFrameRegValue()2856 llvm::Value *LLVMIrConstructor::GetRealFrameRegValue()
2857 {
2858 if (GetGraph()->SupportManagedCode()) {
2859 return llvmbackend::runtime_calls::GetRealFrameRegValue(&builder_, arkInterface_);
2860 }
2861 ASSERT(GetGraph()->GetMode().IsFastPath() || GetGraph()->GetArch() == Arch::X86_64);
2862 auto regInput = std::find(cc_.begin(), cc_.end(), GetRealFrameReg(GetGraph()->GetArch()));
2863 ASSERT(regInput != cc_.end());
2864 auto frameRegValue = func_->arg_begin() + std::distance(cc_.begin(), regInput);
2865 return frameRegValue;
2866 }
2867
GetOrCreateFunctionForCall(ark::compiler::CallInst * call,void * method)2868 llvm::Function *LLVMIrConstructor::GetOrCreateFunctionForCall(ark::compiler::CallInst *call, void *method)
2869 {
2870 ASSERT(method != nullptr);
2871 auto module = func_->getParent();
2872 auto methodName = arkInterface_->GetUniqMethodName(method);
2873 auto function = module->getFunction(methodName);
2874 if (function == nullptr) {
2875 auto functionProto = GetFunctionTypeForCall(call);
2876 function = CreateFunctionDeclaration(functionProto, methodName, module);
2877 function->addFnAttr("frame-pointer", "all");
2878 function->addFnAttr(
2879 ark::llvmbackend::LLVMArkInterface::SOURCE_LANG_ATTR,
2880 std::to_string(static_cast<uint8_t>(GetGraph()->GetRuntime()->GetMethodSourceLanguage(method))));
2881 }
2882 return function;
2883 }
2884
GetType(DataType::Type pandaType)2885 llvm::Type *LLVMIrConstructor::GetType(DataType::Type pandaType)
2886 {
2887 switch (pandaType) {
2888 case DataType::VOID:
2889 return builder_.getVoidTy();
2890 case DataType::POINTER:
2891 return builder_.getPtrTy();
2892 case DataType::REFERENCE:
2893 return builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE);
2894 case DataType::BOOL:
2895 case DataType::UINT8:
2896 case DataType::INT8:
2897 case DataType::UINT16:
2898 case DataType::INT16:
2899 case DataType::UINT32:
2900 case DataType::INT32:
2901 return builder_.getInt32Ty();
2902 case DataType::UINT64:
2903 case DataType::INT64:
2904 return builder_.getInt64Ty();
2905 case DataType::FLOAT32:
2906 return builder_.getFloatTy();
2907 case DataType::FLOAT64:
2908 return builder_.getDoubleTy();
2909 default:
2910 ASSERT_DO(false, (std::cerr << "No handler for panda type = '" << DataType::ToString(pandaType)
2911 << "' to llvm type conversion." << std::endl));
2912 UNREACHABLE();
2913 }
2914 }
2915
2916 /**
2917 * Return exact llvm::Type corresponding to the panda type.
2918 *
2919 * Use this method when exact panda type is indeed required.
2920 * It is the case for:
2921 * - array loads and stores. If 32-bit version were used, then the neighbour array elements would be overwritten or read
2922 * - object field loads and stores. The reason the same as in the case above.
2923 * - object static field loads and stores. The reason the same as in the cases above.
2924 * - comparisons. Sometimes boolean is compared with i32 or other integral type.
2925 * The exact type could be obtained from the compareInst->GetOperandsType(),
2926 * which should be used to coerce its operands
2927 * - Runtime calls. Some runtime call function declarations have narrower types than 32-bit version. To invoke them
2928 * the argument should be coerced to the exact type.
2929 */
GetExactType(DataType::Type targetType)2930 llvm::Type *LLVMIrConstructor::GetExactType(DataType::Type targetType)
2931 {
2932 switch (targetType) {
2933 case DataType::VOID:
2934 return builder_.getVoidTy();
2935 case DataType::POINTER:
2936 return builder_.getPtrTy();
2937 case DataType::REFERENCE:
2938 return builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE);
2939 case DataType::BOOL:
2940 case DataType::UINT8:
2941 case DataType::INT8:
2942 return builder_.getInt8Ty();
2943 case DataType::UINT16:
2944 case DataType::INT16:
2945 return builder_.getInt16Ty();
2946 case DataType::UINT32:
2947 case DataType::INT32:
2948 return builder_.getInt32Ty();
2949 case DataType::UINT64:
2950 case DataType::INT64:
2951 return builder_.getInt64Ty();
2952 case DataType::FLOAT32:
2953 return builder_.getFloatTy();
2954 case DataType::FLOAT64:
2955 return builder_.getDoubleTy();
2956 default:
2957 ASSERT_DO(false, (std::cerr << "No handler for panda type = '" << DataType::ToString(targetType)
2958 << "' to llvm type conversion." << std::endl));
2959 UNREACHABLE();
2960 }
2961 }
2962
GetCastOp(DataType::Type from,DataType::Type to)2963 llvm::Instruction::CastOps LLVMIrConstructor::GetCastOp(DataType::Type from, DataType::Type to)
2964 {
2965 Arch arch = GetGraph()->GetArch();
2966 if (IsInteger(from) && IsInteger(to) && DataType::GetTypeSize(from, arch) > DataType::GetTypeSize(to, arch)) {
2967 // narrowing, e.g. U32TOU8, I64TOI32
2968 return llvm::Instruction::Trunc;
2969 }
2970 if (IsSignedInteger(from) && IsInteger(to) && DataType::GetTypeSize(from, arch) < DataType::GetTypeSize(to, arch)) {
2971 // signed int widening, e.g. I32TOI64, I32TOU64
2972 return llvm::Instruction::SExt;
2973 }
2974 if (IsUnsignedInteger(from) && IsInteger(to) &&
2975 DataType::GetTypeSize(from, arch) < DataType::GetTypeSize(to, arch)) {
2976 // unsigned int widening, e.g. U32TOI64, U8TOU64
2977 return llvm::Instruction::ZExt;
2978 }
2979 if (IsUnsignedInteger(from) && DataType::IsFloatType(to)) {
2980 // unsigned int to float, e.g. U32TOF64, U64TOF64
2981 return llvm::Instruction::UIToFP;
2982 }
2983 if (IsSignedInteger(from) && DataType::IsFloatType(to)) {
2984 // signed int to float e.g. I32TOF64, I64TOF64
2985 return llvm::Instruction::SIToFP;
2986 }
2987 if (DataType::IsFloatType(from) && DataType::IsFloatType(to)) {
2988 if (DataType::GetTypeSize(from, arch) < DataType::GetTypeSize(to, arch)) {
2989 return llvm::Instruction::FPExt;
2990 }
2991 return llvm::Instruction::FPTrunc;
2992 }
2993 if (DataType::IsReference(from) && to == DataType::POINTER) {
2994 return llvm::Instruction::AddrSpaceCast;
2995 }
2996 ASSERT_DO(false, (std::cerr << "Cast from " << DataType::ToString(from) << " to " << DataType::ToString(to))
2997 << " is not supported" << std::endl);
2998 UNREACHABLE();
2999 }
3000
3001 // Various other helpers
3002
3003 /**
3004 * Coerce given {@code value} with {@code sourceType} to the {@code targetType}.
3005 *
3006 * The method may perform truncation or widening cast, or leave the original
3007 * {@code value}, if no cast is necessary.
3008 *
3009 * For integer {@code value} when widening cast is performed the sign of the {@code sourceType} is taken
3010 * into account:
3011 * * {@code value} is zero extended if the {@code sourceType} is unsigned integer
3012 * * {@code value} is sign extended if the {@code sourceType} is signed integer
3013 *
3014 * Reference types are returned as is.
3015 *
3016 * Currently Ark Bytecode:
3017 * * does not differentiate between ints of sizes less than 32 bits, and treats them all as i32/u32
3018 * * leaves resolution of such conversions to the discretion of bytecodes accepting them
3019 * * assumes implicit casts between small integers
3020 *
3021 * Sometimes it causes inconsistencies in LLVM since Ark Compiler IR input has implicit casts too,
3022 * but LLVM does not permit such conversions. This function perform those casts if necessary.
3023 */
CoerceValue(llvm::Value * value,DataType::Type sourceType,DataType::Type targetType)3024 llvm::Value *LLVMIrConstructor::CoerceValue(llvm::Value *value, DataType::Type sourceType, DataType::Type targetType)
3025 {
3026 ASSERT(value != nullptr);
3027 // Other non-integer mistyping prohibited
3028 ASSERT_DO(!IsInteger(targetType) || value->getType()->isIntegerTy(),
3029 std::cerr << "Unexpected data type: " << GetTypeName(value->getType()) << ". Should be an integer."
3030 << std::endl);
3031 ASSERT_DO(!DataType::IsReference(targetType) || value->getType()->isPointerTy(),
3032 std::cerr << "Unexpected data type: " << GetTypeName(value->getType()) << ". Should be a pointer."
3033 << std::endl);
3034 ASSERT_DO(targetType != DataType::FLOAT64 || value->getType()->isDoubleTy(),
3035 std::cerr << "Unexpected data type: " << GetTypeName(value->getType()) << ". Should be a double."
3036 << std::endl);
3037 ASSERT_DO(targetType != DataType::FLOAT32 || value->getType()->isFloatTy(),
3038 std::cerr << "Unexpected data type: " << GetTypeName(value->getType()) << ". Should be a float."
3039 << std::endl);
3040
3041 if (!IsInteger(targetType)) {
3042 return value;
3043 }
3044 ASSERT(value->getType()->isIntegerTy());
3045
3046 auto targetLlvmType = llvm::cast<llvm::IntegerType>(GetExactType(targetType));
3047 auto originalLlvmType = llvm::cast<llvm::IntegerType>(value->getType());
3048 ASSERT(originalLlvmType->getBitWidth() == DataType::GetTypeSize(sourceType, GetGraph()->GetArch()));
3049
3050 llvm::CastInst::CastOps castOp;
3051 if (originalLlvmType->getBitWidth() > targetLlvmType->getBitWidth()) {
3052 castOp = llvm::Instruction::Trunc;
3053 } else if (originalLlvmType->getBitWidth() < targetLlvmType->getBitWidth()) {
3054 if (IsSignedInteger(sourceType)) {
3055 castOp = llvm::Instruction::SExt;
3056 } else {
3057 castOp = llvm::Instruction::ZExt;
3058 }
3059 } else {
3060 return value;
3061 }
3062 return builder_.CreateCast(castOp, value, targetLlvmType);
3063 }
3064
3065 /**
3066 * Used in irtoc C++ inlining.
3067 *
3068 * When we compile irtoc handlers, we do not have ark's types.
3069 * For example, ark::Frame is missing.
3070 * LLVM AOT uses i8* or i64 instead
3071 *
3072 * For example, the irtoc handler could look like:
3073 *
3074 * @code
3075 * void MyHandler(i8* myMbject) {
3076 * var clone = CloneObjectEntrypoint(myObject);
3077 * }
3078 * @endcode
3079 *
3080 * When we compile interpreter handlers with cpp inlining we have the definition of CloneObjectEntrypoint:
3081 *
3082 * @code
3083 * ObjectHeader *CloneObjectEntrypoint(ObjectHeader *obj) {
3084 * ...
3085 * }
3086 * @endcode
3087 *
3088 * and we must invoke the CloneObjectEntrypoint with ObjectHeader* argument, not i8*.
3089 * The CoerceValue method converts i8* to ObjectHeader*
3090 */
CoerceValue(llvm::Value * value,llvm::Type * targetType)3091 llvm::Value *LLVMIrConstructor::CoerceValue(llvm::Value *value, llvm::Type *targetType)
3092 {
3093 auto valueType = value->getType();
3094 if (valueType == targetType) {
3095 return value;
3096 }
3097
3098 if (!valueType->isPointerTy() && targetType->isPointerTy()) {
3099 // DataType::POINTER to targetType.
3100 // Example: i64 -> %"class.ark::Frame"*
3101 return builder_.CreateIntToPtr(value, targetType);
3102 }
3103 if (valueType->isPointerTy() && !targetType->isPointerTy()) {
3104 // valueType to DataType::POINTER
3105 // Example: %"class.ark::coretypes::String"* -> i64
3106 return builder_.CreatePtrToInt(value, targetType);
3107 }
3108
3109 if (valueType->isIntegerTy() && targetType->isIntegerTy()) {
3110 auto valueWidth = llvm::cast<llvm::IntegerType>(valueType)->getBitWidth();
3111 auto targetWidth = llvm::cast<llvm::IntegerType>(targetType)->getBitWidth();
3112 if (valueWidth > targetWidth) {
3113 return builder_.CreateTrunc(value, targetType);
3114 }
3115 if (valueWidth < targetWidth) {
3116 return builder_.CreateZExt(value, targetType);
3117 }
3118 }
3119 if (valueType->isPointerTy() && targetType->isPointerTy()) {
3120 return builder_.CreateAddrSpaceCast(value, targetType);
3121 }
3122 UNREACHABLE();
3123 }
3124
ValueMapAdd(Inst * inst,llvm::Value * value,bool setName)3125 void LLVMIrConstructor::ValueMapAdd(Inst *inst, llvm::Value *value, bool setName)
3126 {
3127 if (!inst->IsMovableObject() && !inst->IsCheck() && llvmbackend::gc_utils::IsGcRefType(value->getType())) {
3128 auto llvmInst = llvm::dyn_cast<llvm::Instruction>(value);
3129 if (llvmInst != nullptr) {
3130 llvmbackend::gc_utils::MarkAsNonMovable(llvmInst);
3131 }
3132 }
3133
3134 auto type = inst->GetType();
3135 auto ltype = GetExactType(type);
3136 ASSERT(inputMap_.count(inst) == 0);
3137 auto it = inputMap_.emplace(inst, GetGraph()->GetLocalAllocator()->Adapter());
3138 ASSERT(it.second);
3139 ArenaUnorderedMap<DataType::Type, llvm::Value *> &typeMap = it.first->second;
3140
3141 if (value == nullptr) {
3142 typeMap.insert({type, nullptr});
3143 return;
3144 }
3145 if (setName) {
3146 value->setName(CreateNameForInst(inst));
3147 }
3148 if (inst->GetOpcode() == Opcode::LiveOut || !ltype->isIntegerTy()) {
3149 typeMap.insert({type, value});
3150 if (type == DataType::POINTER) {
3151 FillValueMapForUsers(&typeMap, inst, value);
3152 }
3153 return;
3154 }
3155 ASSERT(value->getType()->isIntegerTy());
3156 if (value->getType()->getIntegerBitWidth() > ltype->getIntegerBitWidth()) {
3157 value = builder_.CreateTrunc(value, ltype);
3158 } else if (value->getType()->getIntegerBitWidth() < ltype->getIntegerBitWidth()) {
3159 value = builder_.CreateZExt(value, ltype);
3160 }
3161 typeMap.insert({type, value});
3162 FillValueMapForUsers(&typeMap, inst, value);
3163 }
3164
FillValueMapForUsers(ArenaUnorderedMap<DataType::Type,llvm::Value * > * map,Inst * inst,llvm::Value * value)3165 void LLVMIrConstructor::FillValueMapForUsers(ArenaUnorderedMap<DataType::Type, llvm::Value *> *map, Inst *inst,
3166 llvm::Value *value)
3167 {
3168 auto type = inst->GetType();
3169 ASSERT(type != DataType::REFERENCE);
3170 for (auto &userItem : inst->GetUsers()) {
3171 auto user = userItem.GetInst();
3172 for (unsigned i = 0; i < user->GetInputsCount(); i++) {
3173 auto itype = user->GetInputType(i);
3174 auto input = user->GetInput(i).GetInst();
3175 if (input != inst || itype == type || map->count(itype) != 0) {
3176 continue;
3177 }
3178 /*
3179 * When Ark Compiler implicitly converts something -> LLVM side:
3180 * 1. POINTER to REFERENCE (user LiveOut or Store) -> AddrSpaceCast
3181 * 2. POINTER to UINT64 (user is LiveOut) -> no conversion necessary
3182 * 3. LiveIn to REFERENCE -> no conversion necessary
3183 * 4. INT64/UINT64 to REFERENCE (user is LiveOut) -> IntToPtr
3184 * 5. Integers -> use coercing
3185 */
3186 llvm::Value *cvalue;
3187 if (type == DataType::POINTER && itype == DataType::REFERENCE) {
3188 ASSERT(user->GetOpcode() == Opcode::LiveOut || user->GetOpcode() == Opcode::Store);
3189 cvalue = builder_.CreateAddrSpaceCast(value, builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
3190 } else if (type == DataType::POINTER && itype == DataType::UINT64) {
3191 ASSERT(user->GetOpcode() == Opcode::LiveOut);
3192 cvalue = value;
3193 } else if (type == DataType::POINTER) {
3194 continue;
3195 } else if (inst->GetOpcode() == Opcode::LiveIn && itype == DataType::REFERENCE) {
3196 cvalue = value;
3197 } else if ((type == DataType::INT64 || type == DataType::UINT64) && itype == DataType::REFERENCE) {
3198 ASSERT(user->GetOpcode() == Opcode::LiveOut);
3199 cvalue = builder_.CreateIntToPtr(value, builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
3200 } else {
3201 cvalue = CoerceValue(value, type, itype);
3202 }
3203 map->insert({itype, cvalue});
3204 }
3205 }
3206 }
3207
WrapArkCall(Inst * orig,llvm::CallInst * call)3208 void LLVMIrConstructor::WrapArkCall(Inst *orig, llvm::CallInst *call)
3209 {
3210 ASSERT(orig->RequireState());
3211 ASSERT_PRINT(!call->getDebugLoc(), "Debug info must be unset");
3212 // Ark calls may call GC inside, so add statepoint
3213 debugData_->SetLocation(call, orig->GetPc());
3214 EncodeInlineInfo(orig, call);
3215 }
3216
InitializeEntryBlock(bool noInline)3217 void LLVMIrConstructor::InitializeEntryBlock(bool noInline)
3218 {
3219 if (noInline) {
3220 ASSERT(!arkInterface_->IsIrtocMode() && GetGraph()->SupportManagedCode());
3221 func_->addFnAttr(llvm::Attribute::NoInline);
3222 // This type of linkage prevents return value propagation.
3223 // llvm::GlobalValue::isDefinitionExact becomes false and as a result
3224 // llvm::canTrackReturnsInterprocedurally() also false.
3225 func_->setLinkage(llvm::Function::WeakAnyLinkage);
3226 }
3227
3228 if (GetGraph()->SupportManagedCode()) {
3229 func_->addParamAttr(GetMethodArgument()->getArgNo(), llvm::Attribute::NonNull);
3230 if (!GetGraph()->GetRuntime()->IsMethodStatic(GetGraph()->GetMethod())) {
3231 func_->addParamAttr(GetArgument(0)->getArgNo(), llvm::Attribute::NonNull);
3232 }
3233 }
3234
3235 if (func_->hasMetadata(LLVMArkInterface::FUNCTION_MD_INLINE_MODULE) &&
3236 !GetGraph()->GetRuntime()->IsMethodStatic(GetGraph()->GetMethod())) {
3237 SetCurrentBasicBlock(&func_->getEntryBlock());
3238 builder_.CreateCall(KeepThis(func_->getParent()), GetArgument(0));
3239 }
3240 }
3241
MarkAsAllocation(llvm::CallInst * call)3242 void LLVMIrConstructor::MarkAsAllocation(llvm::CallInst *call)
3243 {
3244 llvm::AttrBuilder builder {call->getContext()};
3245 /**
3246 * When we add allockind(alloc) attribute, then llvm can assume that the function is allocation function.
3247 * With this assumption llvm can remove dead allocations
3248 */
3249 builder.addAllocKindAttr(llvm::AllocFnKind::Alloc);
3250 call->addFnAttr(builder.getAttribute(llvm::Attribute::AllocKind));
3251 call->addRetAttr(llvm::Attribute::NonNull);
3252 call->addRetAttr(llvm::Attribute::NoAlias);
3253 }
3254
3255 // Instruction Visitors
3256
3257 // Constant and NullPtr are processed directly in GetInputValue
VisitConstant(GraphVisitor * v,Inst * inst)3258 void LLVMIrConstructor::VisitConstant([[maybe_unused]] GraphVisitor *v, [[maybe_unused]] Inst *inst)
3259 {
3260 ASSERT(inst->GetBasicBlock()->IsStartBlock());
3261 }
3262
VisitNullPtr(GraphVisitor * v,Inst * inst)3263 void LLVMIrConstructor::VisitNullPtr([[maybe_unused]] GraphVisitor *v, [[maybe_unused]] Inst *inst)
3264 {
3265 ASSERT(inst->GetBasicBlock()->IsStartBlock());
3266 }
3267
VisitLiveIn(GraphVisitor * v,Inst * inst)3268 void LLVMIrConstructor::VisitLiveIn(GraphVisitor *v, Inst *inst)
3269 {
3270 auto ctor = static_cast<LLVMIrConstructor *>(v);
3271 ASSERT(inst->GetBasicBlock()->IsStartBlock());
3272 ASSERT(!ctor->GetGraph()->SupportManagedCode());
3273
3274 auto regInput = std::find(ctor->cc_.begin(), ctor->cc_.end(), inst->CastToLiveIn()->GetDstReg());
3275 ASSERT(regInput != ctor->cc_.end());
3276 auto idx = std::distance(ctor->cc_.begin(), regInput);
3277 auto n = ctor->func_->arg_begin() + idx;
3278 ctor->ValueMapAdd(inst, ctor->CoerceValue(n, ctor->GetExactType(inst->GetType())));
3279 }
3280
VisitParameter(GraphVisitor * v,Inst * inst)3281 void LLVMIrConstructor::VisitParameter(GraphVisitor *v, Inst *inst)
3282 {
3283 ASSERT(inst->GetBasicBlock()->IsStartBlock());
3284 auto ctor = static_cast<LLVMIrConstructor *>(v);
3285 ASSERT(ctor->GetGraph()->SupportManagedCode() || ctor->GetGraph()->GetMode().IsFastPath());
3286 auto n = ctor->GetArgument(inst->CastToParameter()->GetArgNumber());
3287 ctor->ValueMapAdd(inst, n, false);
3288 }
3289
VisitReturnVoid(GraphVisitor * v,Inst * inst)3290 void LLVMIrConstructor::VisitReturnVoid(GraphVisitor *v, Inst *inst)
3291 {
3292 auto ctor = static_cast<LLVMIrConstructor *>(v);
3293 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
3294 auto builtin = BarrierReturnVoid(ctor->func_->getParent());
3295 auto builtinCall = ctor->builder_.CreateCall(builtin);
3296 builtinCall->addFnAttr(llvm::Attribute::get(builtinCall->getContext(), "needs-mem-barrier"));
3297 }
3298 ctor->builder_.CreateRetVoid();
3299 }
3300
VisitReturn(GraphVisitor * v,Inst * inst)3301 void LLVMIrConstructor::VisitReturn(GraphVisitor *v, Inst *inst)
3302 {
3303 auto ctor = static_cast<LLVMIrConstructor *>(v);
3304 auto ret = ctor->GetInputValue(inst, 0);
3305
3306 auto type = inst->GetType();
3307 if (DataType::IsLessInt32(type)) {
3308 ret = ctor->CoerceValue(ret, type, DataType::INT32);
3309 }
3310
3311 ctor->builder_.CreateRet(ret);
3312 }
3313
VisitReturnInlined(GraphVisitor * v,Inst * inst)3314 void LLVMIrConstructor::VisitReturnInlined(GraphVisitor *v, Inst *inst)
3315 {
3316 auto ctor = static_cast<LLVMIrConstructor *>(v);
3317
3318 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
3319 auto builtin = BarrierReturnVoid(ctor->func_->getParent());
3320 auto builtinCall = ctor->builder_.CreateCall(builtin);
3321 builtinCall->addFnAttr(llvm::Attribute::get(builtinCall->getContext(), "needs-mem-barrier"));
3322 }
3323 }
3324
VisitReturnI(GraphVisitor * v,Inst * inst)3325 void LLVMIrConstructor::VisitReturnI(GraphVisitor *v, Inst *inst)
3326 {
3327 auto ctor = static_cast<LLVMIrConstructor *>(v);
3328 llvm::Value *ret = ctor->builder_.getInt64(inst->CastToReturnI()->GetImm());
3329
3330 auto type = inst->GetType();
3331 if (DataType::IsInt32Bit(type)) {
3332 ret = ctor->CoerceValue(ret, DataType::INT64, DataType::INT32);
3333 }
3334
3335 ctor->builder_.CreateRet(ret);
3336 }
3337
3338 // No-op "pseudo" instructions
VisitTry(GraphVisitor * v,Inst * inst)3339 void LLVMIrConstructor::VisitTry([[maybe_unused]] GraphVisitor *v, [[maybe_unused]] Inst *inst) {}
VisitSaveState(GraphVisitor * v,Inst * inst)3340 void LLVMIrConstructor::VisitSaveState([[maybe_unused]] GraphVisitor *v, [[maybe_unused]] Inst *inst) {}
VisitSaveStateDeoptimize(GraphVisitor * v,Inst * inst)3341 void LLVMIrConstructor::VisitSaveStateDeoptimize([[maybe_unused]] GraphVisitor *v, [[maybe_unused]] Inst *inst) {}
VisitSafePoint(GraphVisitor * v,Inst * inst)3342 void LLVMIrConstructor::VisitSafePoint([[maybe_unused]] GraphVisitor *v, [[maybe_unused]] Inst *inst) {}
3343 // NOP and Deoptimize* required after adding CheckElim* passes
VisitNOP(GraphVisitor * v,Inst * inst)3344 void LLVMIrConstructor::VisitNOP([[maybe_unused]] GraphVisitor *v, [[maybe_unused]] Inst *inst) {}
3345
VisitLiveOut(GraphVisitor * v,Inst * inst)3346 void LLVMIrConstructor::VisitLiveOut(GraphVisitor *v, Inst *inst)
3347 {
3348 auto ctor = static_cast<LLVMIrConstructor *>(v);
3349 ASSERT(!ctor->GetGraph()->SupportManagedCode());
3350 auto input = ctor->GetInputValue(inst, 0);
3351
3352 auto regInput = std::find(ctor->cc_.begin(), ctor->cc_.end(), inst->GetDstReg());
3353 ASSERT(regInput != ctor->cc_.end());
3354 size_t idx = std::distance(ctor->cc_.begin(), regInput);
3355 ASSERT(ctor->ccValues_[idx] == nullptr);
3356
3357 // LiveOut not allowed for real frame register
3358 ASSERT(ctor->GetGraph()->GetArch() == Arch::AARCH64 || idx + 1 != ctor->cc_.size());
3359 auto value = ctor->CoerceValue(input, ctor->GetExactType(inst->GetType()));
3360 ctor->ccValues_[idx] = value;
3361 ctor->ValueMapAdd(inst, value, false);
3362 }
3363
VisitSubOverflowCheck(GraphVisitor * v,Inst * inst)3364 void LLVMIrConstructor::VisitSubOverflowCheck(GraphVisitor *v, Inst *inst)
3365 {
3366 auto ctor = static_cast<LLVMIrConstructor *>(v);
3367 auto dtype = inst->GetType();
3368 auto ltype = ctor->GetExactType(dtype);
3369 auto src0 = ctor->GetInputValue(inst, 0);
3370 auto src1 = ctor->GetInputValue(inst, 1);
3371 ASSERT(inst->GetInputType(0) == inst->GetInputType(1));
3372
3373 auto arch = ctor->GetGraph()->GetArch();
3374 auto dtypeSize = DataType::GetTypeSize(dtype, arch);
3375 auto srcTypeSize = DataType::GetTypeSize(inst->GetInputType(0), arch);
3376 ASSERT(DataType::Is32Bits(dtype, arch) || DataType::Is64Bits(dtype, arch));
3377 if (srcTypeSize < dtypeSize) {
3378 src0 = ctor->builder_.CreateSExt(src0, ltype);
3379 src1 = ctor->builder_.CreateSExt(src1, ltype);
3380 }
3381 if (dtypeSize < srcTypeSize) {
3382 src0 = ctor->builder_.CreateTrunc(src0, ltype);
3383 src1 = ctor->builder_.CreateTrunc(src1, ltype);
3384 }
3385
3386 auto ssubOverflow = ctor->builder_.CreateBinaryIntrinsic(llvm::Intrinsic::ssub_with_overflow, src0, src1);
3387 auto result = ctor->builder_.CreateExtractValue(ssubOverflow, {0}, "ssub");
3388 auto deoptimize = ctor->builder_.CreateExtractValue(ssubOverflow, {1}, "obit");
3389
3390 auto exception = RuntimeInterface::EntrypointId::DEOPTIMIZE;
3391 ctor->CreateDeoptimizationBranch(inst, deoptimize, exception);
3392
3393 ctor->ValueMapAdd(inst, result, false);
3394 }
3395
VisitDeoptimize(GraphVisitor * v,Inst * inst)3396 void LLVMIrConstructor::VisitDeoptimize(GraphVisitor *v, Inst *inst)
3397 {
3398 auto ctor = static_cast<LLVMIrConstructor *>(v);
3399 auto type = inst->CastToDeoptimize()->GetDeoptimizeType();
3400 auto exception = RuntimeInterface::EntrypointId::DEOPTIMIZE;
3401 uint64_t value = static_cast<uint64_t>(type) | (inst->GetId() << MinimumBitsToStore(DeoptimizeType::COUNT));
3402 auto call = ctor->CreateEntrypointCall(exception, inst, {ctor->builder_.getInt64(value)});
3403 call->addFnAttr(llvm::Attribute::get(call->getContext(), "may-deoptimize"));
3404 ctor->builder_.CreateUnreachable();
3405 }
3406
VisitDeoptimizeIf(GraphVisitor * v,Inst * inst)3407 void LLVMIrConstructor::VisitDeoptimizeIf(GraphVisitor *v, Inst *inst)
3408 {
3409 auto ctor = static_cast<LLVMIrConstructor *>(v);
3410 auto exception = RuntimeInterface::EntrypointId::DEOPTIMIZE;
3411 auto deoptimize = ctor->builder_.CreateIsNotNull(ctor->GetInputValue(inst, 0));
3412 ctor->CreateDeoptimizationBranch(inst, deoptimize, exception);
3413 }
3414
VisitNegativeCheck(GraphVisitor * v,Inst * inst)3415 void LLVMIrConstructor::VisitNegativeCheck(GraphVisitor *v, Inst *inst)
3416 {
3417 auto ctor = static_cast<LLVMIrConstructor *>(v);
3418 auto val = ctor->GetInputValue(inst, 0);
3419
3420 auto deoptimize = ctor->builder_.CreateICmpSLT(val, llvm::Constant::getNullValue(val->getType()));
3421 auto exception = RuntimeInterface::EntrypointId::NEGATIVE_ARRAY_SIZE_EXCEPTION;
3422 ctor->CreateDeoptimizationBranch(inst, deoptimize, exception, {ctor->ToSSizeT(val)});
3423
3424 ctor->ValueMapAdd(inst, val, false);
3425 }
3426
VisitZeroCheck(GraphVisitor * v,Inst * inst)3427 void LLVMIrConstructor::VisitZeroCheck(GraphVisitor *v, Inst *inst)
3428 {
3429 auto ctor = static_cast<LLVMIrConstructor *>(v);
3430 auto val = ctor->GetInputValue(inst, 0);
3431
3432 auto deoptimize = ctor->builder_.CreateIsNull(val);
3433 auto exception = RuntimeInterface::EntrypointId::ARITHMETIC_EXCEPTION;
3434 ctor->CreateDeoptimizationBranch(inst, deoptimize, exception);
3435
3436 ctor->ValueMapAdd(inst, val, false);
3437 }
3438
VisitNullCheck(GraphVisitor * v,Inst * inst)3439 void LLVMIrConstructor::VisitNullCheck(GraphVisitor *v, Inst *inst)
3440 {
3441 auto ctor = static_cast<LLVMIrConstructor *>(v);
3442 auto obj = ctor->GetInputValue(inst, 0);
3443 auto obj64 = obj;
3444
3445 if (compiler::g_options.IsCompilerImplicitNullCheck()) {
3446 // LLVM's ImplicitNullChecks pass can't operate with 32-bit pointers, but it is enough
3447 // to create address space cast to an usual 64-bit pointer before comparing with null.
3448 obj64 = ctor->builder_.CreateAddrSpaceCast(obj, ctor->builder_.getPtrTy());
3449 }
3450
3451 auto deoptimize = ctor->builder_.CreateIsNull(obj64);
3452 auto exception = RuntimeInterface::EntrypointId::NULL_POINTER_EXCEPTION;
3453 ctor->CreateDeoptimizationBranch(inst, deoptimize, exception);
3454
3455 ctor->ValueMapAdd(inst, obj, false);
3456 }
3457
VisitBoundsCheck(GraphVisitor * v,Inst * inst)3458 void LLVMIrConstructor::VisitBoundsCheck(GraphVisitor *v, Inst *inst)
3459 {
3460 auto ctor = static_cast<LLVMIrConstructor *>(v);
3461 auto length = ctor->GetInputValue(inst, 0);
3462 ASSERT_TYPE(length, ctor->builder_.getInt32Ty());
3463 auto index = ctor->GetInputValue(inst, 1);
3464 ASSERT(index->getType()->isIntegerTy());
3465
3466 auto deoptimize = ctor->builder_.CreateICmpUGE(index, length);
3467 auto exception = inst->CastToBoundsCheck()->IsArray()
3468 ? RuntimeInterface::EntrypointId::ARRAY_INDEX_OUT_OF_BOUNDS_EXCEPTION
3469 : RuntimeInterface::EntrypointId::STRING_INDEX_OUT_OF_BOUNDS_EXCEPTION;
3470 ctor->CreateDeoptimizationBranch(inst, deoptimize, exception, {ctor->ToSSizeT(index), ctor->ToSizeT(length)});
3471
3472 ctor->ValueMapAdd(inst, index, false);
3473 }
3474
VisitRefTypeCheck(GraphVisitor * v,Inst * inst)3475 void LLVMIrConstructor::VisitRefTypeCheck(GraphVisitor *v, Inst *inst)
3476 {
3477 auto ctor = static_cast<LLVMIrConstructor *>(v);
3478
3479 auto array = ctor->GetInputValue(inst, 0);
3480 auto ref = ctor->GetInputValue(inst, 1);
3481
3482 auto &ctx = ctor->func_->getContext();
3483 auto compareBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "comparison"), ctor->func_);
3484 auto compBaseBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "comp_base"), ctor->func_);
3485 auto slowPathBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "slow_path"), ctor->func_);
3486 auto outBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "out"), ctor->func_);
3487
3488 auto runtime = ctor->GetGraph()->GetRuntime();
3489 auto arch = ctor->GetGraph()->GetArch();
3490
3491 auto cmp = ctor->builder_.CreateIsNotNull(ref);
3492 ctor->builder_.CreateCondBr(cmp, compareBb, outBb);
3493
3494 // Get element class from array
3495 ctor->SetCurrentBasicBlock(compareBb);
3496 auto arrayClass = CreateLoadClassFromObject(array, &ctor->builder_, ctor->arkInterface_);
3497 auto elementTypeOffset = runtime->GetClassComponentTypeOffset(arch);
3498 auto int8Ty = ctor->builder_.getInt8Ty();
3499 auto elementClassPtr = ctor->builder_.CreateConstInBoundsGEP1_32(int8Ty, arrayClass, elementTypeOffset);
3500 auto elementClass = ctor->builder_.CreateLoad(ctor->builder_.getPtrTy(), elementClassPtr);
3501 // And class from stored object
3502 auto refClass = CreateLoadClassFromObject(ref, &ctor->builder_, ctor->arkInterface_);
3503
3504 // Unlike other checks, there's another check in the runtime function, so don't use CreateDeoptimizationBranch
3505 cmp = ctor->builder_.CreateICmpNE(elementClass, refClass);
3506 auto branchWeights =
3507 llvm::MDBuilder(ctx).createBranchWeights(llvmbackend::Metadata::BranchWeights::LIKELY_BRANCH_WEIGHT,
3508 llvmbackend::Metadata::BranchWeights::UNLIKELY_BRANCH_WEIGHT);
3509 ctor->builder_.CreateCondBr(cmp, compBaseBb, outBb, branchWeights);
3510
3511 // If the array's element class is Object (Base class is null) - no further check needed
3512 ctor->SetCurrentBasicBlock(compBaseBb);
3513 auto baseTypeOffset = runtime->GetClassBaseOffset(arch);
3514 auto baseClassPtr = ctor->builder_.CreateConstInBoundsGEP1_32(int8Ty, elementClass, baseTypeOffset);
3515 auto baseClass = ctor->builder_.CreateLoad(ctor->builder_.getPtrTy(), baseClassPtr);
3516 auto notObjectArray = ctor->builder_.CreateIsNotNull(baseClass);
3517 ctor->builder_.CreateCondBr(notObjectArray, slowPathBb, outBb);
3518
3519 ctor->SetCurrentBasicBlock(slowPathBb);
3520 if (inst->CanDeoptimize()) {
3521 auto entrypoint = RuntimeInterface::EntrypointId::CHECK_STORE_ARRAY_REFERENCE_DEOPTIMIZE;
3522 auto call = ctor->CreateEntrypointCall(entrypoint, inst, {array, ref});
3523 call->addFnAttr(llvm::Attribute::get(call->getContext(), "may-deoptimize"));
3524 } else {
3525 ctor->CreateEntrypointCall(RuntimeInterface::EntrypointId::CHECK_STORE_ARRAY_REFERENCE, inst, {array, ref});
3526 }
3527 ctor->builder_.CreateBr(outBb);
3528
3529 ctor->SetCurrentBasicBlock(outBb);
3530 ctor->ValueMapAdd(inst, ref, false);
3531 }
3532
VisitLoadString(GraphVisitor * v,Inst * inst)3533 void LLVMIrConstructor::VisitLoadString(GraphVisitor *v, Inst *inst)
3534 {
3535 auto ctor = static_cast<LLVMIrConstructor *>(v);
3536
3537 llvm::Value *result;
3538 if (g_options.IsCompilerAotLoadStringPlt() &&
3539 !ctor->GetGraph()->GetRuntime()->IsMethodStaticConstructor(ctor->GetGraph()->GetMethod())) {
3540 auto aotData = ctor->GetGraph()->GetAotData();
3541 ASSERT(aotData != nullptr);
3542
3543 auto typeId = inst->CastToLoadString()->GetTypeId();
3544 auto typeVal = ctor->builder_.getInt32(typeId);
3545 auto slotVal = ctor->builder_.getInt32(ctor->arkInterface_->GetStringSlotId(aotData, typeId));
3546 ctor->arkInterface_->GetOrCreateRuntimeFunctionType(
3547 ctor->func_->getContext(), ctor->func_->getParent(), LLVMArkInterface::RuntimeCallType::ENTRYPOINT,
3548 static_cast<LLVMArkInterface::EntrypointId>(RuntimeInterface::EntrypointId::RESOLVE_STRING_AOT));
3549
3550 auto builtin = LoadString(ctor->func_->getParent());
3551 auto call = ctor->builder_.CreateCall(builtin, {typeVal, slotVal}, ctor->CreateSaveStateBundle(inst));
3552 ctor->WrapArkCall(inst, call);
3553 result = call;
3554 } else {
3555 auto stringType = ctor->builder_.getInt32(inst->CastToLoadString()->GetTypeId());
3556 auto entrypointId = RuntimeInterface::EntrypointId::RESOLVE_STRING;
3557 result = ctor->CreateEntrypointCall(entrypointId, inst, {ctor->GetMethodArgument(), stringType});
3558 }
3559 ctor->ValueMapAdd(inst, result);
3560 }
3561
VisitLenArray(GraphVisitor * v,Inst * inst)3562 void LLVMIrConstructor::VisitLenArray(GraphVisitor *v, Inst *inst)
3563 {
3564 auto ctor = static_cast<LLVMIrConstructor *>(v);
3565 auto array = ctor->GetInputValue(inst, 0);
3566 auto runtime = ctor->GetGraph()->GetRuntime();
3567 bool isString = !inst->CastToLenArray()->IsArray();
3568 auto &builder = ctor->builder_;
3569
3570 auto arrayInput = inst->GetDataFlowInput(0);
3571 // Try to extract array length from constructor
3572 if (arrayInput->GetOpcode() == Opcode::NewArray) {
3573 auto size = ctor->GetInputValue(arrayInput, NewArrayInst::INDEX_SIZE);
3574 ctor->ValueMapAdd(inst, size);
3575 return;
3576 }
3577 auto builtin = LenArray(ctor->func_->getParent());
3578 auto arch = ctor->GetGraph()->GetArch();
3579 auto offset = isString ? runtime->GetStringLengthOffset(arch) : runtime->GetArrayLengthOffset(arch);
3580 auto len = ctor->builder_.CreateCall(builtin, {array, builder.getInt32(offset)});
3581
3582 ctor->ValueMapAdd(inst, len);
3583 }
3584
VisitLoadArray(GraphVisitor * v,Inst * inst)3585 void LLVMIrConstructor::VisitLoadArray(GraphVisitor *v, Inst *inst)
3586 {
3587 auto ctor = static_cast<LLVMIrConstructor *>(v);
3588 auto loadArray = inst->CastToLoadArray();
3589
3590 auto array = ctor->GetInputValue(inst, 0);
3591 ASSERT_TYPE(array, ctor->builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
3592
3593 auto dtype = inst->GetType();
3594 auto ltype = ctor->GetExactType(dtype);
3595 auto arch = ctor->GetGraph()->GetArch();
3596 uint32_t dataOffset = ctor->GetGraph()->GetRuntime()->GetArrayDataOffset(arch);
3597 if (!loadArray->IsArray()) {
3598 dataOffset = ctor->GetGraph()->GetRuntime()->GetStringDataOffset(arch);
3599 }
3600 auto ptrData = ctor->builder_.CreateConstInBoundsGEP1_32(ctor->builder_.getInt8Ty(), array, dataOffset);
3601
3602 llvm::Value *ptrElem = ctor->builder_.CreateInBoundsGEP(ltype, ptrData, ctor->GetInputValue(inst, 1));
3603
3604 llvm::Value *n = ctor->builder_.CreateLoad(ltype, ptrElem);
3605 ctor->ValueMapAdd(inst, n);
3606 }
3607
VisitLoadCompressedStringChar(GraphVisitor * v,Inst * inst)3608 void LLVMIrConstructor::VisitLoadCompressedStringChar(GraphVisitor *v, Inst *inst)
3609 {
3610 auto ctor = static_cast<LLVMIrConstructor *>(v);
3611 auto loadString = inst->CastToLoadCompressedStringChar();
3612
3613 ASSERT(inst->GetType() == DataType::UINT16);
3614
3615 auto array = ctor->GetInputValue(loadString, 0);
3616 auto index = ctor->GetInputValue(loadString, 1);
3617 auto length = ctor->GetInputValue(loadString, 2);
3618
3619 ASSERT(ctor->GetGraph()->GetRuntime()->GetStringCompressionMask() == 1U);
3620 auto compressionMask = ctor->builder_.getInt32(ctor->GetGraph()->GetRuntime()->GetStringCompressionMask());
3621 auto dataOff = ctor->GetGraph()->GetRuntime()->GetStringDataOffset(ctor->GetGraph()->GetArch());
3622 auto chars = ctor->builder_.CreateConstInBoundsGEP1_64(ctor->builder_.getInt8Ty(), array, dataOff);
3623 auto isCompressed = ctor->builder_.CreateIsNull(ctor->builder_.CreateAnd(length, compressionMask));
3624
3625 /**
3626 * int32_t CompressedCharAt(uint8_t *string, int32_t index) {
3627 * int32_t length = LenArray(string, LENGTH_OFFSET, SHIFT);
3628 * bool isCompressed = (length & COMPRESSION_MASK) == 0;
3629 * uint8_t *chars = string + DATA_OFFSET;
3630 *
3631 * uint16_t c;
3632 * if (isCompressed) {
3633 * // compressedBb
3634 * c = static_cast<uint16_t>(chars[index]);
3635 * } else {
3636 * // uncompressedBb
3637 * c = reinterpret_cast<uint16_t *>(chars)[index];
3638 * }
3639 * // Coercing
3640 * return static_cast<int32_t>(c);
3641 * }
3642 */
3643 auto compressedBb =
3644 llvm::BasicBlock::Create(ctor->func_->getContext(), CreateBasicBlockName(inst, "compressed_bb"), ctor->func_);
3645 auto uncompressedBb =
3646 llvm::BasicBlock::Create(ctor->func_->getContext(), CreateBasicBlockName(inst, "uncompressed_bb"), ctor->func_);
3647 auto continuation =
3648 llvm::BasicBlock::Create(ctor->func_->getContext(), CreateBasicBlockName(inst, "char_at_cont"), ctor->func_);
3649 ctor->builder_.CreateCondBr(isCompressed, compressedBb, uncompressedBb);
3650 llvm::Value *compressedChar;
3651 {
3652 ctor->SetCurrentBasicBlock(compressedBb);
3653 ASSERT_TYPE(chars, ctor->builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
3654 auto charAt = ctor->builder_.CreateInBoundsGEP(ctor->builder_.getInt8Ty(), chars, index);
3655 auto character = ctor->builder_.CreateLoad(ctor->builder_.getInt8Ty(), charAt);
3656 compressedChar = ctor->builder_.CreateSExt(character, ctor->builder_.getInt16Ty());
3657 ctor->builder_.CreateBr(continuation);
3658 }
3659
3660 llvm::Value *uncompressedChar;
3661 {
3662 ctor->SetCurrentBasicBlock(uncompressedBb);
3663 auto u16CharAt = ctor->builder_.CreateInBoundsGEP(ctor->builder_.getInt16Ty(), chars, index);
3664 uncompressedChar = ctor->builder_.CreateLoad(ctor->builder_.getInt16Ty(), u16CharAt);
3665 ctor->builder_.CreateBr(continuation);
3666 }
3667 ctor->SetCurrentBasicBlock(continuation);
3668
3669 auto charAt = ctor->builder_.CreatePHI(ctor->builder_.getInt16Ty(), 2U);
3670 charAt->addIncoming(compressedChar, compressedBb);
3671 charAt->addIncoming(uncompressedChar, uncompressedBb);
3672 ctor->ValueMapAdd(inst, charAt);
3673 }
3674
VisitStoreArray(GraphVisitor * v,Inst * inst)3675 void LLVMIrConstructor::VisitStoreArray(GraphVisitor *v, Inst *inst)
3676 {
3677 auto ctor = static_cast<LLVMIrConstructor *>(v);
3678 auto array = ctor->GetInputValue(inst, 0);
3679 ASSERT_TYPE(array, ctor->builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
3680 auto value = ctor->GetInputValue(inst, 2U);
3681
3682 auto dtype = inst->GetType();
3683 auto arch = ctor->GetGraph()->GetArch();
3684 auto ltype = ctor->GetExactType(dtype);
3685 auto dataOff = ctor->GetGraph()->GetRuntime()->GetArrayDataOffset(arch);
3686 auto ptrData = ctor->builder_.CreateConstInBoundsGEP1_32(ctor->builder_.getInt8Ty(), array, dataOff);
3687 auto index = ctor->GetInputValue(inst, 1);
3688 auto ptrElem = ctor->builder_.CreateInBoundsGEP(ltype, ptrData, index);
3689
3690 // Pre
3691 if (inst->CastToStoreArray()->GetNeedBarrier()) {
3692 ctor->CreatePreWRB(inst, ptrElem);
3693 }
3694 // Write
3695 ctor->builder_.CreateStore(value, ptrElem);
3696 // Post
3697 if (inst->CastToStoreArray()->GetNeedBarrier()) {
3698 auto indexOffset = ctor->builder_.CreateBinOp(llvm::Instruction::Shl, index,
3699 ctor->builder_.getInt32(DataType::ShiftByType(dtype, arch)));
3700 auto offset = ctor->builder_.CreateBinOp(llvm::Instruction::Add, indexOffset, ctor->builder_.getInt32(dataOff));
3701 ctor->CreatePostWRB(inst, array, offset, value);
3702 }
3703 }
3704
VisitLoad(GraphVisitor * v,Inst * inst)3705 void LLVMIrConstructor::VisitLoad(GraphVisitor *v, Inst *inst)
3706 {
3707 auto ctor = static_cast<LLVMIrConstructor *>(v);
3708 auto srcPtr = ctor->GetInputValue(inst, 0);
3709 ASSERT(srcPtr->getType()->isPointerTy());
3710
3711 llvm::Value *offset;
3712 auto offsetInput = inst->GetInput(1).GetInst();
3713 auto offsetItype = offsetInput->GetType();
3714 if (offsetItype == DataType::UINT64 || offsetItype == DataType::INT64) {
3715 ASSERT(offsetInput->GetOpcode() != Opcode::Load && offsetInput->GetOpcode() != Opcode::LoadI);
3716 offset = ctor->GetInputValue(inst, 1, true);
3717 } else {
3718 offset = ctor->GetInputValue(inst, 1);
3719 }
3720
3721 ASSERT(srcPtr->getType()->isPointerTy());
3722 auto ptr = ctor->builder_.CreateInBoundsGEP(ctor->builder_.getInt8Ty(), srcPtr, offset);
3723
3724 auto n = ctor->CreateLoadWithOrdering(inst, ptr, ToAtomicOrdering(inst->CastToLoad()->GetVolatile()));
3725 ctor->ValueMapAdd(inst, n);
3726 }
3727
VisitStore(GraphVisitor * v,Inst * inst)3728 void LLVMIrConstructor::VisitStore(GraphVisitor *v, Inst *inst)
3729 {
3730 auto ctor = static_cast<LLVMIrConstructor *>(v);
3731 auto srcPtr = ctor->GetInputValue(inst, 0);
3732 auto value = ctor->GetInputValue(inst, 2U);
3733
3734 llvm::Value *offset;
3735 auto offsetInput = inst->GetInput(1).GetInst();
3736 auto offsetItype = offsetInput->GetType();
3737 if (offsetItype == DataType::UINT64 || offsetItype == DataType::INT64) {
3738 ASSERT(offsetInput->GetOpcode() != Opcode::Load && offsetInput->GetOpcode() != Opcode::LoadI);
3739 offset = ctor->GetInputValue(inst, 1, true);
3740 } else {
3741 offset = ctor->GetInputValue(inst, 1);
3742 }
3743
3744 auto ptrPlus = ctor->builder_.CreateInBoundsGEP(ctor->builder_.getInt8Ty(), srcPtr, offset);
3745
3746 // Pre
3747 if (inst->CastToStore()->GetNeedBarrier()) {
3748 ctor->CreatePreWRB(inst, ptrPlus);
3749 }
3750 // Write
3751 ctor->CreateStoreWithOrdering(value, ptrPlus, ToAtomicOrdering(inst->CastToStore()->GetVolatile()));
3752 // Post
3753 if (inst->CastToStore()->GetNeedBarrier()) {
3754 ctor->CreatePostWRB(inst, srcPtr, offset, value);
3755 }
3756 }
3757
VisitLoadI(GraphVisitor * v,Inst * inst)3758 void LLVMIrConstructor::VisitLoadI(GraphVisitor *v, Inst *inst)
3759 {
3760 auto ctor = static_cast<LLVMIrConstructor *>(v);
3761 auto srcPtr = ctor->GetInputValue(inst, 0);
3762 auto index = inst->CastToLoadI()->GetImm();
3763
3764 ASSERT(srcPtr->getType()->isPointerTy());
3765 auto ptrPlus = ctor->builder_.CreateConstInBoundsGEP1_64(ctor->builder_.getInt8Ty(), srcPtr, index);
3766
3767 auto n = ctor->CreateLoadWithOrdering(inst, ptrPlus, ToAtomicOrdering(inst->CastToLoadI()->GetVolatile()));
3768 ctor->ValueMapAdd(inst, n);
3769 }
3770
VisitStoreI(GraphVisitor * v,Inst * inst)3771 void LLVMIrConstructor::VisitStoreI(GraphVisitor *v, Inst *inst)
3772 {
3773 auto ctor = static_cast<LLVMIrConstructor *>(v);
3774 auto srcPtr = ctor->GetInputValue(inst, 0);
3775 auto value = ctor->GetInputValue(inst, 1);
3776
3777 auto index = inst->CastToStoreI()->GetImm();
3778 ASSERT(srcPtr->getType()->isPointerTy());
3779 auto ptrPlus = ctor->builder_.CreateConstInBoundsGEP1_64(ctor->builder_.getInt8Ty(), srcPtr, index);
3780
3781 // Pre
3782 if (inst->CastToStoreI()->GetNeedBarrier()) {
3783 ctor->CreatePreWRB(inst, ptrPlus);
3784 }
3785 // Write
3786 ctor->CreateStoreWithOrdering(value, ptrPlus, ToAtomicOrdering(inst->CastToStoreI()->GetVolatile()));
3787 // Post
3788 if (inst->CastToStoreI()->GetNeedBarrier()) {
3789 ctor->CreatePostWRB(inst, srcPtr, ctor->builder_.getInt32(index), value);
3790 }
3791 }
3792
VisitLoadObject(GraphVisitor * v,Inst * inst)3793 void LLVMIrConstructor::VisitLoadObject(GraphVisitor *v, Inst *inst)
3794 {
3795 auto ctor = static_cast<LLVMIrConstructor *>(v);
3796 auto obj = ctor->GetInputValue(inst, 0);
3797 ASSERT_TYPE(obj, ctor->builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
3798
3799 auto field = inst->CastToLoadObject()->GetObjField();
3800 auto dataOff = ctor->GetGraph()->GetRuntime()->GetFieldOffset(field);
3801 auto ptrData = ctor->builder_.CreateConstInBoundsGEP1_32(ctor->builder_.getInt8Ty(), obj, dataOff);
3802
3803 auto n = ctor->CreateLoadWithOrdering(inst, ptrData, ToAtomicOrdering(inst->CastToLoadObject()->GetVolatile()));
3804 ctor->ValueMapAdd(inst, n);
3805 }
3806
VisitStoreObject(GraphVisitor * v,Inst * inst)3807 void LLVMIrConstructor::VisitStoreObject(GraphVisitor *v, Inst *inst)
3808 {
3809 auto ctor = static_cast<LLVMIrConstructor *>(v);
3810 auto obj = ctor->GetInputValue(inst, 0);
3811 ASSERT_TYPE(obj, ctor->builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
3812 auto value = ctor->GetInputValue(inst, 1);
3813
3814 auto field = inst->CastToStoreObject()->GetObjField();
3815 auto dataOff = ctor->GetGraph()->GetRuntime()->GetFieldOffset(field);
3816
3817 auto ptrData = ctor->builder_.CreateConstInBoundsGEP1_32(ctor->builder_.getInt8Ty(), obj, dataOff);
3818
3819 // Pre
3820 if (inst->CastToStoreObject()->GetNeedBarrier()) {
3821 ctor->CreatePreWRB(inst, ptrData);
3822 }
3823 // Write
3824 ctor->CreateStoreWithOrdering(value, ptrData, ToAtomicOrdering(inst->CastToStoreObject()->GetVolatile()));
3825 // Post
3826 if (inst->CastToStoreObject()->GetNeedBarrier()) {
3827 ctor->CreatePostWRB(inst, obj, ctor->builder_.getInt32(dataOff), value);
3828 }
3829 }
3830
VisitResolveObjectField(GraphVisitor * v,Inst * inst)3831 void LLVMIrConstructor::VisitResolveObjectField(GraphVisitor *v, Inst *inst)
3832 {
3833 auto ctor = static_cast<LLVMIrConstructor *>(v);
3834
3835 auto typeId = ctor->builder_.getInt32(inst->CastToResolveObjectField()->GetTypeId());
3836
3837 auto entrypointId = RuntimeInterface::EntrypointId::GET_FIELD_OFFSET;
3838 auto offset = ctor->CreateEntrypointCall(entrypointId, inst, {ctor->GetMethodArgument(), typeId});
3839
3840 ctor->ValueMapAdd(inst, offset);
3841 }
3842
VisitLoadResolvedObjectField(GraphVisitor * v,Inst * inst)3843 void LLVMIrConstructor::VisitLoadResolvedObjectField(GraphVisitor *v, Inst *inst)
3844 {
3845 auto ctor = static_cast<LLVMIrConstructor *>(v);
3846 auto obj = ctor->GetInputValue(inst, 0);
3847 ASSERT_TYPE(obj, ctor->builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
3848
3849 auto offset = ctor->GetInputValue(inst, 1);
3850 auto ptrData = ctor->builder_.CreateInBoundsGEP(ctor->builder_.getInt8Ty(), obj, offset);
3851
3852 auto n = ctor->CreateLoadWithOrdering(inst, ptrData, LLVMArkInterface::VOLATILE_ORDER);
3853 ctor->ValueMapAdd(inst, n);
3854 }
3855
VisitStoreResolvedObjectField(GraphVisitor * v,Inst * inst)3856 void LLVMIrConstructor::VisitStoreResolvedObjectField(GraphVisitor *v, Inst *inst)
3857 {
3858 auto ctor = static_cast<LLVMIrConstructor *>(v);
3859 auto obj = ctor->GetInputValue(inst, 0);
3860 ASSERT_TYPE(obj, ctor->builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
3861 auto value = ctor->GetInputValue(inst, 1);
3862
3863 auto offset = ctor->GetInputValue(inst, 2);
3864 auto ptrData = ctor->builder_.CreateInBoundsGEP(ctor->builder_.getInt8Ty(), obj, offset);
3865
3866 // Pre
3867 if (inst->CastToStoreResolvedObjectField()->GetNeedBarrier()) {
3868 ctor->CreatePreWRB(inst, ptrData);
3869 }
3870 // Write
3871 ctor->CreateStoreWithOrdering(value, ptrData, LLVMArkInterface::VOLATILE_ORDER);
3872 // Post
3873 if (inst->CastToStoreResolvedObjectField()->GetNeedBarrier()) {
3874 ctor->CreatePostWRB(inst, obj, offset, value);
3875 }
3876 }
3877
VisitResolveObjectFieldStatic(GraphVisitor * v,Inst * inst)3878 void LLVMIrConstructor::VisitResolveObjectFieldStatic(GraphVisitor *v, Inst *inst)
3879 {
3880 auto ctor = static_cast<LLVMIrConstructor *>(v);
3881 auto resolverInst = inst->CastToResolveObjectFieldStatic();
3882
3883 auto entrypoint = RuntimeInterface::EntrypointId::GET_UNKNOWN_STATIC_FIELD_MEMORY_ADDRESS;
3884
3885 auto typeId = ctor->builder_.getInt32(resolverInst->GetTypeId());
3886 auto slotPtr = llvm::Constant::getNullValue(ctor->builder_.getPtrTy());
3887
3888 auto ptrInt = ctor->CreateEntrypointCall(entrypoint, inst, {ctor->GetMethodArgument(), typeId, slotPtr});
3889 auto n = ctor->builder_.CreateIntToPtr(ptrInt, ctor->builder_.getPtrTy());
3890 ctor->ValueMapAdd(inst, n);
3891 }
3892
VisitLoadResolvedObjectFieldStatic(GraphVisitor * v,Inst * inst)3893 void LLVMIrConstructor::VisitLoadResolvedObjectFieldStatic(GraphVisitor *v, Inst *inst)
3894 {
3895 auto ctor = static_cast<LLVMIrConstructor *>(v);
3896 auto offset = ctor->GetInputValue(inst, 0);
3897
3898 auto casted = ctor->builder_.CreateIntToPtr(offset, ctor->builder_.getPtrTy());
3899 auto n = ctor->CreateLoadWithOrdering(inst, casted, LLVMArkInterface::VOLATILE_ORDER);
3900 ctor->ValueMapAdd(inst, n);
3901 }
3902
VisitStoreResolvedObjectFieldStatic(GraphVisitor * v,Inst * inst)3903 void LLVMIrConstructor::VisitStoreResolvedObjectFieldStatic(GraphVisitor *v, Inst *inst)
3904 {
3905 auto ctor = static_cast<LLVMIrConstructor *>(v);
3906 [[maybe_unused]] auto storeInst = inst->CastToStoreResolvedObjectFieldStatic();
3907
3908 ASSERT(!DataType::IsReference(inst->GetType()));
3909 ASSERT(!storeInst->GetNeedBarrier());
3910
3911 auto value = ctor->GetInputValue(inst, 1);
3912 auto destPtr = ctor->GetInputValue(inst, 0);
3913
3914 [[maybe_unused]] auto dtype = inst->GetType();
3915 ASSERT(value->getType()->getScalarSizeInBits() == DataType::GetTypeSize(dtype, ctor->GetGraph()->GetArch()));
3916 ctor->CreateStoreWithOrdering(value, destPtr, LLVMArkInterface::VOLATILE_ORDER);
3917 }
3918
VisitBitcast(GraphVisitor * v,Inst * inst)3919 void LLVMIrConstructor::VisitBitcast(GraphVisitor *v, Inst *inst)
3920 {
3921 auto ctor = static_cast<LLVMIrConstructor *>(v);
3922 auto type = inst->GetType();
3923 auto llvmTargetType = ctor->GetExactType(type);
3924 auto input = ctor->GetInputValue(inst, 0);
3925 auto itype = inst->GetInputType(0);
3926
3927 llvm::Value *n;
3928 if (itype == DataType::POINTER) {
3929 ASSERT(!llvmTargetType->isPointerTy());
3930 n = ctor->builder_.CreatePtrToInt(input, llvmTargetType);
3931 } else {
3932 if (type == DataType::REFERENCE) {
3933 n = ctor->builder_.CreateIntToPtr(input, ctor->builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
3934 } else if (type == DataType::POINTER) {
3935 n = ctor->builder_.CreateIntToPtr(input, ctor->builder_.getPtrTy());
3936 } else {
3937 n = ctor->builder_.CreateBitCast(input, llvmTargetType);
3938 }
3939 }
3940 ctor->ValueMapAdd(inst, n);
3941 }
3942
VisitCast(GraphVisitor * v,Inst * inst)3943 void LLVMIrConstructor::VisitCast(GraphVisitor *v, Inst *inst)
3944 {
3945 auto ctor = static_cast<LLVMIrConstructor *>(v);
3946 auto x = ctor->GetInputValue(inst, 0);
3947
3948 auto type = inst->GetInputType(0);
3949 auto targetType = inst->GetType();
3950 auto llvmTargetType = ctor->GetExactType(targetType);
3951 // Do not cast if either Ark or LLVM types are the same
3952 if (type == targetType || x->getType() == llvmTargetType) {
3953 ctor->ValueMapAdd(inst, x, false);
3954 return;
3955 }
3956
3957 if (DataType::IsFloatType(type) && IsInteger(targetType)) {
3958 // float to int, e.g. F64TOI32, F32TOI64, F64TOU32, F32TOU64
3959 auto n = ctor->CreateCastToInt(inst);
3960 ctor->ValueMapAdd(inst, n);
3961 return;
3962 }
3963 auto op = ctor->GetCastOp(type, targetType);
3964 if (targetType == DataType::BOOL) {
3965 ASSERT(op == llvm::Instruction::Trunc);
3966 auto u1 = ctor->builder_.CreateIsNotNull(x, CreateNameForInst(inst));
3967 auto n = ctor->builder_.CreateZExt(u1, ctor->builder_.getInt8Ty());
3968 ctor->ValueMapAdd(inst, n, false);
3969 return;
3970 }
3971 auto n = ctor->builder_.CreateCast(op, x, llvmTargetType);
3972 ctor->ValueMapAdd(inst, n);
3973 }
3974
VisitAnd(GraphVisitor * v,Inst * inst)3975 void LLVMIrConstructor::VisitAnd(GraphVisitor *v, Inst *inst)
3976 {
3977 auto ctor = static_cast<LLVMIrConstructor *>(v);
3978 auto n = ctor->CreateBinaryOp(inst, llvm::Instruction::And);
3979 ctor->ValueMapAdd(inst, n);
3980 }
3981
VisitAndI(GraphVisitor * v,Inst * inst)3982 void LLVMIrConstructor::VisitAndI(GraphVisitor *v, Inst *inst)
3983 {
3984 auto ctor = static_cast<LLVMIrConstructor *>(v);
3985 auto n = ctor->CreateBinaryImmOp(inst, llvm::Instruction::And, inst->CastToAndI()->GetImm());
3986 ctor->ValueMapAdd(inst, n);
3987 }
3988
VisitOr(GraphVisitor * v,Inst * inst)3989 void LLVMIrConstructor::VisitOr(GraphVisitor *v, Inst *inst)
3990 {
3991 auto ctor = static_cast<LLVMIrConstructor *>(v);
3992 auto n = ctor->CreateBinaryOp(inst, llvm::Instruction::Or);
3993 ctor->ValueMapAdd(inst, n);
3994 }
3995
VisitOrI(GraphVisitor * v,Inst * inst)3996 void LLVMIrConstructor::VisitOrI(GraphVisitor *v, Inst *inst)
3997 {
3998 auto ctor = static_cast<LLVMIrConstructor *>(v);
3999 auto n = ctor->CreateBinaryImmOp(inst, llvm::Instruction::Or, inst->CastToOrI()->GetImm());
4000 ctor->ValueMapAdd(inst, n);
4001 }
4002
VisitXor(GraphVisitor * v,Inst * inst)4003 void LLVMIrConstructor::VisitXor(GraphVisitor *v, Inst *inst)
4004 {
4005 auto ctor = static_cast<LLVMIrConstructor *>(v);
4006 auto n = ctor->CreateBinaryOp(inst, llvm::Instruction::Xor);
4007 ctor->ValueMapAdd(inst, n);
4008 }
4009
VisitXorI(GraphVisitor * v,Inst * inst)4010 void LLVMIrConstructor::VisitXorI(GraphVisitor *v, Inst *inst)
4011 {
4012 auto ctor = static_cast<LLVMIrConstructor *>(v);
4013 auto n = ctor->CreateBinaryImmOp(inst, llvm::Instruction::Xor, inst->CastToXorI()->GetImm());
4014 ctor->ValueMapAdd(inst, n);
4015 }
4016
VisitShl(GraphVisitor * v,Inst * inst)4017 void LLVMIrConstructor::VisitShl(GraphVisitor *v, Inst *inst)
4018 {
4019 auto ctor = static_cast<LLVMIrConstructor *>(v);
4020 auto n = ctor->CreateShiftOp(inst, llvm::Instruction::Shl);
4021 ctor->ValueMapAdd(inst, n);
4022 }
4023
VisitShlI(GraphVisitor * v,Inst * inst)4024 void LLVMIrConstructor::VisitShlI(GraphVisitor *v, Inst *inst)
4025 {
4026 auto ctor = static_cast<LLVMIrConstructor *>(v);
4027 auto n = ctor->CreateBinaryImmOp(inst, llvm::Instruction::Shl, inst->CastToShlI()->GetImm());
4028 ctor->ValueMapAdd(inst, n);
4029 }
4030
VisitShr(GraphVisitor * v,Inst * inst)4031 void LLVMIrConstructor::VisitShr(GraphVisitor *v, Inst *inst)
4032 {
4033 auto ctor = static_cast<LLVMIrConstructor *>(v);
4034 auto n = ctor->CreateShiftOp(inst, llvm::Instruction::LShr);
4035 ctor->ValueMapAdd(inst, n);
4036 }
4037
VisitShrI(GraphVisitor * v,Inst * inst)4038 void LLVMIrConstructor::VisitShrI(GraphVisitor *v, Inst *inst)
4039 {
4040 auto ctor = static_cast<LLVMIrConstructor *>(v);
4041 auto n = ctor->CreateBinaryImmOp(inst, llvm::Instruction::LShr, inst->CastToShrI()->GetImm());
4042 ctor->ValueMapAdd(inst, n);
4043 }
4044
VisitAShr(GraphVisitor * v,Inst * inst)4045 void LLVMIrConstructor::VisitAShr(GraphVisitor *v, Inst *inst)
4046 {
4047 auto ctor = static_cast<LLVMIrConstructor *>(v);
4048 auto n = ctor->CreateShiftOp(inst, llvm::Instruction::AShr);
4049 ctor->ValueMapAdd(inst, n);
4050 }
4051
VisitAShrI(GraphVisitor * v,Inst * inst)4052 void LLVMIrConstructor::VisitAShrI(GraphVisitor *v, Inst *inst)
4053 {
4054 auto ctor = static_cast<LLVMIrConstructor *>(v);
4055 auto n = ctor->CreateBinaryImmOp(inst, llvm::Instruction::AShr, inst->CastToAShrI()->GetImm());
4056 ctor->ValueMapAdd(inst, n);
4057 }
4058
VisitAdd(GraphVisitor * v,Inst * inst)4059 void LLVMIrConstructor::VisitAdd(GraphVisitor *v, Inst *inst)
4060 {
4061 auto ctor = static_cast<LLVMIrConstructor *>(v);
4062 llvm::Value *n;
4063 if (IsFloatType(inst->GetType())) {
4064 n = ctor->CreateBinaryOp(inst, llvm::Instruction::FAdd);
4065 } else if (IsTypeNumeric(inst->GetType())) {
4066 n = ctor->CreateBinaryOp(inst, llvm::Instruction::Add);
4067 } else {
4068 UNREACHABLE();
4069 }
4070 ctor->ValueMapAdd(inst, n);
4071 }
4072
VisitAddI(GraphVisitor * v,Inst * inst)4073 void LLVMIrConstructor::VisitAddI(GraphVisitor *v, Inst *inst)
4074 {
4075 auto ctor = static_cast<LLVMIrConstructor *>(v);
4076 auto n = ctor->CreateBinaryImmOp(inst, llvm::Instruction::Add, inst->CastToAddI()->GetImm());
4077 ctor->ValueMapAdd(inst, n);
4078 }
4079
VisitSub(GraphVisitor * v,Inst * inst)4080 void LLVMIrConstructor::VisitSub(GraphVisitor *v, Inst *inst)
4081 {
4082 auto ctor = static_cast<LLVMIrConstructor *>(v);
4083 llvm::Value *n;
4084 if (IsFloatType(inst->GetType())) {
4085 n = ctor->CreateBinaryOp(inst, llvm::Instruction::FSub);
4086 } else if (IsTypeNumeric(inst->GetType())) {
4087 n = ctor->CreateBinaryOp(inst, llvm::Instruction::Sub);
4088 } else {
4089 UNREACHABLE();
4090 }
4091 ctor->ValueMapAdd(inst, n);
4092 }
4093
VisitSubI(GraphVisitor * v,Inst * inst)4094 void LLVMIrConstructor::VisitSubI(GraphVisitor *v, Inst *inst)
4095 {
4096 auto ctor = static_cast<LLVMIrConstructor *>(v);
4097 auto n = ctor->CreateBinaryImmOp(inst, llvm::Instruction::Sub, inst->CastToSubI()->GetImm());
4098 ctor->ValueMapAdd(inst, n);
4099 }
4100
VisitMul(GraphVisitor * v,Inst * inst)4101 void LLVMIrConstructor::VisitMul(GraphVisitor *v, Inst *inst)
4102 {
4103 auto ctor = static_cast<LLVMIrConstructor *>(v);
4104 llvm::Value *n;
4105 if (IsFloatType(inst->GetType())) {
4106 n = ctor->CreateBinaryOp(inst, llvm::Instruction::FMul);
4107 } else if (IsTypeNumeric(inst->GetType())) {
4108 n = ctor->CreateBinaryOp(inst, llvm::Instruction::Mul);
4109 } else {
4110 UNREACHABLE();
4111 }
4112 ctor->ValueMapAdd(inst, n);
4113 }
4114
VisitMulI(GraphVisitor * v,Inst * inst)4115 void LLVMIrConstructor::VisitMulI(GraphVisitor *v, Inst *inst)
4116 {
4117 auto ctor = static_cast<LLVMIrConstructor *>(v);
4118 auto n = ctor->CreateBinaryImmOp(inst, llvm::Instruction::Mul, inst->CastToMulI()->GetImm());
4119 ctor->ValueMapAdd(inst, n);
4120 }
4121
VisitDiv(GraphVisitor * v,Inst * inst)4122 void LLVMIrConstructor::VisitDiv(GraphVisitor *v, Inst *inst)
4123 {
4124 auto ctor = static_cast<LLVMIrConstructor *>(v);
4125 auto type = inst->GetType();
4126 llvm::Value *n;
4127 if (IsFloatType(type)) {
4128 n = ctor->CreateBinaryOp(inst, llvm::Instruction::FDiv);
4129 } else if (IsInteger(type)) {
4130 if (IsSignedInteger(type)) {
4131 n = ctor->CreateSignDivMod(inst, llvm::Instruction::SDiv);
4132 } else {
4133 n = ctor->CreateBinaryOp(inst, llvm::Instruction::UDiv);
4134 }
4135 } else {
4136 UNREACHABLE();
4137 }
4138 ctor->ValueMapAdd(inst, n);
4139 }
4140
VisitMod(GraphVisitor * v,Inst * inst)4141 void LLVMIrConstructor::VisitMod(GraphVisitor *v, Inst *inst)
4142 {
4143 auto ctor = static_cast<LLVMIrConstructor *>(v);
4144 auto type = inst->GetType();
4145 llvm::Value *n;
4146 if (IsFloatType(type)) {
4147 n = ctor->CreateBinaryOp(inst, llvm::Instruction::FRem);
4148 } else if (IsInteger(type)) {
4149 if (IsSignedInteger(type)) {
4150 n = ctor->CreateSignDivMod(inst, llvm::Instruction::SRem);
4151 } else {
4152 n = ctor->CreateBinaryOp(inst, llvm::Instruction::URem);
4153 }
4154 } else {
4155 UNREACHABLE();
4156 }
4157 ctor->ValueMapAdd(inst, n);
4158 }
4159
VisitMin(GraphVisitor * v,Inst * inst)4160 void LLVMIrConstructor::VisitMin(GraphVisitor *v, Inst *inst)
4161 {
4162 ASSERT(g_options.IsCompilerEncodeIntrinsics());
4163 auto ctor = static_cast<LLVMIrConstructor *>(v);
4164 auto operType = inst->CastToMin()->GetType();
4165 llvm::Value *x = ctor->GetInputValue(inst, 0);
4166 llvm::Value *y = ctor->GetInputValue(inst, 1);
4167 llvm::Intrinsic::ID llvmId = 0;
4168
4169 if (DataType::IsFloatType(operType)) {
4170 llvmId = llvm::Intrinsic::minimum;
4171 } else if (IsInteger(operType)) {
4172 llvmId = DataType::IsTypeSigned(operType) ? llvm::Intrinsic::smin : llvm::Intrinsic::umin;
4173 } else {
4174 ASSERT_DO(false, (std::cerr << "Min is not supported for type " << DataType::ToString(operType) << std::endl));
4175 UNREACHABLE();
4176 }
4177 auto min = ctor->builder_.CreateBinaryIntrinsic(llvmId, x, y);
4178 ctor->ValueMapAdd(inst, min);
4179 }
4180
VisitMax(GraphVisitor * v,Inst * inst)4181 void LLVMIrConstructor::VisitMax(GraphVisitor *v, Inst *inst)
4182 {
4183 ASSERT(g_options.IsCompilerEncodeIntrinsics());
4184 auto ctor = static_cast<LLVMIrConstructor *>(v);
4185 auto operType = inst->CastToMax()->GetType();
4186 llvm::Value *x = ctor->GetInputValue(inst, 0);
4187 llvm::Value *y = ctor->GetInputValue(inst, 1);
4188 llvm::Intrinsic::ID llvmId = 0;
4189
4190 if (DataType::IsFloatType(operType)) {
4191 llvmId = llvm::Intrinsic::maximum;
4192 } else if (IsInteger(operType)) {
4193 llvmId = DataType::IsTypeSigned(operType) ? llvm::Intrinsic::smax : llvm::Intrinsic::umax;
4194 } else {
4195 ASSERT_DO(false, (std::cerr << "Max is not supported for type " << DataType::ToString(operType) << std::endl));
4196 UNREACHABLE();
4197 }
4198 auto max = ctor->builder_.CreateBinaryIntrinsic(llvmId, x, y);
4199 ctor->ValueMapAdd(inst, max);
4200 }
4201
VisitCompare(GraphVisitor * v,Inst * inst)4202 void LLVMIrConstructor::VisitCompare(GraphVisitor *v, Inst *inst)
4203 {
4204 auto ctor = static_cast<LLVMIrConstructor *>(v);
4205 auto compareInst = inst->CastToCompare();
4206 auto operandsType = compareInst->GetOperandsType();
4207
4208 llvm::Value *x = ctor->GetInputValue(inst, 0);
4209 llvm::Value *y = ctor->GetInputValue(inst, 1);
4210
4211 llvm::Value *n = nullptr;
4212 if (IsInteger(operandsType) || DataType::IsReference(operandsType)) {
4213 n = ctor->CreateCondition(compareInst->GetCc(), x, y);
4214 } else {
4215 n = ctor->builder_.CreateFCmp(FCmpCodeConvert(compareInst->GetCc()), x, y);
4216 }
4217 ctor->ValueMapAdd(inst, n);
4218 }
4219
VisitCmp(GraphVisitor * v,Inst * inst)4220 void LLVMIrConstructor::VisitCmp(GraphVisitor *v, Inst *inst)
4221 {
4222 auto ctor = static_cast<LLVMIrConstructor *>(v);
4223 CmpInst *cmpInst = inst->CastToCmp();
4224 DataType::Type operandsType = cmpInst->GetOperandsType();
4225
4226 auto x = ctor->GetInputValue(inst, 0);
4227 auto y = ctor->GetInputValue(inst, 1);
4228 llvm::Value *n;
4229 if (DataType::IsFloatType(operandsType)) {
4230 n = ctor->CreateFloatComparison(cmpInst, x, y);
4231 } else if (IsInteger(operandsType)) {
4232 n = ctor->CreateIntegerComparison(cmpInst, x, y);
4233 } else {
4234 ASSERT_DO(false, (std::cerr << "Unsupported comparison for operands of type = "
4235 << DataType::ToString(operandsType) << std::endl));
4236 UNREACHABLE();
4237 }
4238 ctor->ValueMapAdd(inst, n);
4239 }
4240
VisitNeg(GraphVisitor * v,Inst * inst)4241 void LLVMIrConstructor::VisitNeg(GraphVisitor *v, Inst *inst)
4242 {
4243 auto ctor = static_cast<LLVMIrConstructor *>(v);
4244 auto inputType = inst->GetInputType(0);
4245 auto toNegate = ctor->GetInputValue(inst, 0);
4246 llvm::Value *n;
4247 if (inputType == DataType::Type::FLOAT64 || inputType == DataType::Type::FLOAT32) {
4248 n = ctor->builder_.CreateFNeg(toNegate);
4249 } else if (IsInteger(inputType)) {
4250 n = ctor->builder_.CreateNeg(toNegate);
4251 } else {
4252 ASSERT_DO(false, (std::cerr << "Negation is not supported for" << DataType::ToString(inputType) << std::endl));
4253 UNREACHABLE();
4254 }
4255 ctor->ValueMapAdd(inst, n);
4256 }
4257
VisitNot(GraphVisitor * v,Inst * inst)4258 void LLVMIrConstructor::VisitNot(GraphVisitor *v, Inst *inst)
4259 {
4260 ASSERT(IsInteger(inst->GetInputType(0)));
4261
4262 auto ctor = static_cast<LLVMIrConstructor *>(v);
4263 auto input = ctor->GetInputValue(inst, 0);
4264
4265 auto notOperator = ctor->builder_.CreateNot(input);
4266 ctor->ValueMapAdd(inst, notOperator);
4267 }
4268
VisitIfImm(GraphVisitor * v,Inst * inst)4269 void LLVMIrConstructor::VisitIfImm(GraphVisitor *v, Inst *inst)
4270 {
4271 auto ctor = static_cast<LLVMIrConstructor *>(v);
4272 auto x = ctor->GetInputValue(inst, 0);
4273 auto ifimm = inst->CastToIfImm();
4274
4275 llvm::Value *cond = nullptr;
4276 if (ifimm->GetCc() == ConditionCode::CC_NE && ifimm->GetImm() == 0 && x->getType()->isIntegerTy()) {
4277 ASSERT(ifimm->GetOperandsType() == DataType::BOOL);
4278 cond = ctor->builder_.CreateTrunc(x, ctor->builder_.getInt1Ty());
4279 } else {
4280 ASSERT(x->getType()->isIntOrPtrTy());
4281 llvm::Constant *immCst;
4282 if (x->getType()->isPointerTy()) {
4283 if (ifimm->GetImm() == 0) {
4284 immCst = llvm::ConstantPointerNull::get(llvm::cast<llvm::PointerType>(x->getType()));
4285 } else {
4286 immCst = llvm::ConstantInt::getSigned(x->getType(), ifimm->GetImm());
4287 immCst = llvm::ConstantExpr::getPointerCast(immCst, x->getType());
4288 }
4289 } else {
4290 immCst = llvm::ConstantInt::getSigned(x->getType(), ifimm->GetImm());
4291 }
4292 cond = ctor->CreateCondition(ifimm->GetCc(), x, immCst);
4293 }
4294 ctor->CreateIf(inst, cond, ifimm->IsLikely(), ifimm->IsUnlikely());
4295 }
4296
VisitIf(GraphVisitor * v,Inst * inst)4297 void LLVMIrConstructor::VisitIf(GraphVisitor *v, Inst *inst)
4298 {
4299 auto ctor = static_cast<LLVMIrConstructor *>(v);
4300 auto x = ctor->GetInputValue(inst, 0);
4301 auto y = ctor->GetInputValue(inst, 1);
4302 ASSERT(x->getType()->isIntOrPtrTy());
4303 ASSERT(y->getType()->isIntOrPtrTy());
4304 auto ifi = inst->CastToIf();
4305 auto cond = ctor->CreateCondition(ifi->GetCc(), x, y);
4306 ctor->CreateIf(inst, cond, ifi->IsLikely(), ifi->IsUnlikely());
4307 }
4308
VisitCallIndirect(GraphVisitor * v,Inst * inst)4309 void LLVMIrConstructor::VisitCallIndirect(GraphVisitor *v, Inst *inst)
4310 {
4311 auto ctor = static_cast<LLVMIrConstructor *>(v);
4312 auto ptr = ctor->GetInputValue(inst, 0);
4313 ASSERT_TYPE(ptr, ctor->builder_.getPtrTy());
4314 // Build FunctionType
4315 ArenaVector<llvm::Type *> argTypes(ctor->GetGraph()->GetLocalAllocator()->Adapter());
4316 ArenaVector<llvm::Value *> args(ctor->GetGraph()->GetLocalAllocator()->Adapter());
4317 for (size_t i = 1; i < inst->GetInputs().Size(); ++i) {
4318 argTypes.push_back(ctor->GetType(inst->GetInput(i).GetInst()->GetType()));
4319 args.push_back(ctor->GetInputValue(inst, i));
4320 }
4321 auto retType = ctor->GetType(inst->GetType());
4322 auto funcType = llvm::FunctionType::get(retType, argTypes, false);
4323 auto call = ctor->builder_.CreateCall(funcType, ptr, args);
4324 if (!retType->isVoidTy()) {
4325 ctor->ValueMapAdd(inst, call);
4326 }
4327 }
4328
VisitCall(GraphVisitor * v,Inst * inst)4329 void LLVMIrConstructor::VisitCall(GraphVisitor *v, Inst *inst)
4330 {
4331 auto ctor = static_cast<LLVMIrConstructor *>(v);
4332 ASSERT(!ctor->GetGraph()->SupportManagedCode());
4333
4334 // Prepare external call if needed
4335 auto externalId = inst->CastToCall()->GetCallMethodId();
4336 auto runtime = ctor->GetGraph()->GetRuntime();
4337 auto externalName = runtime->GetExternalMethodName(ctor->GetGraph()->GetMethod(), externalId);
4338 auto function = ctor->func_->getParent()->getFunction(externalName);
4339 if (function == nullptr) {
4340 ArenaVector<llvm::Type *> argTypes(ctor->GetGraph()->GetLocalAllocator()->Adapter());
4341 for (size_t i = 0; i < inst->GetInputs().Size(); ++i) {
4342 argTypes.push_back(ctor->GetType(inst->GetInputType(i)));
4343 }
4344 auto ftype = llvm::FunctionType::get(ctor->GetType(inst->GetType()), argTypes, false);
4345 function =
4346 llvm::Function::Create(ftype, llvm::Function::ExternalLinkage, externalName, ctor->func_->getParent());
4347 }
4348 // Arguments
4349 ArenaVector<llvm::Value *> args(ctor->GetGraph()->GetLocalAllocator()->Adapter());
4350 for (size_t i = 0; i < inst->GetInputs().Size(); ++i) {
4351 args.push_back(ctor->CoerceValue(ctor->GetInputValue(inst, i), function->getArg(i)->getType()));
4352 }
4353 // Call
4354 auto call = ctor->builder_.CreateCall(function->getFunctionType(), function, args);
4355
4356 if (IsNoAliasIrtocFunction(externalName)) {
4357 ASSERT(call->getType()->isPointerTy());
4358 call->addRetAttr(llvm::Attribute::NoAlias);
4359 } else {
4360 ASSERT(call->getType()->isPointerTy() ^ !IsPtrIgnIrtocFunction(externalName));
4361 }
4362
4363 // Check if function has debug info
4364 if (function->getSubprogram() != nullptr) {
4365 ctor->debugData_->SetLocation(call, inst->GetPc());
4366 }
4367
4368 if (inst->GetType() != DataType::VOID) {
4369 ctor->ValueMapAdd(inst, ctor->CoerceValue(call, ctor->GetType(inst->GetType())));
4370 }
4371 }
4372
VisitPhi(GraphVisitor * v,Inst * inst)4373 void LLVMIrConstructor::VisitPhi(GraphVisitor *v, Inst *inst)
4374 {
4375 auto ctor = static_cast<LLVMIrConstructor *>(v);
4376 auto ltype = ctor->GetExactType(inst->GetType());
4377 auto block = ctor->GetCurrentBasicBlock();
4378
4379 // PHI need adjusted insert point if ValueMapAdd already created coerced values for other PHIs
4380 auto nonPhi = block->getFirstNonPHI();
4381 if (nonPhi != nullptr) {
4382 ctor->builder_.SetInsertPoint(nonPhi);
4383 }
4384
4385 auto phi = ctor->builder_.CreatePHI(ltype, inst->GetInputsCount());
4386 ctor->SetCurrentBasicBlock(block);
4387 ctor->ValueMapAdd(inst, phi);
4388 }
4389
VisitMultiArray(GraphVisitor * v,Inst * inst)4390 void LLVMIrConstructor::VisitMultiArray(GraphVisitor *v, Inst *inst)
4391 {
4392 auto ctor = static_cast<LLVMIrConstructor *>(v);
4393
4394 ArenaVector<llvm::Value *> args(ctor->GetGraph()->GetLocalAllocator()->Adapter());
4395 args.push_back(ctor->GetInputValue(inst, 0));
4396
4397 auto sizesCount = inst->GetInputsCount() - 2U;
4398 args.push_back(ctor->builder_.getInt32(sizesCount));
4399 auto sizes = ctor->CreateAllocaForArgs(ctor->builder_.getInt64Ty(), sizesCount);
4400
4401 // Store multi-array sizes
4402 for (size_t i = 1; i <= sizesCount; i++) {
4403 auto size = ctor->GetInputValue(inst, i);
4404
4405 auto type = inst->GetInputType(i);
4406 if (type != DataType::INT64) {
4407 size = ctor->CoerceValue(size, type, DataType::INT64);
4408 }
4409
4410 auto gep = ctor->builder_.CreateConstInBoundsGEP1_32(ctor->builder_.getInt64Ty(), sizes, i - 1);
4411 ctor->builder_.CreateStore(size, gep);
4412 }
4413 args.push_back(sizes);
4414
4415 auto entrypointId = RuntimeInterface::EntrypointId::CREATE_MULTI_ARRAY;
4416 auto result = ctor->CreateEntrypointCall(entrypointId, inst, args);
4417 ctor->MarkAsAllocation(result);
4418 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
4419 result->addFnAttr(llvm::Attribute::get(result->getContext(), "needs-mem-barrier"));
4420 }
4421 ctor->ValueMapAdd(inst, result);
4422 }
4423
VisitInitEmptyString(GraphVisitor * v,Inst * inst)4424 void LLVMIrConstructor::VisitInitEmptyString(GraphVisitor *v, Inst *inst)
4425 {
4426 auto ctor = static_cast<LLVMIrConstructor *>(v);
4427 auto eid = RuntimeInterface::EntrypointId::CREATE_EMPTY_STRING;
4428 auto result = ctor->CreateEntrypointCall(eid, inst);
4429 ctor->MarkAsAllocation(result);
4430 ctor->ValueMapAdd(inst, result);
4431 }
4432
VisitInitString(GraphVisitor * v,Inst * inst)4433 void LLVMIrConstructor::VisitInitString(GraphVisitor *v, Inst *inst)
4434 {
4435 auto ctor = static_cast<LLVMIrConstructor *>(v);
4436 auto strInit = inst->CastToInitString();
4437 auto arg = ctor->GetInputValue(inst, 0);
4438 if (strInit->IsFromString()) {
4439 auto result = ctor->CreateNewStringFromStringTlab(inst, arg);
4440 ctor->ValueMapAdd(inst, result);
4441 } else {
4442 auto lengthOffset = ctor->GetGraph()->GetRuntime()->GetArrayLengthOffset(ctor->GetGraph()->GetArch());
4443 auto lengthPtr = ctor->builder_.CreateConstInBoundsGEP1_32(ctor->builder_.getInt8Ty(), arg, lengthOffset);
4444 auto length = ctor->builder_.CreateLoad(ctor->builder_.getInt32Ty(), lengthPtr);
4445 auto result = ctor->CreateNewStringFromCharsTlab(
4446 inst, llvm::Constant::getNullValue(ctor->builder_.getInt32Ty()), length, arg);
4447 ctor->ValueMapAdd(inst, result);
4448 }
4449 }
4450
VisitNewArray(GraphVisitor * v,Inst * inst)4451 void LLVMIrConstructor::VisitNewArray(GraphVisitor *v, Inst *inst)
4452 {
4453 auto ctor = static_cast<LLVMIrConstructor *>(v);
4454 auto method = inst->CastToNewArray()->GetMethod();
4455
4456 auto type = ctor->GetInputValue(inst, 0);
4457 auto size = ctor->ToSizeT(ctor->GetInputValue(inst, 1));
4458 auto arrayType = inst->CastToNewArray()->GetTypeId();
4459 auto runtime = ctor->GetGraph()->GetRuntime();
4460 auto maxTlabSize = runtime->GetTLABMaxSize();
4461 if (maxTlabSize == 0) {
4462 auto result = ctor->CreateNewArrayWithRuntime(inst);
4463 ctor->ValueMapAdd(inst, result);
4464 return;
4465 }
4466
4467 auto lenInst = inst->GetDataFlowInput(0);
4468 auto classArraySize = runtime->GetClassArraySize(ctor->GetGraph()->GetArch());
4469 uint64_t arraySize = 0;
4470 uint64_t elementSize = runtime->GetArrayElementSize(method, arrayType);
4471 uint64_t alignment = runtime->GetTLABAlignment();
4472 ASSERT(alignment != 0);
4473
4474 if (lenInst->GetOpcode() == Opcode::Constant) {
4475 ASSERT(lenInst->GetType() == DataType::INT64);
4476 arraySize = lenInst->CastToConstant()->GetIntValue() * elementSize + classArraySize;
4477 arraySize = (arraySize & ~(alignment - 1U)) + ((arraySize % alignment) != 0U ? alignment : 0U);
4478 if (arraySize > maxTlabSize) {
4479 auto result = ctor->CreateNewArrayWithRuntime(inst);
4480 ctor->ValueMapAdd(inst, result);
4481 return;
4482 }
4483 }
4484 auto eid = GetAllocateArrayTlabEntrypoint(elementSize);
4485 auto result = ctor->CreateFastPathCall(inst, eid, {type, size});
4486 ctor->MarkAsAllocation(result);
4487 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
4488 result->addFnAttr(llvm::Attribute::get(result->getContext(), "needs-mem-barrier"));
4489 }
4490 ctor->ValueMapAdd(inst, result);
4491 }
4492
VisitNewObject(GraphVisitor * v,Inst * inst)4493 void LLVMIrConstructor::VisitNewObject(GraphVisitor *v, Inst *inst)
4494 {
4495 auto ctor = static_cast<LLVMIrConstructor *>(v);
4496
4497 auto newObjInst = inst->CastToNewObject();
4498 auto srcInst = newObjInst->GetInput(0).GetInst();
4499
4500 auto runtime = ctor->GetGraph()->GetRuntime();
4501 auto maxTlabSize = runtime->GetTLABMaxSize();
4502 if (maxTlabSize == 0 || srcInst->GetOpcode() != Opcode::LoadAndInitClass) {
4503 auto runtimeCall = ctor->CreateNewObjectWithRuntime(inst);
4504 ctor->ValueMapAdd(inst, runtimeCall);
4505 return;
4506 }
4507
4508 auto klass = srcInst->CastToLoadAndInitClass()->GetClass();
4509 if (klass == nullptr || !runtime->CanUseTlabForClass(klass)) {
4510 auto runtimeCall = ctor->CreateNewObjectWithRuntime(inst);
4511 ctor->ValueMapAdd(inst, runtimeCall);
4512 return;
4513 }
4514 auto classSize = runtime->GetClassSize(klass);
4515 auto alignment = runtime->GetTLABAlignment();
4516 ASSERT(alignment != 0);
4517
4518 classSize = (classSize & ~(alignment - 1U)) + ((classSize % alignment) != 0U ? alignment : 0U);
4519 if (classSize > maxTlabSize) {
4520 auto runtimeCall = ctor->CreateNewObjectWithRuntime(inst);
4521 ctor->ValueMapAdd(inst, runtimeCall);
4522 return;
4523 }
4524
4525 auto initClass = ctor->GetInputValue(inst, 0);
4526 auto klassSize = ctor->ToSizeT(ctor->builder_.getInt32(classSize));
4527 auto eid = RuntimeInterface::EntrypointId::ALLOCATE_OBJECT_TLAB;
4528 auto result = ctor->CreateFastPathCall(inst, eid, {initClass, klassSize});
4529 ctor->MarkAsAllocation(result);
4530 if (inst->GetFlag(inst_flags::MEM_BARRIER)) {
4531 result->addFnAttr(llvm::Attribute::get(result->getContext(), "needs-mem-barrier"));
4532 }
4533 ctor->ValueMapAdd(inst, result);
4534 }
4535
VisitCallStatic(GraphVisitor * v,Inst * inst)4536 void LLVMIrConstructor::VisitCallStatic(GraphVisitor *v, Inst *inst)
4537 {
4538 auto call = inst->CastToCallStatic();
4539 if (call->IsInlined()) {
4540 return;
4541 }
4542
4543 auto ctor = static_cast<LLVMIrConstructor *>(v);
4544 auto methodPtr = ctor->GetGraph()->GetMethod();
4545 auto methodId = call->GetCallMethodId();
4546 auto callee = ctor->GetGraph()->GetRuntime()->GetMethodById(methodPtr, methodId);
4547 ASSERT(callee != nullptr);
4548 // Create a declare statement if we haven't met this function yet
4549 auto function = ctor->GetOrCreateFunctionForCall(call, callee);
4550 ctor->arkInterface_->RememberFunctionCall(ctor->func_, function, methodId);
4551
4552 // Replaced to real callee in the PandaRuntimeLowering
4553 auto args = ctor->GetArgumentsForCall(ctor->GetMethodArgument(), call);
4554 auto result = ctor->builder_.CreateCall(function, args, ctor->CreateSaveStateBundle(inst));
4555 ctor->WrapArkCall(inst, result);
4556
4557 if (inst->GetType() != DataType::VOID) {
4558 ctor->ValueMapAdd(inst, result);
4559 }
4560
4561 if (ctor->GetGraph()->GetRuntime()->IsMethodExternal(methodPtr, callee)) {
4562 result->addAttributeAtIndex(llvm::AttributeList::FunctionIndex, llvm::Attribute::NoInline);
4563 }
4564 if (IsAlwaysThrowBasicBlock(inst)) {
4565 result->addAttributeAtIndex(llvm::AttributeList::FunctionIndex, llvm::Attribute::NoInline);
4566 result->addFnAttr(llvm::Attribute::get(ctor->func_->getContext(), "keep-noinline"));
4567 }
4568 }
4569
VisitResolveStatic(GraphVisitor * v,Inst * inst)4570 void LLVMIrConstructor::VisitResolveStatic(GraphVisitor *v, Inst *inst)
4571 {
4572 auto ctor = static_cast<LLVMIrConstructor *>(v);
4573 auto call = inst->CastToResolveStatic();
4574
4575 auto slotPtr = llvm::Constant::getNullValue(ctor->builder_.getPtrTy());
4576 auto methodPtr = ctor->CreateEntrypointCall(
4577 RuntimeInterface::EntrypointId::GET_UNKNOWN_CALLEE_METHOD, inst,
4578 {ctor->GetMethodArgument(), ctor->ToSizeT(ctor->builder_.getInt32(call->GetCallMethodId())), slotPtr});
4579 auto method = ctor->builder_.CreateIntToPtr(methodPtr, ctor->builder_.getPtrTy());
4580
4581 ctor->ValueMapAdd(inst, method);
4582 }
4583
VisitCallResolvedStatic(GraphVisitor * v,Inst * inst)4584 void LLVMIrConstructor::VisitCallResolvedStatic(GraphVisitor *v, Inst *inst)
4585 {
4586 auto ctor = static_cast<LLVMIrConstructor *>(v);
4587 auto call = inst->CastToCallResolvedStatic();
4588 if (call->IsInlined()) {
4589 return;
4590 }
4591
4592 auto method = ctor->GetInputValue(inst, 0);
4593
4594 llvm::FunctionType *fType = ctor->GetFunctionTypeForCall(call);
4595 auto args = ctor->GetArgumentsForCall(method, call, true); // skip first input
4596
4597 auto offset = ctor->GetGraph()->GetRuntime()->GetCompiledEntryPointOffset(ctor->GetGraph()->GetArch());
4598 auto entrypointPtr = ctor->builder_.CreateConstInBoundsGEP1_32(ctor->builder_.getInt8Ty(), method, offset);
4599 auto entrypoint = ctor->builder_.CreateLoad(ctor->builder_.getPtrTy(), entrypointPtr);
4600
4601 auto result = ctor->builder_.CreateCall(fType, entrypoint, args, ctor->CreateSaveStateBundle(inst));
4602 if (inst->GetType() != DataType::VOID) {
4603 ctor->ValueMapAdd(inst, result);
4604 }
4605 ctor->WrapArkCall(inst, result);
4606 }
4607
4608 template <typename T>
CreateDeclForVirtualCall(T * inst,LLVMIrConstructor * ctor,LLVMArkInterface * arkInterface)4609 llvm::Function *CreateDeclForVirtualCall(T *inst, LLVMIrConstructor *ctor, LLVMArkInterface *arkInterface)
4610 {
4611 arkInterface->GetOrCreateRuntimeFunctionType(
4612 ctor->GetFunc()->getContext(), ctor->GetFunc()->getParent(), LLVMArkInterface::RuntimeCallType::ENTRYPOINT,
4613 static_cast<LLVMArkInterface::EntrypointId>(RuntimeInterface::EntrypointId::RESOLVE_VIRTUAL_CALL_AOT));
4614 arkInterface->GetOrCreateRuntimeFunctionType(
4615 ctor->GetFunc()->getContext(), ctor->GetFunc()->getParent(), LLVMArkInterface::RuntimeCallType::ENTRYPOINT,
4616 static_cast<LLVMArkInterface::EntrypointId>(RuntimeInterface::EntrypointId::INTF_INLINE_CACHE));
4617
4618 auto methodPtr = ctor->GetGraph()->GetMethod();
4619 auto methodId = inst->GetCallMethodId();
4620 auto callee = ctor->GetGraph()->GetRuntime()->GetMethodById(methodPtr, methodId);
4621 ASSERT(callee != nullptr);
4622
4623 std::stringstream ssUniqName;
4624 ssUniqName << "f_" << std::hex << inst;
4625 auto uniqName = ssUniqName.str();
4626 auto methodName = arkInterface->GetUniqMethodName(callee) + "_" + uniqName;
4627 auto functionProto = ctor->GetFunctionTypeForCall(inst);
4628 auto func = CreateFunctionDeclaration(functionProto, methodName, ctor->GetFunc()->getParent());
4629
4630 func->addFnAttr("frame-pointer", "all");
4631 arkInterface->PutVirtualFunction(inst->GetCallMethod(), func);
4632 return func;
4633 }
4634
VisitCallVirtual(GraphVisitor * v,Inst * inst)4635 void LLVMIrConstructor::VisitCallVirtual(GraphVisitor *v, Inst *inst)
4636 {
4637 auto ctor = static_cast<LLVMIrConstructor *>(v);
4638 auto call = inst->CastToCallVirtual();
4639 if (call->IsInlined()) {
4640 return;
4641 }
4642 ASSERT_PRINT(ctor->GetGraph()->GetAotData()->GetUseCha(),
4643 std::string("GetUseCha must be 'true' but was 'false' for method = '") +
4644 ctor->GetGraph()->GetRuntime()->GetMethodFullName(ctor->GetGraph()->GetMethod()) + "'");
4645
4646 ASSERT(!ctor->GetGraph()->GetRuntime()->IsInterfaceMethod(call->GetCallMethod()));
4647 auto methodId = call->GetCallMethodId();
4648
4649 auto func = CreateDeclForVirtualCall(call, ctor, ctor->arkInterface_);
4650 auto args = ctor->GetArgumentsForCall(ctor->GetMethodArgument(), call);
4651 auto result = ctor->builder_.CreateCall(func, args, ctor->CreateSaveStateBundle(inst));
4652 result->addFnAttr(llvm::Attribute::get(result->getContext(), "original-method-id", std::to_string(methodId)));
4653 if (inst->GetType() != DataType::VOID) {
4654 ctor->ValueMapAdd(inst, result);
4655 }
4656 ctor->WrapArkCall(inst, result);
4657 }
4658
VisitResolveVirtual(GraphVisitor * v,Inst * inst)4659 void LLVMIrConstructor::VisitResolveVirtual(GraphVisitor *v, Inst *inst)
4660 {
4661 auto ctor = static_cast<LLVMIrConstructor *>(v);
4662 auto resolver = inst->CastToResolveVirtual();
4663
4664 llvm::Value *method = nullptr;
4665 if (resolver->GetCallMethod() == nullptr) {
4666 llvm::Value *thiz = ctor->GetInputValue(inst, 0);
4667 method = ctor->CreateResolveVirtualCallBuiltin(inst, thiz, resolver->GetCallMethodId());
4668 ASSERT(method->getType()->isPointerTy());
4669 } else {
4670 ASSERT(ctor->GetGraph()->GetRuntime()->IsInterfaceMethod(resolver->GetCallMethod()));
4671 method = CreateDeclForVirtualCall(resolver, ctor, ctor->arkInterface_);
4672 }
4673 ctor->ValueMapAdd(inst, method, false);
4674 }
4675
VisitCallResolvedVirtual(GraphVisitor * v,Inst * inst)4676 void LLVMIrConstructor::VisitCallResolvedVirtual(GraphVisitor *v, Inst *inst)
4677 {
4678 auto ctor = static_cast<LLVMIrConstructor *>(v);
4679 auto call = inst->CastToCallResolvedVirtual();
4680 if (call->IsInlined()) {
4681 return;
4682 }
4683 auto runtime = ctor->GetGraph()->GetRuntime();
4684 auto method = ctor->GetInputValue(inst, 0);
4685 auto args = ctor->GetArgumentsForCall(method, call, true);
4686
4687 llvm::CallInst *result = nullptr;
4688 if (call->GetCallMethod() == nullptr) {
4689 llvm::FunctionType *fType = ctor->GetFunctionTypeForCall(call);
4690
4691 auto offset = runtime->GetCompiledEntryPointOffset(ctor->GetGraph()->GetArch());
4692 auto entrypointPtr = ctor->builder_.CreateConstInBoundsGEP1_32(ctor->builder_.getInt8Ty(), method, offset);
4693 auto entrypoint = ctor->builder_.CreateLoad(ctor->builder_.getPtrTy(), entrypointPtr);
4694 result = ctor->builder_.CreateCall(fType, entrypoint, args, ctor->CreateSaveStateBundle(inst));
4695 } else {
4696 ASSERT(runtime->IsInterfaceMethod(call->GetCallMethod()));
4697 auto *func = llvm::cast<llvm::Function>(method);
4698 result = ctor->builder_.CreateCall(func, args, ctor->CreateSaveStateBundle(inst));
4699 auto methodId = call->GetCallMethodId();
4700 result->addFnAttr(llvm::Attribute::get(result->getContext(), "original-method-id", std::to_string(methodId)));
4701 }
4702 if (inst->GetType() != DataType::VOID) {
4703 ctor->ValueMapAdd(inst, result);
4704 }
4705 ctor->WrapArkCall(inst, result);
4706 }
4707
VisitAbs(GraphVisitor * v,Inst * inst)4708 void LLVMIrConstructor::VisitAbs(GraphVisitor *v, Inst *inst)
4709 {
4710 ASSERT(g_options.IsCompilerEncodeIntrinsics());
4711 auto ctor = static_cast<LLVMIrConstructor *>(v);
4712
4713 DataType::Type pandaType = inst->GetInputType(0);
4714 auto argument = ctor->GetInputValue(inst, 0);
4715 llvm::Value *result = nullptr;
4716 if (DataType::IsFloatType(pandaType)) {
4717 result = ctor->builder_.CreateUnaryIntrinsic(llvm::Intrinsic::fabs, argument, nullptr);
4718 } else if (IsInteger(pandaType)) {
4719 result = ctor->builder_.CreateBinaryIntrinsic(llvm::Intrinsic::abs, argument, ctor->builder_.getFalse());
4720 } else {
4721 ASSERT_DO(false, (std::cerr << "Abs is not supported for type " << DataType::ToString(pandaType) << std::endl));
4722 UNREACHABLE();
4723 }
4724 ASSERT(result != nullptr);
4725 ctor->ValueMapAdd(inst, result);
4726 }
4727
VisitIntrinsic(GraphVisitor * v,Inst * inst)4728 void LLVMIrConstructor::VisitIntrinsic(GraphVisitor *v, Inst *inst)
4729 {
4730 auto ctor = static_cast<LLVMIrConstructor *>(v);
4731 auto entryId = inst->CastToIntrinsic()->GetIntrinsicId();
4732
4733 // Some Ark intrinsics are lowered into code or LLVM intrinsics, the IntrinsicsLowering pass
4734 // makes final desicion for them to be lowered into code or calling Ark entrypoint.
4735 if (g_options.IsCompilerEncodeIntrinsics()) {
4736 bool lowered = ctor->TryEmitIntrinsic(inst, entryId);
4737 if (lowered) {
4738 return;
4739 }
4740 }
4741 // Create call otherwise
4742 auto result = ctor->CreateIntrinsicCall(inst);
4743 if (inst->GetType() != DataType::VOID) {
4744 ctor->ValueMapAdd(inst, result);
4745 }
4746 }
4747
VisitMonitor(GraphVisitor * v,Inst * inst)4748 void LLVMIrConstructor::VisitMonitor(GraphVisitor *v, Inst *inst)
4749 {
4750 auto ctor = static_cast<LLVMIrConstructor *>(v);
4751 MonitorInst *monitor = inst->CastToMonitor();
4752 auto object = ctor->GetInputValue(inst, 0);
4753 auto eid = monitor->IsEntry() ? RuntimeInterface::EntrypointId::MONITOR_ENTER_FAST_PATH
4754 : RuntimeInterface::EntrypointId::MONITOR_EXIT_FAST_PATH;
4755 auto call = ctor->CreateEntrypointCall(eid, inst, {object});
4756 ASSERT(call->getCallingConv() == llvm::CallingConv::C);
4757 call->setCallingConv(llvm::CallingConv::ArkFast1);
4758 }
4759
VisitSqrt(GraphVisitor * v,Inst * inst)4760 void LLVMIrConstructor::VisitSqrt(GraphVisitor *v, Inst *inst)
4761 {
4762 ASSERT(g_options.IsCompilerEncodeIntrinsics());
4763 auto ctor = static_cast<LLVMIrConstructor *>(v);
4764 auto argument = ctor->GetInputValue(inst, 0);
4765 auto result = ctor->builder_.CreateUnaryIntrinsic(llvm::Intrinsic::sqrt, argument, nullptr);
4766 ctor->ValueMapAdd(inst, result);
4767 }
4768
VisitInitClass(GraphVisitor * v,Inst * inst)4769 void LLVMIrConstructor::VisitInitClass(GraphVisitor *v, Inst *inst)
4770 {
4771 auto ctor = static_cast<LLVMIrConstructor *>(v);
4772 auto classId = inst->CastToInitClass()->GetTypeId();
4773
4774 auto constexpr INITIALIZED = true;
4775 ctor->CreateLoadClassById(inst, classId, INITIALIZED);
4776 }
4777
VisitLoadClass(GraphVisitor * v,Inst * inst)4778 void LLVMIrConstructor::VisitLoadClass(GraphVisitor *v, Inst *inst)
4779 {
4780 auto ctor = static_cast<LLVMIrConstructor *>(v);
4781 auto classId = inst->CastToLoadClass()->GetTypeId();
4782
4783 auto constexpr INITIALIZED = true;
4784 auto clsPtr = ctor->CreateLoadClassById(inst, classId, !INITIALIZED);
4785 ctor->ValueMapAdd(inst, clsPtr);
4786 }
4787
VisitLoadAndInitClass(GraphVisitor * v,Inst * inst)4788 void LLVMIrConstructor::VisitLoadAndInitClass(GraphVisitor *v, Inst *inst)
4789 {
4790 auto ctor = static_cast<LLVMIrConstructor *>(v);
4791 auto classId = inst->CastToLoadAndInitClass()->GetTypeId();
4792
4793 auto constexpr INITIALIZED = true;
4794 auto clsPtr = ctor->CreateLoadClassById(inst, classId, INITIALIZED);
4795 ctor->ValueMapAdd(inst, clsPtr);
4796 }
4797
VisitUnresolvedLoadAndInitClass(GraphVisitor * v,Inst * inst)4798 void LLVMIrConstructor::VisitUnresolvedLoadAndInitClass(GraphVisitor *v, Inst *inst)
4799 {
4800 auto ctor = static_cast<LLVMIrConstructor *>(v);
4801 auto classId = inst->CastToUnresolvedLoadAndInitClass()->GetTypeId();
4802
4803 auto constexpr INITIALIZED = true;
4804 auto clsPtr = ctor->CreateLoadClassById(inst, classId, INITIALIZED);
4805 ctor->ValueMapAdd(inst, clsPtr);
4806 }
4807
VisitLoadStatic(GraphVisitor * v,Inst * inst)4808 void LLVMIrConstructor::VisitLoadStatic(GraphVisitor *v, Inst *inst)
4809 {
4810 auto ctor = static_cast<LLVMIrConstructor *>(v);
4811 auto klass = ctor->GetInputValue(inst, 0);
4812 ASSERT_TYPE(klass, ctor->builder_.getPtrTy());
4813
4814 auto offset = ctor->GetGraph()->GetRuntime()->GetFieldOffset(inst->CastToLoadStatic()->GetObjField());
4815 auto fieldPtr = ctor->builder_.CreateConstInBoundsGEP1_32(ctor->builder_.getInt8Ty(), klass, offset);
4816
4817 auto n = ctor->CreateLoadWithOrdering(inst, fieldPtr, ToAtomicOrdering(inst->CastToLoadStatic()->GetVolatile()));
4818 ctor->ValueMapAdd(inst, n);
4819 }
4820
VisitStoreStatic(GraphVisitor * v,Inst * inst)4821 void LLVMIrConstructor::VisitStoreStatic(GraphVisitor *v, Inst *inst)
4822 {
4823 auto ctor = static_cast<LLVMIrConstructor *>(v);
4824 auto klass = ctor->GetInputValue(inst, 0);
4825 ASSERT_TYPE(klass, ctor->builder_.getPtrTy());
4826 auto value = ctor->GetInputValue(inst, 1);
4827
4828 auto runtime = ctor->GetGraph()->GetRuntime();
4829 auto offset = runtime->GetFieldOffset(inst->CastToStoreStatic()->GetObjField());
4830 auto fieldPtr = ctor->builder_.CreateConstInBoundsGEP1_32(ctor->builder_.getInt8Ty(), klass, offset);
4831
4832 // Pre
4833 if (inst->CastToStoreStatic()->GetNeedBarrier()) {
4834 ctor->CreatePreWRB(inst, fieldPtr);
4835 }
4836 // Write
4837 ctor->CreateStoreWithOrdering(value, fieldPtr, ToAtomicOrdering(inst->CastToStoreStatic()->GetVolatile()));
4838 // Post
4839 if (inst->CastToStoreStatic()->GetNeedBarrier()) {
4840 auto barrierType = runtime->GetPostType();
4841 if (barrierType == mem::BarrierType::POST_INTERREGION_BARRIER) {
4842 ctor->CreatePostWRB(inst, klass, ctor->builder_.getInt32(offset), value);
4843 } else {
4844 auto managed = ctor->CreateLoadManagedClassFromClass(klass);
4845 ctor->CreatePostWRB(inst, managed, ctor->builder_.getInt32(0), value);
4846 }
4847 }
4848 }
4849
VisitUnresolvedStoreStatic(GraphVisitor * v,Inst * inst)4850 void LLVMIrConstructor::VisitUnresolvedStoreStatic(GraphVisitor *v, Inst *inst)
4851 {
4852 auto ctor = static_cast<LLVMIrConstructor *>(v);
4853 auto unresolvedStore = inst->CastToUnresolvedStoreStatic();
4854
4855 ASSERT(unresolvedStore->GetNeedBarrier());
4856 ASSERT(DataType::IsReference(inst->GetType()));
4857
4858 auto typeId = ctor->builder_.getInt32(unresolvedStore->GetTypeId());
4859 auto value = ctor->GetInputValue(inst, 0);
4860
4861 auto entrypoint = RuntimeInterface::EntrypointId::UNRESOLVED_STORE_STATIC_BARRIERED;
4862 ctor->CreateEntrypointCall(entrypoint, inst, {ctor->GetMethodArgument(), typeId, value});
4863 }
4864
VisitLoadConstArray(GraphVisitor * v,Inst * inst)4865 void LLVMIrConstructor::VisitLoadConstArray(GraphVisitor *v, Inst *inst)
4866 {
4867 auto ctor = static_cast<LLVMIrConstructor *>(v);
4868 auto arrayType = inst->CastToLoadConstArray()->GetTypeId();
4869
4870 llvm::Value *result = ctor->CreateEntrypointCall(RuntimeInterface::EntrypointId::RESOLVE_LITERAL_ARRAY, inst,
4871 {ctor->GetMethodArgument(), ctor->builder_.getInt32(arrayType)});
4872 ctor->ValueMapAdd(inst, result);
4873 }
4874
VisitFillConstArray(GraphVisitor * v,Inst * inst)4875 void LLVMIrConstructor::VisitFillConstArray(GraphVisitor *v, Inst *inst)
4876 {
4877 ASSERT(!DataType::IsReference(inst->GetType()));
4878 auto ctor = static_cast<LLVMIrConstructor *>(v);
4879 auto &builder = ctor->builder_;
4880
4881 auto runtime = ctor->GetGraph()->GetRuntime();
4882 auto arrayType = inst->CastToFillConstArray()->GetTypeId();
4883 auto arch = ctor->GetGraph()->GetArch();
4884 auto src = ctor->GetInputValue(inst, 0);
4885 auto offset = runtime->GetArrayDataOffset(arch);
4886 auto arraySize = inst->CastToFillConstArray()->GetImm() << DataType::ShiftByType(inst->GetType(), arch);
4887 auto arrayPtr = builder.CreateConstInBoundsGEP1_64(builder.getInt8Ty(), src, offset);
4888
4889 ASSERT(arraySize != 0);
4890
4891 auto arrOffset = runtime->GetOffsetToConstArrayData(inst->CastToFillConstArray()->GetMethod(), arrayType);
4892 auto pfOffset = runtime->GetPandaFileOffset(arch);
4893 auto fileOffset = runtime->GetBinaryFileBaseOffset(arch);
4894
4895 auto pfPtrPtr = builder.CreateConstInBoundsGEP1_64(builder.getInt8Ty(), ctor->GetMethodArgument(), pfOffset);
4896 auto pfPtr = builder.CreateLoad(builder.getPtrTy(), pfPtrPtr);
4897 auto filePtrPtr = builder.CreateConstInBoundsGEP1_64(builder.getInt8Ty(), pfPtr, fileOffset);
4898 auto filePtr = builder.CreateLoad(builder.getPtrTy(), filePtrPtr);
4899 auto constArrPtr = builder.CreateConstInBoundsGEP1_64(builder.getInt8Ty(), filePtr, arrOffset);
4900
4901 auto align = llvm::MaybeAlign(0);
4902 /**
4903 * LLVM AOT may replace `@llvm.memcpy.inline` with call to ark's `LIB_CALL_MEM_COPY`, see `MustLowerMemCpy` in
4904 * libllvmbackend/llvm_ark_interface.cpp.
4905 */
4906 builder.CreateMemCpyInline(arrayPtr, align, constArrPtr, align, builder.getInt64(arraySize));
4907 }
4908
VisitIsInstance(GraphVisitor * v,Inst * inst)4909 void LLVMIrConstructor::VisitIsInstance(GraphVisitor *v, Inst *inst)
4910 {
4911 auto ctor = static_cast<LLVMIrConstructor *>(v);
4912 auto isInstance = inst->CastToIsInstance();
4913 auto klassType = isInstance->GetClassType();
4914 auto object = ctor->GetInputValue(inst, 0);
4915 llvm::Value *result;
4916 if (klassType == ClassType::UNRESOLVED_CLASS) {
4917 result = ctor->CreateIsInstanceEntrypointCall(inst);
4918 } else {
4919 auto &ctx = ctor->func_->getContext();
4920 auto preBb = ctor->GetCurrentBasicBlock();
4921 auto contBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "isinstance_cont"), ctor->func_);
4922
4923 if (!inst->CastToIsInstance()->GetOmitNullCheck()) {
4924 auto notnullBb =
4925 llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "isinstance_notnull"), ctor->func_);
4926 auto isNullObj = ctor->builder_.CreateIsNull(object);
4927 ctor->builder_.CreateCondBr(isNullObj, contBb, notnullBb);
4928 ctor->SetCurrentBasicBlock(notnullBb);
4929 }
4930
4931 llvm::Value *innerResult = nullptr;
4932 auto klassId = ctor->GetInputValue(inst, 1);
4933 auto klassObj = CreateLoadClassFromObject(object, &ctor->builder_, ctor->arkInterface_);
4934 auto notnullPostBb = ctor->GetCurrentBasicBlock();
4935 auto cmp = ctor->builder_.CreateICmpEQ(klassId, klassObj);
4936 if (klassType == ClassType::FINAL_CLASS) {
4937 innerResult = ctor->builder_.CreateZExt(cmp, ctor->builder_.getInt8Ty());
4938 } else {
4939 auto innerBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "isinstance_inner"), ctor->func_);
4940 ctor->builder_.CreateCondBr(cmp, contBb, innerBb);
4941 ctor->SetCurrentBasicBlock(innerBb);
4942 innerResult = ctor->CreateIsInstanceInnerBlock(inst, klassObj, klassId);
4943 }
4944 auto incomingBlock = ctor->GetCurrentBasicBlock();
4945 ctor->builder_.CreateBr(contBb);
4946
4947 ctor->SetCurrentBasicBlock(contBb);
4948 unsigned amount = 1 + (preBb == notnullPostBb ? 0 : 1) + (notnullPostBb == incomingBlock ? 0 : 1);
4949 auto resultPhi = ctor->builder_.CreatePHI(ctor->builder_.getInt8Ty(), amount);
4950 if (preBb != notnullPostBb) {
4951 resultPhi->addIncoming(ctor->builder_.getInt8(0), preBb);
4952 }
4953 if (notnullPostBb != incomingBlock) {
4954 resultPhi->addIncoming(ctor->builder_.getInt8(1), notnullPostBb);
4955 }
4956 resultPhi->addIncoming(innerResult, incomingBlock);
4957 result = resultPhi;
4958 }
4959
4960 ctor->ValueMapAdd(inst, result);
4961 }
4962
VisitCheckCast(GraphVisitor * v,Inst * inst)4963 void LLVMIrConstructor::VisitCheckCast(GraphVisitor *v, Inst *inst)
4964 {
4965 auto ctor = static_cast<LLVMIrConstructor *>(v);
4966 auto checkCast = inst->CastToCheckCast();
4967 auto klassType = checkCast->GetClassType();
4968 auto src = ctor->GetInputValue(inst, 0);
4969
4970 auto &ctx = ctor->func_->getContext();
4971 auto outBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "checkcast_out"), ctor->func_);
4972
4973 // Nullptr check can be omitted sometimes
4974 if (!inst->CastToCheckCast()->GetOmitNullCheck()) {
4975 auto contBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "checkcast_cont"), ctor->func_);
4976 auto isNullptr = ctor->builder_.CreateIsNull(src);
4977 ctor->builder_.CreateCondBr(isNullptr, outBb, contBb);
4978 ctor->SetCurrentBasicBlock(contBb);
4979 }
4980
4981 if (klassType == ClassType::UNRESOLVED_CLASS ||
4982 (klassType == ClassType::INTERFACE_CLASS && inst->CanDeoptimize())) {
4983 ctor->CreateCheckCastEntrypointCall(inst);
4984 } else if (klassType == ClassType::INTERFACE_CLASS) {
4985 ASSERT(!inst->CanDeoptimize());
4986 auto entrypoint = RuntimeInterface::EntrypointId::CHECK_CAST_INTERFACE;
4987 ctor->CreateFastPathCall(inst, entrypoint, {src, ctor->GetInputValue(inst, 1)});
4988 } else {
4989 auto klassId = ctor->GetInputValue(inst, 1);
4990 auto klassObj = CreateLoadClassFromObject(src, &ctor->builder_, ctor->arkInterface_);
4991 if (klassType == ClassType::FINAL_CLASS) {
4992 auto cmpNe = ctor->builder_.CreateICmpNE(klassId, klassObj);
4993 auto exception = RuntimeInterface::EntrypointId::CLASS_CAST_EXCEPTION;
4994 ctor->CreateDeoptimizationBranch(inst, cmpNe, exception, {klassId, src});
4995 } else {
4996 auto cmpEq = ctor->builder_.CreateICmpEQ(klassId, klassObj);
4997 auto innerBb = llvm::BasicBlock::Create(ctx, CreateBasicBlockName(inst, "checkcast_inner"), ctor->func_);
4998 ctor->builder_.CreateCondBr(cmpEq, outBb, innerBb);
4999 ctor->SetCurrentBasicBlock(innerBb);
5000 ctor->CreateCheckCastInner(inst, klassObj, klassId);
5001 }
5002 }
5003 ctor->builder_.CreateBr(outBb);
5004 ctor->SetCurrentBasicBlock(outBb);
5005 }
5006
VisitLoadType(GraphVisitor * v,Inst * inst)5007 void LLVMIrConstructor::VisitLoadType(GraphVisitor *v, Inst *inst)
5008 {
5009 auto ctor = static_cast<LLVMIrConstructor *>(v);
5010
5011 auto constexpr INITIALIZED = true;
5012 auto klass = ctor->CreateLoadClassById(inst, inst->CastToLoadType()->GetTypeId(), !INITIALIZED);
5013 auto result = ctor->CreateLoadManagedClassFromClass(klass);
5014 ctor->ValueMapAdd(inst, result);
5015 }
5016
VisitUnresolvedLoadType(GraphVisitor * v,Inst * inst)5017 void LLVMIrConstructor::VisitUnresolvedLoadType(GraphVisitor *v, Inst *inst)
5018 {
5019 auto ctor = static_cast<LLVMIrConstructor *>(v);
5020
5021 auto constexpr INITIALIZED = true;
5022 auto klass = ctor->CreateLoadClassById(inst, inst->CastToUnresolvedLoadType()->GetTypeId(), !INITIALIZED);
5023 auto result = ctor->CreateLoadManagedClassFromClass(klass);
5024 ctor->ValueMapAdd(inst, result);
5025 }
5026
VisitGetInstanceClass(GraphVisitor * v,Inst * inst)5027 void LLVMIrConstructor::VisitGetInstanceClass(GraphVisitor *v, Inst *inst)
5028 {
5029 auto ctor = static_cast<LLVMIrConstructor *>(v);
5030
5031 auto object = ctor->GetInputValue(inst, 0);
5032 auto klass = CreateLoadClassFromObject(object, &ctor->builder_, ctor->arkInterface_);
5033 ctor->ValueMapAdd(inst, klass);
5034 }
5035
VisitThrow(GraphVisitor * v,Inst * inst)5036 void LLVMIrConstructor::VisitThrow(GraphVisitor *v, Inst *inst)
5037 {
5038 auto ctor = static_cast<LLVMIrConstructor *>(v);
5039 auto obj = ctor->GetInputValue(inst, 0);
5040
5041 auto exception = RuntimeInterface::EntrypointId::THROW_EXCEPTION;
5042 ctor->CreateEntrypointCall(exception, inst, {obj});
5043 ctor->builder_.CreateUnreachable();
5044 }
5045
VisitCatchPhi(GraphVisitor * v,Inst * inst)5046 void LLVMIrConstructor::VisitCatchPhi([[maybe_unused]] GraphVisitor *v, Inst *inst)
5047 {
5048 UnexpectedLowering(inst);
5049 }
5050
VisitLoadRuntimeClass(GraphVisitor * v,Inst * inst)5051 void LLVMIrConstructor::VisitLoadRuntimeClass(GraphVisitor *v, Inst *inst)
5052 {
5053 auto ctor = static_cast<LLVMIrConstructor *>(v);
5054
5055 auto offset = ctor->GetGraph()->GetRuntime()->GetTlsPromiseClassPointerOffset(ctor->GetGraph()->GetArch());
5056 auto result = llvmbackend::runtime_calls::LoadTLSValue(&ctor->builder_, ctor->arkInterface_, offset,
5057 ctor->builder_.getPtrTy());
5058 ctor->ValueMapAdd(inst, result);
5059 }
5060
VisitLoadUndefined(GraphVisitor * v,Inst * inst)5061 void LLVMIrConstructor::VisitLoadUndefined(GraphVisitor *v, Inst *inst)
5062 {
5063 auto ctor = static_cast<LLVMIrConstructor *>(v);
5064
5065 auto offset = ctor->GetGraph()->GetRuntime()->GetTlsUndefinedObjectOffset(ctor->GetGraph()->GetArch());
5066 auto result = llvmbackend::runtime_calls::LoadTLSValue(&ctor->builder_, ctor->arkInterface_, offset,
5067 ctor->builder_.getPtrTy(LLVMArkInterface::GC_ADDR_SPACE));
5068 ctor->ValueMapAdd(inst, result);
5069 }
5070
VisitCallLaunchVirtual(GraphVisitor * v,Inst * inst)5071 void LLVMIrConstructor::VisitCallLaunchVirtual(GraphVisitor *v, Inst *inst)
5072 {
5073 auto ctor = static_cast<LLVMIrConstructor *>(v);
5074 ctor->CreateLaunchCall(inst->CastToCallLaunchVirtual());
5075 }
5076
VisitCallResolvedLaunchStatic(GraphVisitor * v,Inst * inst)5077 void LLVMIrConstructor::VisitCallResolvedLaunchStatic(GraphVisitor *v, Inst *inst)
5078 {
5079 auto ctor = static_cast<LLVMIrConstructor *>(v);
5080 ctor->CreateLaunchCall(inst->CastToCallResolvedLaunchStatic());
5081 }
5082
VisitCallResolvedLaunchVirtual(GraphVisitor * v,Inst * inst)5083 void LLVMIrConstructor::VisitCallResolvedLaunchVirtual(GraphVisitor *v, Inst *inst)
5084 {
5085 auto ctor = static_cast<LLVMIrConstructor *>(v);
5086 ctor->CreateLaunchCall(inst->CastToCallResolvedLaunchVirtual());
5087 }
5088
VisitLoadImmediate(GraphVisitor * v,Inst * inst)5089 void LLVMIrConstructor::VisitLoadImmediate(GraphVisitor *v, Inst *inst)
5090 {
5091 auto ctor = static_cast<LLVMIrConstructor *>(v);
5092 auto loadImm = inst->CastToLoadImmediate();
5093 ASSERT_DO(loadImm->IsTlsOffset(), (std::cerr << "Unsupported llvm lowering for \n", inst->Dump(&std::cerr, true)));
5094 ASSERT(inst->GetType() == DataType::POINTER);
5095 auto result = llvmbackend::runtime_calls::LoadTLSValue(&ctor->builder_, ctor->arkInterface_,
5096 loadImm->GetTlsOffset(), ctor->builder_.getPtrTy());
5097 ctor->ValueMapAdd(inst, result);
5098 }
5099
VisitDefault(Inst * inst)5100 void LLVMIrConstructor::VisitDefault([[maybe_unused]] Inst *inst)
5101 {
5102 ASSERT_DO(false, (std::cerr << "Unsupported llvm lowering for \n", inst->Dump(&std::cerr, true)));
5103 UNREACHABLE();
5104 }
5105
LLVMIrConstructor(Graph * graph,llvm::Module * module,llvm::LLVMContext * context,LLVMArkInterface * arkInterface,const std::unique_ptr<DebugDataBuilder> & debugData)5106 LLVMIrConstructor::LLVMIrConstructor(Graph *graph, llvm::Module *module, llvm::LLVMContext *context,
5107 LLVMArkInterface *arkInterface, const std::unique_ptr<DebugDataBuilder> &debugData)
5108 : graph_(graph),
5109 builder_(llvm::IRBuilder<>(*context)),
5110 inputMap_(graph->GetLocalAllocator()->Adapter()),
5111 blockTailMap_(graph->GetLocalAllocator()->Adapter()),
5112 blockHeadMap_(graph->GetLocalAllocator()->Adapter()),
5113 arkInterface_(arkInterface),
5114 debugData_(debugData),
5115 cc_(graph->GetLocalAllocator()->Adapter()),
5116 ccValues_(graph->GetLocalAllocator()->Adapter())
5117 {
5118 llvm::CallingConv::ID callingConv = llvm::CallingConv::C;
5119 // Assign regmaps
5120 if (graph->GetMode().IsInterpreter()) {
5121 if (graph->GetArch() == Arch::AARCH64) {
5122 cc_.assign({AARCH64_PC, AARCH64_ACC, AARCH64_ACC_TAG, AARCH64_FP, AARCH64_DISPATCH, AARCH64_MOFFSET,
5123 AARCH64_METHOD_PTR, GetThreadReg(Arch::AARCH64)});
5124 } else if (graph->GetArch() == Arch::X86_64) {
5125 cc_.assign({X86_64_PC, X86_64_ACC, X86_64_ACC_TAG, X86_64_FP, X86_64_DISPATCH, GetThreadReg(Arch::X86_64),
5126 X86_64_REAL_FP});
5127 } else {
5128 LLVM_LOG(FATAL, IR) << "Unsupported architecture for arkintcc";
5129 }
5130 callingConv = llvm::CallingConv::ArkInt;
5131 } else if (graph->GetMode().IsFastPath()) {
5132 ASSERT(graph->GetArch() == Arch::AARCH64);
5133 for (size_t i = 0; i < graph->GetRuntime()->GetMethodTotalArgumentsCount(graph->GetMethod()); i++) {
5134 cc_.push_back(i);
5135 }
5136 // Get calling convention excluding thread and frame registers
5137 callingConv = GetFastPathCallingConv(cc_.size());
5138 cc_.push_back(GetThreadReg(Arch::AARCH64));
5139 cc_.push_back(AARCH64_REAL_FP);
5140 }
5141 ccValues_.assign(cc_.size(), nullptr);
5142
5143 // Create function
5144 auto funcProto = GetEntryFunctionType();
5145 auto methodName = arkInterface_->GetUniqMethodName(graph_->GetMethod());
5146 func_ = CreateFunctionDeclaration(funcProto, methodName, module);
5147 ASSERT(func_->getCallingConv() == llvm::CallingConv::C);
5148 func_->setCallingConv(callingConv);
5149
5150 // Scenario of code generation for FastPath having zero arguments and return value is not tested
5151 ASSERT(callingConv != llvm::CallingConv::ArkFast0 || func_->getReturnType()->isVoidTy());
5152
5153 if (graph->SupportManagedCode()) {
5154 func_->setGC(std::string {llvmbackend::LLVMArkInterface::GC_STRATEGY});
5155 }
5156
5157 auto klassId = graph_->GetRuntime()->GetClassIdForMethod(graph_->GetMethod());
5158 auto klassIdMd = llvm::ConstantAsMetadata::get(builder_.getInt32(klassId));
5159 func_->addMetadata(llvmbackend::LLVMArkInterface::FUNCTION_MD_CLASS_ID, *llvm::MDNode::get(*context, {klassIdMd}));
5160
5161 if (!arkInterface_->IsIrtocMode()) {
5162 func_->addMetadata("use-ark-frame", *llvm::MDNode::get(*context, {}));
5163 }
5164 }
5165
BuildIr(bool preventInlining)5166 bool LLVMIrConstructor::BuildIr(bool preventInlining)
5167 {
5168 LLVM_LOG(DEBUG, IR) << "Building IR for LLVM";
5169
5170 // Set Argument Names
5171 // Special arguments
5172 auto it = func_->arg_begin();
5173 if (graph_->SupportManagedCode()) {
5174 (it++)->setName("method");
5175 }
5176 // Actual arguments
5177 auto idx = 0;
5178 while (it != func_->arg_end()) {
5179 std::stringstream name;
5180 name << "a" << idx++;
5181 (it++)->setName(name.str());
5182 }
5183
5184 auto method = graph_->GetMethod();
5185 auto runtime = graph_->GetRuntime();
5186 arkInterface_->RememberFunctionOrigin(func_, method);
5187 func_->addFnAttr(ark::llvmbackend::LLVMArkInterface::SOURCE_LANG_ATTR,
5188 std::to_string(static_cast<uint8_t>(runtime->GetMethodSourceLanguage(method))));
5189
5190 if (!graph_->GetMode().IsFastPath()) {
5191 debugData_->BeginSubprogram(func_, runtime->GetFullFileName(method), runtime->GetMethodId(method));
5192 } else {
5193 func_->addFnAttr(llvm::Attribute::NoInline);
5194 }
5195
5196 auto normalMarkerHolder = MarkerHolder(graph_);
5197 auto normal = normalMarkerHolder.GetMarker();
5198
5199 graph_->GetStartBlock()->SetMarker(normal);
5200 MarkNormalBlocksRecursive(graph_->GetStartBlock(), normal);
5201
5202 // First step - create blocks, leaving LLVM EntryBlock untouched
5203 BuildBasicBlocks(normal);
5204 InitializeEntryBlock(preventInlining);
5205
5206 // Second step - visit all instructions, including StartBlock, but not filling PHI inputs
5207 BuildInstructions(normal);
5208
5209 // Third step - fill the PHIs inputs
5210 for (auto block : graph_->GetBlocksRPO()) {
5211 FillPhiInputs(block, normal);
5212 }
5213
5214 if (!graph_->GetMode().IsFastPath()) {
5215 debugData_->EndSubprogram(func_);
5216 }
5217 if (!arkInterface_->IsIrtocMode()) {
5218 func_->addFnAttr("frame-pointer", "all");
5219 }
5220 #ifndef NDEBUG
5221 // Only for tests
5222 BreakIrIfNecessary();
5223 #endif
5224 // verifyFunction returns false if there are no errors. But we return true if everything is ok.
5225 auto verified = !verifyFunction(*func_, &llvm::errs());
5226 if (!verified) {
5227 func_->print(llvm::errs());
5228 }
5229 return verified;
5230 }
5231
InsertArkFrameInfo(llvm::Module * module,Arch arch)5232 void LLVMIrConstructor::InsertArkFrameInfo(llvm::Module *module, Arch arch)
5233 {
5234 constexpr std::string_view ARK_CALLER_SLOTS_MD = "ark.frame.info";
5235 ASSERT(module->getNamedMetadata(ARK_CALLER_SLOTS_MD) == nullptr);
5236 auto arkFrameInfoMd = module->getOrInsertNamedMetadata(ARK_CALLER_SLOTS_MD);
5237 auto builder = llvm::IRBuilder<>(module->getContext());
5238
5239 // The fist param is a difference between Ark's fp and the start of LLVM frame.
5240 auto md = llvm::ConstantAsMetadata::get(builder.getInt32(0U));
5241 arkFrameInfoMd->addOperand(llvm::MDNode::get(module->getContext(), {md}));
5242
5243 // The second param contains offsets of caller-saved registers inside the ark's frame
5244 std::vector<size_t> callParamsRegs;
5245 switch (arch) {
5246 case Arch::AARCH64: {
5247 auto src = ArchCallingConvention<Arch::AARCH64>::Target::CALL_PARAMS_REGS;
5248 callParamsRegs = std::vector<size_t>(src.begin(), src.end());
5249 break;
5250 }
5251 case Arch::X86_64: {
5252 auto src = ArchCallingConvention<Arch::X86_64>::Target::CALL_PARAMS_REGS;
5253 callParamsRegs = std::vector<size_t>(src.begin(), src.end());
5254 break;
5255 }
5256 default:
5257 UNREACHABLE();
5258 }
5259
5260 CFrameLayout frameLayout(arch, 0);
5261 const auto callerRegsSlotStart = frameLayout.GetCallerFirstSlot(false);
5262 const auto callerRegsCount = frameLayout.GetCallerRegistersCount(false);
5263 std::vector<llvm::Metadata *> argOffsets;
5264 for (auto paramRegId : callParamsRegs) {
5265 int slot = callerRegsSlotStart + (callerRegsCount - 1 - paramRegId);
5266 slot += frameLayout.GetStackStartSlot();
5267 constexpr auto FP_ORIGIN = CFrameLayout::OffsetOrigin::FP;
5268 constexpr auto OFFSET_IN_BYTES = CFrameLayout::OffsetUnit::BYTES;
5269 auto offset = -frameLayout.GetOffset<FP_ORIGIN, OFFSET_IN_BYTES>(slot);
5270 ASSERT(std::numeric_limits<int32_t>::min() <= offset);
5271 ASSERT(offset <= std::numeric_limits<int32_t>::max());
5272 if (arch == Arch::AARCH64) {
5273 offset -= frameLayout.GetSlotSize() * 2U;
5274 }
5275 argOffsets.push_back(llvm::ConstantAsMetadata::get(builder.getInt32(offset)));
5276 }
5277 arkFrameInfoMd->addOperand(llvm::MDNode::get(module->getContext(), argOffsets));
5278
5279 // The third param is actual frame size
5280 auto val = frameLayout.GetFrameSize<CFrameLayout::OffsetUnit::BYTES>();
5281 // LLVM will store LR & FP
5282 if (arch == Arch::AARCH64) {
5283 val -= frameLayout.GetSlotSize() * 2U;
5284 }
5285 auto vmd = llvm::ConstantAsMetadata::get(builder.getInt32(val));
5286 arkFrameInfoMd->addOperand(llvm::MDNode::get(module->getContext(), {vmd}));
5287 }
5288
ProvideSafepointPoll(llvm::Module * module,LLVMArkInterface * arkInterface,Arch arch)5289 void LLVMIrConstructor::ProvideSafepointPoll(llvm::Module *module, LLVMArkInterface *arkInterface, Arch arch)
5290 {
5291 // Has been already provided
5292 ASSERT(module->getFunction(LLVMArkInterface::GC_SAFEPOINT_POLL_NAME) == nullptr);
5293 auto &ctx = module->getContext();
5294 auto builder = llvm::IRBuilder<>(ctx);
5295
5296 // Create a gc.safepoint_poll itself
5297 auto pollFtype = llvm::FunctionType::get(builder.getVoidTy(), false);
5298 auto poll = llvm::Function::Create(pollFtype, llvm::Function::ExternalLinkage,
5299 LLVMArkInterface::GC_SAFEPOINT_POLL_NAME, module);
5300 poll->setDoesNotThrow();
5301
5302 // Creating a body
5303 auto entry = llvm::BasicBlock::Create(ctx, "bb", poll);
5304 builder.SetInsertPoint(entry);
5305
5306 int64_t flagAddrOffset = arkInterface->GetRuntime()->GetFlagAddrOffset(arch);
5307 auto trigger =
5308 llvmbackend::runtime_calls::LoadTLSValue(&builder, arkInterface, flagAddrOffset, builder.getInt16Ty());
5309 auto needSafepoint = builder.CreateICmpNE(trigger, builder.getInt16(0), "need_safepoint");
5310 // Create a ret instuction immediately to split bb right before it
5311 auto ret = builder.CreateRetVoid();
5312
5313 // Split into IF-THEN before RET and insert a safepoint call into THEN block
5314 auto weights =
5315 llvm::MDBuilder(ctx).createBranchWeights(llvmbackend::Metadata::BranchWeights::UNLIKELY_BRANCH_WEIGHT,
5316 llvmbackend::Metadata::BranchWeights::LIKELY_BRANCH_WEIGHT);
5317
5318 builder.SetInsertPoint(llvm::SplitBlockAndInsertIfThen(needSafepoint, ret, false, weights));
5319 builder.GetInsertBlock()->setName("safepoint");
5320 auto eid = RuntimeInterface::EntrypointId::SAFEPOINT;
5321 arkInterface->GetOrCreateRuntimeFunctionType(ctx, module, LLVMArkInterface::RuntimeCallType::ENTRYPOINT,
5322 static_cast<LLVMArkInterface::EntrypointId>(eid));
5323 auto threadReg = llvmbackend::runtime_calls::GetThreadRegValue(&builder, arkInterface);
5324 auto spCall = llvmbackend::runtime_calls::CreateEntrypointCallCommon(
5325 &builder, threadReg, arkInterface, static_cast<llvmbackend::runtime_calls::EntrypointId>(eid));
5326
5327 spCall->addFnAttr(llvm::Attribute::get(ctx, "safepoint"));
5328 }
5329
CheckGraph(Graph * graph)5330 std::string LLVMIrConstructor::CheckGraph(Graph *graph)
5331 {
5332 ASSERT(!graph->IsDynamicMethod());
5333 for (auto basicBlock : graph->GetBlocksRPO()) {
5334 for (auto inst : basicBlock->AllInsts()) {
5335 bool canCompile = LLVMIrConstructor::CanCompile(inst);
5336 if (!canCompile) {
5337 // It means we have one of the following cases:
5338 // * meet some brand-new opcode in Ark Compiler IR
5339 // * dynamic intrinsic call (in non-dynamic method!)
5340 // * not yet patched SLOW_PATH_ENTRY call in Irtoc code
5341 std::stringstream sstream;
5342 sstream << GetOpcodeString(inst->GetOpcode()) << " unexpected in LLVM lowering. Method = "
5343 << graph->GetRuntime()->GetMethodFullName(graph->GetMethod());
5344 std::string error = sstream.str();
5345 LLVM_LOG(ERROR, IR) << error;
5346 return error;
5347 }
5348 }
5349 }
5350 return "";
5351 }
5352
CanCompile(Inst * inst)5353 bool LLVMIrConstructor::CanCompile(Inst *inst)
5354 {
5355 if (inst->IsIntrinsic()) {
5356 auto iid = inst->CastToIntrinsic()->GetIntrinsicId();
5357 // We support only slowpaths where the second immediate is an external function
5358 if (iid == RuntimeInterface::IntrinsicId::INTRINSIC_SLOW_PATH_ENTRY) {
5359 return inst->CastToIntrinsic()->GetImms().size() > 1;
5360 }
5361 return CanCompileIntrinsic(iid);
5362 }
5363 // Check if we have method that can handle it
5364 // CC-OFFNXT(C_RULE_SWITCH_BRANCH_CHECKER) autogenerated code
5365 switch (inst->GetOpcode()) {
5366 default:
5367 UNREACHABLE_CONSTEXPR();
5368 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
5369 #define INST_DEF(OPCODE, ...) \
5370 case Opcode::OPCODE: { \
5371 /* CC-OFFNXT(G.PRE.05) function gen */ \
5372 return &LLVMIrConstructor::Visit##OPCODE != &GraphVisitor::Visit##OPCODE; \
5373 }
5374 OPCODE_LIST(INST_DEF)
5375 }
5376 #undef INST_DEF
5377 }
5378
5379 #ifndef NDEBUG
BreakIrIfNecessary()5380 void LLVMIrConstructor::BreakIrIfNecessary()
5381 {
5382 if (llvmbackend::g_options.GetLlvmBreakIrRegex().empty()) {
5383 return;
5384 }
5385
5386 std::regex regex {llvmbackend::g_options.GetLlvmBreakIrRegex()};
5387
5388 if (!std::regex_match(func_->getName().str(), regex)) {
5389 return;
5390 }
5391
5392 LLVM_LOG(DEBUG, IR) << "Breaking IR for '" << func_->getName().str() << "' because it matches regex = '"
5393 << llvmbackend::g_options.GetLlvmBreakIrRegex() << "'";
5394
5395 for (auto &basicBlock : *func_) {
5396 basicBlock.getTerminator()->eraseFromParent();
5397 }
5398 }
5399 #endif
5400
5401 #include "llvm_ir_constructor_gen.inl"
5402
5403 } // namespace ark::compiler
5404