1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IRBuilder class, which is used as a convenient way
10 // to create LLVM instructions with a consistent and simplified interface.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/IR/Constant.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/DerivedTypes.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/GlobalVariable.h"
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Operator.h"
27 #include "llvm/IR/Statepoint.h"
28 #include "llvm/IR/Type.h"
29 #include "llvm/IR/Value.h"
30 #include "llvm/Support/Casting.h"
31 #include "llvm/Support/MathExtras.h"
32 #include <cassert>
33 #include <cstdint>
34 #include <vector>
35
36 using namespace llvm;
37
38 /// CreateGlobalString - Make a new global variable with an initializer that
39 /// has array of i8 type filled in with the nul terminated string value
40 /// specified. If Name is specified, it is the name of the global variable
41 /// created.
CreateGlobalString(StringRef Str,const Twine & Name,unsigned AddressSpace)42 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
43 const Twine &Name,
44 unsigned AddressSpace) {
45 Constant *StrConstant = ConstantDataArray::getString(Context, Str);
46 Module &M = *BB->getParent()->getParent();
47 auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
48 GlobalValue::PrivateLinkage, StrConstant, Name,
49 nullptr, GlobalVariable::NotThreadLocal,
50 AddressSpace);
51 GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
52 GV->setAlignment(Align::None());
53 return GV;
54 }
55
getCurrentFunctionReturnType() const56 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
57 assert(BB && BB->getParent() && "No current function!");
58 return BB->getParent()->getReturnType();
59 }
60
getCastedInt8PtrValue(Value * Ptr)61 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
62 auto *PT = cast<PointerType>(Ptr->getType());
63 if (PT->getElementType()->isIntegerTy(8))
64 return Ptr;
65
66 // Otherwise, we need to insert a bitcast.
67 PT = getInt8PtrTy(PT->getAddressSpace());
68 BitCastInst *BCI = new BitCastInst(Ptr, PT, "");
69 BB->getInstList().insert(InsertPt, BCI);
70 SetInstDebugLocation(BCI);
71 return BCI;
72 }
73
createCallHelper(Function * Callee,ArrayRef<Value * > Ops,IRBuilderBase * Builder,const Twine & Name="",Instruction * FMFSource=nullptr)74 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
75 IRBuilderBase *Builder,
76 const Twine &Name = "",
77 Instruction *FMFSource = nullptr) {
78 CallInst *CI = CallInst::Create(Callee, Ops, Name);
79 if (FMFSource)
80 CI->copyFastMathFlags(FMFSource);
81 Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI);
82 Builder->SetInstDebugLocation(CI);
83 return CI;
84 }
85
createInvokeHelper(Function * Invokee,BasicBlock * NormalDest,BasicBlock * UnwindDest,ArrayRef<Value * > Ops,IRBuilderBase * Builder,const Twine & Name="")86 static InvokeInst *createInvokeHelper(Function *Invokee, BasicBlock *NormalDest,
87 BasicBlock *UnwindDest,
88 ArrayRef<Value *> Ops,
89 IRBuilderBase *Builder,
90 const Twine &Name = "") {
91 InvokeInst *II =
92 InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
93 Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
94 II);
95 Builder->SetInstDebugLocation(II);
96 return II;
97 }
98
CreateMemSet(Value * Ptr,Value * Val,Value * Size,MaybeAlign Align,bool isVolatile,MDNode * TBAATag,MDNode * ScopeTag,MDNode * NoAliasTag)99 CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
100 MaybeAlign Align, bool isVolatile,
101 MDNode *TBAATag, MDNode *ScopeTag,
102 MDNode *NoAliasTag) {
103 Ptr = getCastedInt8PtrValue(Ptr);
104 Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
105 Type *Tys[] = { Ptr->getType(), Size->getType() };
106 Module *M = BB->getParent()->getParent();
107 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
108
109 CallInst *CI = createCallHelper(TheFn, Ops, this);
110
111 if (Align)
112 cast<MemSetInst>(CI)->setDestAlignment(Align->value());
113
114 // Set the TBAA info if present.
115 if (TBAATag)
116 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
117
118 if (ScopeTag)
119 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
120
121 if (NoAliasTag)
122 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
123
124 return CI;
125 }
126
CreateElementUnorderedAtomicMemSet(Value * Ptr,Value * Val,Value * Size,Align Alignment,uint32_t ElementSize,MDNode * TBAATag,MDNode * ScopeTag,MDNode * NoAliasTag)127 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
128 Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
129 MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
130
131 Ptr = getCastedInt8PtrValue(Ptr);
132 Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
133 Type *Tys[] = {Ptr->getType(), Size->getType()};
134 Module *M = BB->getParent()->getParent();
135 Function *TheFn = Intrinsic::getDeclaration(
136 M, Intrinsic::memset_element_unordered_atomic, Tys);
137
138 CallInst *CI = createCallHelper(TheFn, Ops, this);
139
140 cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
141
142 // Set the TBAA info if present.
143 if (TBAATag)
144 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
145
146 if (ScopeTag)
147 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
148
149 if (NoAliasTag)
150 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
151
152 return CI;
153 }
154
CreateMemCpy(Value * Dst,unsigned DstAlign,Value * Src,unsigned SrcAlign,Value * Size,bool isVolatile,MDNode * TBAATag,MDNode * TBAAStructTag,MDNode * ScopeTag,MDNode * NoAliasTag)155 CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src,
156 unsigned SrcAlign, Value *Size,
157 bool isVolatile, MDNode *TBAATag,
158 MDNode *TBAAStructTag, MDNode *ScopeTag,
159 MDNode *NoAliasTag) {
160 return CreateMemCpy(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign),
161 Size, isVolatile, TBAATag, TBAAStructTag, ScopeTag,
162 NoAliasTag);
163 }
164
CreateMemCpy(Value * Dst,MaybeAlign DstAlign,Value * Src,MaybeAlign SrcAlign,Value * Size,bool isVolatile,MDNode * TBAATag,MDNode * TBAAStructTag,MDNode * ScopeTag,MDNode * NoAliasTag)165 CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, MaybeAlign DstAlign,
166 Value *Src, MaybeAlign SrcAlign,
167 Value *Size, bool isVolatile,
168 MDNode *TBAATag, MDNode *TBAAStructTag,
169 MDNode *ScopeTag, MDNode *NoAliasTag) {
170 Dst = getCastedInt8PtrValue(Dst);
171 Src = getCastedInt8PtrValue(Src);
172
173 Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
174 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
175 Module *M = BB->getParent()->getParent();
176 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
177
178 CallInst *CI = createCallHelper(TheFn, Ops, this);
179
180 auto* MCI = cast<MemCpyInst>(CI);
181 if (DstAlign)
182 MCI->setDestAlignment(*DstAlign);
183 if (SrcAlign)
184 MCI->setSourceAlignment(*SrcAlign);
185
186 // Set the TBAA info if present.
187 if (TBAATag)
188 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
189
190 // Set the TBAA Struct info if present.
191 if (TBAAStructTag)
192 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
193
194 if (ScopeTag)
195 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
196
197 if (NoAliasTag)
198 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
199
200 return CI;
201 }
202
CreateElementUnorderedAtomicMemCpy(Value * Dst,unsigned DstAlign,Value * Src,unsigned SrcAlign,Value * Size,uint32_t ElementSize,MDNode * TBAATag,MDNode * TBAAStructTag,MDNode * ScopeTag,MDNode * NoAliasTag)203 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
204 Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
205 uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
206 MDNode *ScopeTag, MDNode *NoAliasTag) {
207 assert(DstAlign >= ElementSize &&
208 "Pointer alignment must be at least element size");
209 assert(SrcAlign >= ElementSize &&
210 "Pointer alignment must be at least element size");
211 Dst = getCastedInt8PtrValue(Dst);
212 Src = getCastedInt8PtrValue(Src);
213
214 Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
215 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
216 Module *M = BB->getParent()->getParent();
217 Function *TheFn = Intrinsic::getDeclaration(
218 M, Intrinsic::memcpy_element_unordered_atomic, Tys);
219
220 CallInst *CI = createCallHelper(TheFn, Ops, this);
221
222 // Set the alignment of the pointer args.
223 auto *AMCI = cast<AtomicMemCpyInst>(CI);
224 AMCI->setDestAlignment(DstAlign);
225 AMCI->setSourceAlignment(SrcAlign);
226
227 // Set the TBAA info if present.
228 if (TBAATag)
229 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
230
231 // Set the TBAA Struct info if present.
232 if (TBAAStructTag)
233 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
234
235 if (ScopeTag)
236 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
237
238 if (NoAliasTag)
239 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
240
241 return CI;
242 }
243
CreateMemMove(Value * Dst,MaybeAlign DstAlign,Value * Src,MaybeAlign SrcAlign,Value * Size,bool isVolatile,MDNode * TBAATag,MDNode * ScopeTag,MDNode * NoAliasTag)244 CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
245 Value *Src, MaybeAlign SrcAlign,
246 Value *Size, bool isVolatile,
247 MDNode *TBAATag, MDNode *ScopeTag,
248 MDNode *NoAliasTag) {
249 Dst = getCastedInt8PtrValue(Dst);
250 Src = getCastedInt8PtrValue(Src);
251
252 Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
253 Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
254 Module *M = BB->getParent()->getParent();
255 Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
256
257 CallInst *CI = createCallHelper(TheFn, Ops, this);
258
259 auto *MMI = cast<MemMoveInst>(CI);
260 if (DstAlign)
261 MMI->setDestAlignment(*DstAlign);
262 if (SrcAlign)
263 MMI->setSourceAlignment(*SrcAlign);
264
265 // Set the TBAA info if present.
266 if (TBAATag)
267 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
268
269 if (ScopeTag)
270 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
271
272 if (NoAliasTag)
273 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
274
275 return CI;
276 }
277
CreateElementUnorderedAtomicMemMove(Value * Dst,unsigned DstAlign,Value * Src,unsigned SrcAlign,Value * Size,uint32_t ElementSize,MDNode * TBAATag,MDNode * TBAAStructTag,MDNode * ScopeTag,MDNode * NoAliasTag)278 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
279 Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
280 uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
281 MDNode *ScopeTag, MDNode *NoAliasTag) {
282 assert(DstAlign >= ElementSize &&
283 "Pointer alignment must be at least element size");
284 assert(SrcAlign >= ElementSize &&
285 "Pointer alignment must be at least element size");
286 Dst = getCastedInt8PtrValue(Dst);
287 Src = getCastedInt8PtrValue(Src);
288
289 Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
290 Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
291 Module *M = BB->getParent()->getParent();
292 Function *TheFn = Intrinsic::getDeclaration(
293 M, Intrinsic::memmove_element_unordered_atomic, Tys);
294
295 CallInst *CI = createCallHelper(TheFn, Ops, this);
296
297 // Set the alignment of the pointer args.
298 CI->addParamAttr(
299 0, Attribute::getWithAlignment(CI->getContext(), Align(DstAlign)));
300 CI->addParamAttr(
301 1, Attribute::getWithAlignment(CI->getContext(), Align(SrcAlign)));
302
303 // Set the TBAA info if present.
304 if (TBAATag)
305 CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
306
307 // Set the TBAA Struct info if present.
308 if (TBAAStructTag)
309 CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
310
311 if (ScopeTag)
312 CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
313
314 if (NoAliasTag)
315 CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
316
317 return CI;
318 }
319
getReductionIntrinsic(IRBuilderBase * Builder,Intrinsic::ID ID,Value * Src)320 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
321 Value *Src) {
322 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
323 Value *Ops[] = {Src};
324 Type *Tys[] = { Src->getType() };
325 auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
326 return createCallHelper(Decl, Ops, Builder);
327 }
328
CreateFAddReduce(Value * Acc,Value * Src)329 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
330 Module *M = GetInsertBlock()->getParent()->getParent();
331 Value *Ops[] = {Acc, Src};
332 Type *Tys[] = {Acc->getType(), Src->getType()};
333 auto Decl = Intrinsic::getDeclaration(
334 M, Intrinsic::experimental_vector_reduce_v2_fadd, Tys);
335 return createCallHelper(Decl, Ops, this);
336 }
337
CreateFMulReduce(Value * Acc,Value * Src)338 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
339 Module *M = GetInsertBlock()->getParent()->getParent();
340 Value *Ops[] = {Acc, Src};
341 Type *Tys[] = {Acc->getType(), Src->getType()};
342 auto Decl = Intrinsic::getDeclaration(
343 M, Intrinsic::experimental_vector_reduce_v2_fmul, Tys);
344 return createCallHelper(Decl, Ops, this);
345 }
346
CreateAddReduce(Value * Src)347 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
348 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
349 Src);
350 }
351
CreateMulReduce(Value * Src)352 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
353 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
354 Src);
355 }
356
CreateAndReduce(Value * Src)357 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
358 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
359 Src);
360 }
361
CreateOrReduce(Value * Src)362 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
363 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
364 Src);
365 }
366
CreateXorReduce(Value * Src)367 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
368 return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
369 Src);
370 }
371
CreateIntMaxReduce(Value * Src,bool IsSigned)372 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
373 auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
374 : Intrinsic::experimental_vector_reduce_umax;
375 return getReductionIntrinsic(this, ID, Src);
376 }
377
CreateIntMinReduce(Value * Src,bool IsSigned)378 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
379 auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
380 : Intrinsic::experimental_vector_reduce_umin;
381 return getReductionIntrinsic(this, ID, Src);
382 }
383
CreateFPMaxReduce(Value * Src,bool NoNaN)384 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
385 auto Rdx = getReductionIntrinsic(
386 this, Intrinsic::experimental_vector_reduce_fmax, Src);
387 if (NoNaN) {
388 FastMathFlags FMF;
389 FMF.setNoNaNs();
390 Rdx->setFastMathFlags(FMF);
391 }
392 return Rdx;
393 }
394
CreateFPMinReduce(Value * Src,bool NoNaN)395 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
396 auto Rdx = getReductionIntrinsic(
397 this, Intrinsic::experimental_vector_reduce_fmin, Src);
398 if (NoNaN) {
399 FastMathFlags FMF;
400 FMF.setNoNaNs();
401 Rdx->setFastMathFlags(FMF);
402 }
403 return Rdx;
404 }
405
CreateLifetimeStart(Value * Ptr,ConstantInt * Size)406 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
407 assert(isa<PointerType>(Ptr->getType()) &&
408 "lifetime.start only applies to pointers.");
409 Ptr = getCastedInt8PtrValue(Ptr);
410 if (!Size)
411 Size = getInt64(-1);
412 else
413 assert(Size->getType() == getInt64Ty() &&
414 "lifetime.start requires the size to be an i64");
415 Value *Ops[] = { Size, Ptr };
416 Module *M = BB->getParent()->getParent();
417 Function *TheFn =
418 Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
419 return createCallHelper(TheFn, Ops, this);
420 }
421
CreateLifetimeEnd(Value * Ptr,ConstantInt * Size)422 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
423 assert(isa<PointerType>(Ptr->getType()) &&
424 "lifetime.end only applies to pointers.");
425 Ptr = getCastedInt8PtrValue(Ptr);
426 if (!Size)
427 Size = getInt64(-1);
428 else
429 assert(Size->getType() == getInt64Ty() &&
430 "lifetime.end requires the size to be an i64");
431 Value *Ops[] = { Size, Ptr };
432 Module *M = BB->getParent()->getParent();
433 Function *TheFn =
434 Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
435 return createCallHelper(TheFn, Ops, this);
436 }
437
CreateInvariantStart(Value * Ptr,ConstantInt * Size)438 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
439
440 assert(isa<PointerType>(Ptr->getType()) &&
441 "invariant.start only applies to pointers.");
442 Ptr = getCastedInt8PtrValue(Ptr);
443 if (!Size)
444 Size = getInt64(-1);
445 else
446 assert(Size->getType() == getInt64Ty() &&
447 "invariant.start requires the size to be an i64");
448
449 Value *Ops[] = {Size, Ptr};
450 // Fill in the single overloaded type: memory object type.
451 Type *ObjectPtr[1] = {Ptr->getType()};
452 Module *M = BB->getParent()->getParent();
453 Function *TheFn =
454 Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
455 return createCallHelper(TheFn, Ops, this);
456 }
457
CreateAssumption(Value * Cond)458 CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
459 assert(Cond->getType() == getInt1Ty() &&
460 "an assumption condition must be of type i1");
461
462 Value *Ops[] = { Cond };
463 Module *M = BB->getParent()->getParent();
464 Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
465 return createCallHelper(FnAssume, Ops, this);
466 }
467
468 /// Create a call to a Masked Load intrinsic.
469 /// \p Ptr - base pointer for the load
470 /// \p Align - alignment of the source location
471 /// \p Mask - vector of booleans which indicates what vector lanes should
472 /// be accessed in memory
473 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
474 /// of the result
475 /// \p Name - name of the result variable
CreateMaskedLoad(Value * Ptr,unsigned Align,Value * Mask,Value * PassThru,const Twine & Name)476 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
477 Value *Mask, Value *PassThru,
478 const Twine &Name) {
479 auto *PtrTy = cast<PointerType>(Ptr->getType());
480 Type *DataTy = PtrTy->getElementType();
481 assert(DataTy->isVectorTy() && "Ptr should point to a vector");
482 assert(Mask && "Mask should not be all-ones (null)");
483 if (!PassThru)
484 PassThru = UndefValue::get(DataTy);
485 Type *OverloadedTypes[] = { DataTy, PtrTy };
486 Value *Ops[] = { Ptr, getInt32(Align), Mask, PassThru};
487 return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
488 OverloadedTypes, Name);
489 }
490
491 /// Create a call to a Masked Store intrinsic.
492 /// \p Val - data to be stored,
493 /// \p Ptr - base pointer for the store
494 /// \p Align - alignment of the destination location
495 /// \p Mask - vector of booleans which indicates what vector lanes should
496 /// be accessed in memory
CreateMaskedStore(Value * Val,Value * Ptr,unsigned Align,Value * Mask)497 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
498 unsigned Align, Value *Mask) {
499 auto *PtrTy = cast<PointerType>(Ptr->getType());
500 Type *DataTy = PtrTy->getElementType();
501 assert(DataTy->isVectorTy() && "Ptr should point to a vector");
502 assert(Mask && "Mask should not be all-ones (null)");
503 Type *OverloadedTypes[] = { DataTy, PtrTy };
504 Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
505 return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
506 }
507
508 /// Create a call to a Masked intrinsic, with given intrinsic Id,
509 /// an array of operands - Ops, and an array of overloaded types -
510 /// OverloadedTypes.
CreateMaskedIntrinsic(Intrinsic::ID Id,ArrayRef<Value * > Ops,ArrayRef<Type * > OverloadedTypes,const Twine & Name)511 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
512 ArrayRef<Value *> Ops,
513 ArrayRef<Type *> OverloadedTypes,
514 const Twine &Name) {
515 Module *M = BB->getParent()->getParent();
516 Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
517 return createCallHelper(TheFn, Ops, this, Name);
518 }
519
520 /// Create a call to a Masked Gather intrinsic.
521 /// \p Ptrs - vector of pointers for loading
522 /// \p Align - alignment for one element
523 /// \p Mask - vector of booleans which indicates what vector lanes should
524 /// be accessed in memory
525 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
526 /// of the result
527 /// \p Name - name of the result variable
CreateMaskedGather(Value * Ptrs,unsigned Align,Value * Mask,Value * PassThru,const Twine & Name)528 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align,
529 Value *Mask, Value *PassThru,
530 const Twine& Name) {
531 auto PtrsTy = cast<VectorType>(Ptrs->getType());
532 auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
533 unsigned NumElts = PtrsTy->getVectorNumElements();
534 Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
535
536 if (!Mask)
537 Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
538 NumElts));
539
540 if (!PassThru)
541 PassThru = UndefValue::get(DataTy);
542
543 Type *OverloadedTypes[] = {DataTy, PtrsTy};
544 Value * Ops[] = {Ptrs, getInt32(Align), Mask, PassThru};
545
546 // We specify only one type when we create this intrinsic. Types of other
547 // arguments are derived from this type.
548 return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
549 Name);
550 }
551
552 /// Create a call to a Masked Scatter intrinsic.
553 /// \p Data - data to be stored,
554 /// \p Ptrs - the vector of pointers, where the \p Data elements should be
555 /// stored
556 /// \p Align - alignment for one element
557 /// \p Mask - vector of booleans which indicates what vector lanes should
558 /// be accessed in memory
CreateMaskedScatter(Value * Data,Value * Ptrs,unsigned Align,Value * Mask)559 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
560 unsigned Align, Value *Mask) {
561 auto PtrsTy = cast<VectorType>(Ptrs->getType());
562 auto DataTy = cast<VectorType>(Data->getType());
563 unsigned NumElts = PtrsTy->getVectorNumElements();
564
565 #ifndef NDEBUG
566 auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
567 assert(NumElts == DataTy->getVectorNumElements() &&
568 PtrTy->getElementType() == DataTy->getElementType() &&
569 "Incompatible pointer and data types");
570 #endif
571
572 if (!Mask)
573 Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
574 NumElts));
575
576 Type *OverloadedTypes[] = {DataTy, PtrsTy};
577 Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask};
578
579 // We specify only one type when we create this intrinsic. Types of other
580 // arguments are derived from this type.
581 return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
582 }
583
584 template <typename T0, typename T1, typename T2, typename T3>
585 static std::vector<Value *>
getStatepointArgs(IRBuilderBase & B,uint64_t ID,uint32_t NumPatchBytes,Value * ActualCallee,uint32_t Flags,ArrayRef<T0> CallArgs,ArrayRef<T1> TransitionArgs,ArrayRef<T2> DeoptArgs,ArrayRef<T3> GCArgs)586 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
587 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
588 ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
589 ArrayRef<T3> GCArgs) {
590 std::vector<Value *> Args;
591 Args.push_back(B.getInt64(ID));
592 Args.push_back(B.getInt32(NumPatchBytes));
593 Args.push_back(ActualCallee);
594 Args.push_back(B.getInt32(CallArgs.size()));
595 Args.push_back(B.getInt32(Flags));
596 Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
597 Args.push_back(B.getInt32(TransitionArgs.size()));
598 Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
599 Args.push_back(B.getInt32(DeoptArgs.size()));
600 Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
601 Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
602
603 return Args;
604 }
605
606 template <typename T0, typename T1, typename T2, typename T3>
CreateGCStatepointCallCommon(IRBuilderBase * Builder,uint64_t ID,uint32_t NumPatchBytes,Value * ActualCallee,uint32_t Flags,ArrayRef<T0> CallArgs,ArrayRef<T1> TransitionArgs,ArrayRef<T2> DeoptArgs,ArrayRef<T3> GCArgs,const Twine & Name)607 static CallInst *CreateGCStatepointCallCommon(
608 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
609 Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
610 ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
611 const Twine &Name) {
612 // Extract out the type of the callee.
613 auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
614 assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
615 "actual callee must be a callable value");
616
617 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
618 // Fill in the one generic type'd argument (the function is also vararg)
619 Type *ArgTypes[] = { FuncPtrType };
620 Function *FnStatepoint =
621 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
622 ArgTypes);
623
624 std::vector<Value *> Args =
625 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
626 CallArgs, TransitionArgs, DeoptArgs, GCArgs);
627 return createCallHelper(FnStatepoint, Args, Builder, Name);
628 }
629
CreateGCStatepointCall(uint64_t ID,uint32_t NumPatchBytes,Value * ActualCallee,ArrayRef<Value * > CallArgs,ArrayRef<Value * > DeoptArgs,ArrayRef<Value * > GCArgs,const Twine & Name)630 CallInst *IRBuilderBase::CreateGCStatepointCall(
631 uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
632 ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
633 ArrayRef<Value *> GCArgs, const Twine &Name) {
634 return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
635 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
636 CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
637 }
638
CreateGCStatepointCall(uint64_t ID,uint32_t NumPatchBytes,Value * ActualCallee,uint32_t Flags,ArrayRef<Use> CallArgs,ArrayRef<Use> TransitionArgs,ArrayRef<Use> DeoptArgs,ArrayRef<Value * > GCArgs,const Twine & Name)639 CallInst *IRBuilderBase::CreateGCStatepointCall(
640 uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
641 ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
642 ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
643 return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
644 this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
645 DeoptArgs, GCArgs, Name);
646 }
647
CreateGCStatepointCall(uint64_t ID,uint32_t NumPatchBytes,Value * ActualCallee,ArrayRef<Use> CallArgs,ArrayRef<Value * > DeoptArgs,ArrayRef<Value * > GCArgs,const Twine & Name)648 CallInst *IRBuilderBase::CreateGCStatepointCall(
649 uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
650 ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
651 ArrayRef<Value *> GCArgs, const Twine &Name) {
652 return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
653 this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
654 CallArgs, None, DeoptArgs, GCArgs, Name);
655 }
656
657 template <typename T0, typename T1, typename T2, typename T3>
CreateGCStatepointInvokeCommon(IRBuilderBase * Builder,uint64_t ID,uint32_t NumPatchBytes,Value * ActualInvokee,BasicBlock * NormalDest,BasicBlock * UnwindDest,uint32_t Flags,ArrayRef<T0> InvokeArgs,ArrayRef<T1> TransitionArgs,ArrayRef<T2> DeoptArgs,ArrayRef<T3> GCArgs,const Twine & Name)658 static InvokeInst *CreateGCStatepointInvokeCommon(
659 IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
660 Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
661 uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
662 ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
663 // Extract out the type of the callee.
664 auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
665 assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
666 "actual callee must be a callable value");
667
668 Module *M = Builder->GetInsertBlock()->getParent()->getParent();
669 // Fill in the one generic type'd argument (the function is also vararg)
670 Function *FnStatepoint = Intrinsic::getDeclaration(
671 M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
672
673 std::vector<Value *> Args =
674 getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
675 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
676 return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder,
677 Name);
678 }
679
CreateGCStatepointInvoke(uint64_t ID,uint32_t NumPatchBytes,Value * ActualInvokee,BasicBlock * NormalDest,BasicBlock * UnwindDest,ArrayRef<Value * > InvokeArgs,ArrayRef<Value * > DeoptArgs,ArrayRef<Value * > GCArgs,const Twine & Name)680 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
681 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
682 BasicBlock *NormalDest, BasicBlock *UnwindDest,
683 ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
684 ArrayRef<Value *> GCArgs, const Twine &Name) {
685 return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
686 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
687 uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
688 DeoptArgs, GCArgs, Name);
689 }
690
CreateGCStatepointInvoke(uint64_t ID,uint32_t NumPatchBytes,Value * ActualInvokee,BasicBlock * NormalDest,BasicBlock * UnwindDest,uint32_t Flags,ArrayRef<Use> InvokeArgs,ArrayRef<Use> TransitionArgs,ArrayRef<Use> DeoptArgs,ArrayRef<Value * > GCArgs,const Twine & Name)691 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
692 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
693 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
694 ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
695 ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
696 return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
697 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
698 InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
699 }
700
CreateGCStatepointInvoke(uint64_t ID,uint32_t NumPatchBytes,Value * ActualInvokee,BasicBlock * NormalDest,BasicBlock * UnwindDest,ArrayRef<Use> InvokeArgs,ArrayRef<Value * > DeoptArgs,ArrayRef<Value * > GCArgs,const Twine & Name)701 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
702 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
703 BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
704 ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
705 return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
706 this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
707 uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
708 Name);
709 }
710
CreateGCResult(Instruction * Statepoint,Type * ResultType,const Twine & Name)711 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
712 Type *ResultType,
713 const Twine &Name) {
714 Intrinsic::ID ID = Intrinsic::experimental_gc_result;
715 Module *M = BB->getParent()->getParent();
716 Type *Types[] = {ResultType};
717 Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
718
719 Value *Args[] = {Statepoint};
720 return createCallHelper(FnGCResult, Args, this, Name);
721 }
722
CreateGCRelocate(Instruction * Statepoint,int BaseOffset,int DerivedOffset,Type * ResultType,const Twine & Name)723 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
724 int BaseOffset,
725 int DerivedOffset,
726 Type *ResultType,
727 const Twine &Name) {
728 Module *M = BB->getParent()->getParent();
729 Type *Types[] = {ResultType};
730 Function *FnGCRelocate =
731 Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
732
733 Value *Args[] = {Statepoint,
734 getInt32(BaseOffset),
735 getInt32(DerivedOffset)};
736 return createCallHelper(FnGCRelocate, Args, this, Name);
737 }
738
CreateUnaryIntrinsic(Intrinsic::ID ID,Value * V,Instruction * FMFSource,const Twine & Name)739 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
740 Instruction *FMFSource,
741 const Twine &Name) {
742 Module *M = BB->getModule();
743 Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
744 return createCallHelper(Fn, {V}, this, Name, FMFSource);
745 }
746
CreateBinaryIntrinsic(Intrinsic::ID ID,Value * LHS,Value * RHS,Instruction * FMFSource,const Twine & Name)747 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
748 Value *RHS,
749 Instruction *FMFSource,
750 const Twine &Name) {
751 Module *M = BB->getModule();
752 Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
753 return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
754 }
755
CreateIntrinsic(Intrinsic::ID ID,ArrayRef<Type * > Types,ArrayRef<Value * > Args,Instruction * FMFSource,const Twine & Name)756 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
757 ArrayRef<Type *> Types,
758 ArrayRef<Value *> Args,
759 Instruction *FMFSource,
760 const Twine &Name) {
761 Module *M = BB->getModule();
762 Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
763 return createCallHelper(Fn, Args, this, Name, FMFSource);
764 }
765