1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit Builtin calls as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "CodeGenFunction.h"
15 #include "CGObjCRuntime.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/AST/Decl.h"
20 #include "clang/Basic/TargetBuiltins.h"
21 #include "clang/Basic/TargetInfo.h"
22 #include "clang/CodeGen/CGFunctionInfo.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/Intrinsics.h"
25
26 using namespace clang;
27 using namespace CodeGen;
28 using namespace llvm;
29
30 /// getBuiltinLibFunction - Given a builtin id for a function like
31 /// "__builtin_fabsf", return a Function* for "fabsf".
getBuiltinLibFunction(const FunctionDecl * FD,unsigned BuiltinID)32 llvm::Value *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
33 unsigned BuiltinID) {
34 assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
35
36 // Get the name, skip over the __builtin_ prefix (if necessary).
37 StringRef Name;
38 GlobalDecl D(FD);
39
40 // If the builtin has been declared explicitly with an assembler label,
41 // use the mangled name. This differs from the plain label on platforms
42 // that prefix labels.
43 if (FD->hasAttr<AsmLabelAttr>())
44 Name = getMangledName(D);
45 else
46 Name = Context.BuiltinInfo.GetName(BuiltinID) + 10;
47
48 llvm::FunctionType *Ty =
49 cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
50
51 return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
52 }
53
54 /// Emit the conversions required to turn the given value into an
55 /// integer of the given size.
EmitToInt(CodeGenFunction & CGF,llvm::Value * V,QualType T,llvm::IntegerType * IntType)56 static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
57 QualType T, llvm::IntegerType *IntType) {
58 V = CGF.EmitToMemory(V, T);
59
60 if (V->getType()->isPointerTy())
61 return CGF.Builder.CreatePtrToInt(V, IntType);
62
63 assert(V->getType() == IntType);
64 return V;
65 }
66
EmitFromInt(CodeGenFunction & CGF,llvm::Value * V,QualType T,llvm::Type * ResultType)67 static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
68 QualType T, llvm::Type *ResultType) {
69 V = CGF.EmitFromMemory(V, T);
70
71 if (ResultType->isPointerTy())
72 return CGF.Builder.CreateIntToPtr(V, ResultType);
73
74 assert(V->getType() == ResultType);
75 return V;
76 }
77
78 /// Utility to insert an atomic instruction based on Instrinsic::ID
79 /// and the expression node.
EmitBinaryAtomic(CodeGenFunction & CGF,llvm::AtomicRMWInst::BinOp Kind,const CallExpr * E)80 static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
81 llvm::AtomicRMWInst::BinOp Kind,
82 const CallExpr *E) {
83 QualType T = E->getType();
84 assert(E->getArg(0)->getType()->isPointerType());
85 assert(CGF.getContext().hasSameUnqualifiedType(T,
86 E->getArg(0)->getType()->getPointeeType()));
87 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
88
89 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
90 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
91
92 llvm::IntegerType *IntType =
93 llvm::IntegerType::get(CGF.getLLVMContext(),
94 CGF.getContext().getTypeSize(T));
95 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
96
97 llvm::Value *Args[2];
98 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
99 Args[1] = CGF.EmitScalarExpr(E->getArg(1));
100 llvm::Type *ValueType = Args[1]->getType();
101 Args[1] = EmitToInt(CGF, Args[1], T, IntType);
102
103 llvm::Value *Result =
104 CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1],
105 llvm::SequentiallyConsistent);
106 Result = EmitFromInt(CGF, Result, T, ValueType);
107 return RValue::get(Result);
108 }
109
110 /// Utility to insert an atomic instruction based Instrinsic::ID and
111 /// the expression node, where the return value is the result of the
112 /// operation.
EmitBinaryAtomicPost(CodeGenFunction & CGF,llvm::AtomicRMWInst::BinOp Kind,const CallExpr * E,Instruction::BinaryOps Op)113 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
114 llvm::AtomicRMWInst::BinOp Kind,
115 const CallExpr *E,
116 Instruction::BinaryOps Op) {
117 QualType T = E->getType();
118 assert(E->getArg(0)->getType()->isPointerType());
119 assert(CGF.getContext().hasSameUnqualifiedType(T,
120 E->getArg(0)->getType()->getPointeeType()));
121 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
122
123 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
124 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
125
126 llvm::IntegerType *IntType =
127 llvm::IntegerType::get(CGF.getLLVMContext(),
128 CGF.getContext().getTypeSize(T));
129 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
130
131 llvm::Value *Args[2];
132 Args[1] = CGF.EmitScalarExpr(E->getArg(1));
133 llvm::Type *ValueType = Args[1]->getType();
134 Args[1] = EmitToInt(CGF, Args[1], T, IntType);
135 Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
136
137 llvm::Value *Result =
138 CGF.Builder.CreateAtomicRMW(Kind, Args[0], Args[1],
139 llvm::SequentiallyConsistent);
140 Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
141 Result = EmitFromInt(CGF, Result, T, ValueType);
142 return RValue::get(Result);
143 }
144
145 /// EmitFAbs - Emit a call to fabs/fabsf/fabsl, depending on the type of ValTy,
146 /// which must be a scalar floating point type.
EmitFAbs(CodeGenFunction & CGF,Value * V,QualType ValTy)147 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V, QualType ValTy) {
148 const BuiltinType *ValTyP = ValTy->getAs<BuiltinType>();
149 assert(ValTyP && "isn't scalar fp type!");
150
151 StringRef FnName;
152 switch (ValTyP->getKind()) {
153 default: llvm_unreachable("Isn't a scalar fp type!");
154 case BuiltinType::Float: FnName = "fabsf"; break;
155 case BuiltinType::Double: FnName = "fabs"; break;
156 case BuiltinType::LongDouble: FnName = "fabsl"; break;
157 }
158
159 // The prototype is something that takes and returns whatever V's type is.
160 llvm::FunctionType *FT = llvm::FunctionType::get(V->getType(), V->getType(),
161 false);
162 llvm::Value *Fn = CGF.CGM.CreateRuntimeFunction(FT, FnName);
163
164 return CGF.EmitNounwindRuntimeCall(Fn, V, "abs");
165 }
166
emitLibraryCall(CodeGenFunction & CGF,const FunctionDecl * Fn,const CallExpr * E,llvm::Value * calleeValue)167 static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *Fn,
168 const CallExpr *E, llvm::Value *calleeValue) {
169 return CGF.EmitCall(E->getCallee()->getType(), calleeValue, E->getLocStart(),
170 ReturnValueSlot(), E->arg_begin(), E->arg_end(), Fn);
171 }
172
173 /// \brief Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
174 /// depending on IntrinsicID.
175 ///
176 /// \arg CGF The current codegen function.
177 /// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
178 /// \arg X The first argument to the llvm.*.with.overflow.*.
179 /// \arg Y The second argument to the llvm.*.with.overflow.*.
180 /// \arg Carry The carry returned by the llvm.*.with.overflow.*.
181 /// \returns The result (i.e. sum/product) returned by the intrinsic.
EmitOverflowIntrinsic(CodeGenFunction & CGF,const llvm::Intrinsic::ID IntrinsicID,llvm::Value * X,llvm::Value * Y,llvm::Value * & Carry)182 static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
183 const llvm::Intrinsic::ID IntrinsicID,
184 llvm::Value *X, llvm::Value *Y,
185 llvm::Value *&Carry) {
186 // Make sure we have integers of the same width.
187 assert(X->getType() == Y->getType() &&
188 "Arguments must be the same type. (Did you forget to make sure both "
189 "arguments have the same integer width?)");
190
191 llvm::Value *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
192 llvm::Value *Tmp = CGF.Builder.CreateCall2(Callee, X, Y);
193 Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
194 return CGF.Builder.CreateExtractValue(Tmp, 0);
195 }
196
EmitBuiltinExpr(const FunctionDecl * FD,unsigned BuiltinID,const CallExpr * E)197 RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
198 unsigned BuiltinID, const CallExpr *E) {
199 // See if we can constant fold this builtin. If so, don't emit it at all.
200 Expr::EvalResult Result;
201 if (E->EvaluateAsRValue(Result, CGM.getContext()) &&
202 !Result.hasSideEffects()) {
203 if (Result.Val.isInt())
204 return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
205 Result.Val.getInt()));
206 if (Result.Val.isFloat())
207 return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
208 Result.Val.getFloat()));
209 }
210
211 switch (BuiltinID) {
212 default: break; // Handle intrinsics and libm functions below.
213 case Builtin::BI__builtin___CFStringMakeConstantString:
214 case Builtin::BI__builtin___NSStringMakeConstantString:
215 return RValue::get(CGM.EmitConstantExpr(E, E->getType(), nullptr));
216 case Builtin::BI__builtin_stdarg_start:
217 case Builtin::BI__builtin_va_start:
218 case Builtin::BI__va_start:
219 case Builtin::BI__builtin_va_end: {
220 Value *ArgValue = (BuiltinID == Builtin::BI__va_start)
221 ? EmitScalarExpr(E->getArg(0))
222 : EmitVAListRef(E->getArg(0));
223 llvm::Type *DestType = Int8PtrTy;
224 if (ArgValue->getType() != DestType)
225 ArgValue = Builder.CreateBitCast(ArgValue, DestType,
226 ArgValue->getName().data());
227
228 Intrinsic::ID inst = (BuiltinID == Builtin::BI__builtin_va_end) ?
229 Intrinsic::vaend : Intrinsic::vastart;
230 return RValue::get(Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue));
231 }
232 case Builtin::BI__builtin_va_copy: {
233 Value *DstPtr = EmitVAListRef(E->getArg(0));
234 Value *SrcPtr = EmitVAListRef(E->getArg(1));
235
236 llvm::Type *Type = Int8PtrTy;
237
238 DstPtr = Builder.CreateBitCast(DstPtr, Type);
239 SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
240 return RValue::get(Builder.CreateCall2(CGM.getIntrinsic(Intrinsic::vacopy),
241 DstPtr, SrcPtr));
242 }
243 case Builtin::BI__builtin_abs:
244 case Builtin::BI__builtin_labs:
245 case Builtin::BI__builtin_llabs: {
246 Value *ArgValue = EmitScalarExpr(E->getArg(0));
247
248 Value *NegOp = Builder.CreateNeg(ArgValue, "neg");
249 Value *CmpResult =
250 Builder.CreateICmpSGE(ArgValue,
251 llvm::Constant::getNullValue(ArgValue->getType()),
252 "abscond");
253 Value *Result =
254 Builder.CreateSelect(CmpResult, ArgValue, NegOp, "abs");
255
256 return RValue::get(Result);
257 }
258
259 case Builtin::BI__builtin_conj:
260 case Builtin::BI__builtin_conjf:
261 case Builtin::BI__builtin_conjl: {
262 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
263 Value *Real = ComplexVal.first;
264 Value *Imag = ComplexVal.second;
265 Value *Zero =
266 Imag->getType()->isFPOrFPVectorTy()
267 ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType())
268 : llvm::Constant::getNullValue(Imag->getType());
269
270 Imag = Builder.CreateFSub(Zero, Imag, "sub");
271 return RValue::getComplex(std::make_pair(Real, Imag));
272 }
273 case Builtin::BI__builtin_creal:
274 case Builtin::BI__builtin_crealf:
275 case Builtin::BI__builtin_creall:
276 case Builtin::BIcreal:
277 case Builtin::BIcrealf:
278 case Builtin::BIcreall: {
279 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
280 return RValue::get(ComplexVal.first);
281 }
282
283 case Builtin::BI__builtin_cimag:
284 case Builtin::BI__builtin_cimagf:
285 case Builtin::BI__builtin_cimagl:
286 case Builtin::BIcimag:
287 case Builtin::BIcimagf:
288 case Builtin::BIcimagl: {
289 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
290 return RValue::get(ComplexVal.second);
291 }
292
293 case Builtin::BI__builtin_ctzs:
294 case Builtin::BI__builtin_ctz:
295 case Builtin::BI__builtin_ctzl:
296 case Builtin::BI__builtin_ctzll: {
297 Value *ArgValue = EmitScalarExpr(E->getArg(0));
298
299 llvm::Type *ArgType = ArgValue->getType();
300 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
301
302 llvm::Type *ResultType = ConvertType(E->getType());
303 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
304 Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef);
305 if (Result->getType() != ResultType)
306 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
307 "cast");
308 return RValue::get(Result);
309 }
310 case Builtin::BI__builtin_clzs:
311 case Builtin::BI__builtin_clz:
312 case Builtin::BI__builtin_clzl:
313 case Builtin::BI__builtin_clzll: {
314 Value *ArgValue = EmitScalarExpr(E->getArg(0));
315
316 llvm::Type *ArgType = ArgValue->getType();
317 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
318
319 llvm::Type *ResultType = ConvertType(E->getType());
320 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
321 Value *Result = Builder.CreateCall2(F, ArgValue, ZeroUndef);
322 if (Result->getType() != ResultType)
323 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
324 "cast");
325 return RValue::get(Result);
326 }
327 case Builtin::BI__builtin_ffs:
328 case Builtin::BI__builtin_ffsl:
329 case Builtin::BI__builtin_ffsll: {
330 // ffs(x) -> x ? cttz(x) + 1 : 0
331 Value *ArgValue = EmitScalarExpr(E->getArg(0));
332
333 llvm::Type *ArgType = ArgValue->getType();
334 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
335
336 llvm::Type *ResultType = ConvertType(E->getType());
337 Value *Tmp = Builder.CreateAdd(Builder.CreateCall2(F, ArgValue,
338 Builder.getTrue()),
339 llvm::ConstantInt::get(ArgType, 1));
340 Value *Zero = llvm::Constant::getNullValue(ArgType);
341 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
342 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
343 if (Result->getType() != ResultType)
344 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
345 "cast");
346 return RValue::get(Result);
347 }
348 case Builtin::BI__builtin_parity:
349 case Builtin::BI__builtin_parityl:
350 case Builtin::BI__builtin_parityll: {
351 // parity(x) -> ctpop(x) & 1
352 Value *ArgValue = EmitScalarExpr(E->getArg(0));
353
354 llvm::Type *ArgType = ArgValue->getType();
355 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
356
357 llvm::Type *ResultType = ConvertType(E->getType());
358 Value *Tmp = Builder.CreateCall(F, ArgValue);
359 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
360 if (Result->getType() != ResultType)
361 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
362 "cast");
363 return RValue::get(Result);
364 }
365 case Builtin::BI__builtin_popcount:
366 case Builtin::BI__builtin_popcountl:
367 case Builtin::BI__builtin_popcountll: {
368 Value *ArgValue = EmitScalarExpr(E->getArg(0));
369
370 llvm::Type *ArgType = ArgValue->getType();
371 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
372
373 llvm::Type *ResultType = ConvertType(E->getType());
374 Value *Result = Builder.CreateCall(F, ArgValue);
375 if (Result->getType() != ResultType)
376 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
377 "cast");
378 return RValue::get(Result);
379 }
380 case Builtin::BI__builtin_expect: {
381 Value *ArgValue = EmitScalarExpr(E->getArg(0));
382 llvm::Type *ArgType = ArgValue->getType();
383
384 Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
385 Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
386
387 Value *Result = Builder.CreateCall2(FnExpect, ArgValue, ExpectedValue,
388 "expval");
389 return RValue::get(Result);
390 }
391 case Builtin::BI__builtin_bswap16:
392 case Builtin::BI__builtin_bswap32:
393 case Builtin::BI__builtin_bswap64: {
394 Value *ArgValue = EmitScalarExpr(E->getArg(0));
395 llvm::Type *ArgType = ArgValue->getType();
396 Value *F = CGM.getIntrinsic(Intrinsic::bswap, ArgType);
397 return RValue::get(Builder.CreateCall(F, ArgValue));
398 }
399 case Builtin::BI__builtin_object_size: {
400 // We rely on constant folding to deal with expressions with side effects.
401 assert(!E->getArg(0)->HasSideEffects(getContext()) &&
402 "should have been constant folded");
403
404 // We pass this builtin onto the optimizer so that it can
405 // figure out the object size in more complex cases.
406 llvm::Type *ResType = ConvertType(E->getType());
407
408 // LLVM only supports 0 and 2, make sure that we pass along that
409 // as a boolean.
410 Value *Ty = EmitScalarExpr(E->getArg(1));
411 ConstantInt *CI = dyn_cast<ConstantInt>(Ty);
412 assert(CI);
413 uint64_t val = CI->getZExtValue();
414 CI = ConstantInt::get(Builder.getInt1Ty(), (val & 0x2) >> 1);
415 // FIXME: Get right address space.
416 llvm::Type *Tys[] = { ResType, Builder.getInt8PtrTy(0) };
417 Value *F = CGM.getIntrinsic(Intrinsic::objectsize, Tys);
418 return RValue::get(Builder.CreateCall2(F, EmitScalarExpr(E->getArg(0)),CI));
419 }
420 case Builtin::BI__builtin_prefetch: {
421 Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
422 // FIXME: Technically these constants should of type 'int', yes?
423 RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
424 llvm::ConstantInt::get(Int32Ty, 0);
425 Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
426 llvm::ConstantInt::get(Int32Ty, 3);
427 Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
428 Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
429 return RValue::get(Builder.CreateCall4(F, Address, RW, Locality, Data));
430 }
431 case Builtin::BI__builtin_readcyclecounter: {
432 Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
433 return RValue::get(Builder.CreateCall(F));
434 }
435 case Builtin::BI__builtin___clear_cache: {
436 Value *Begin = EmitScalarExpr(E->getArg(0));
437 Value *End = EmitScalarExpr(E->getArg(1));
438 Value *F = CGM.getIntrinsic(Intrinsic::clear_cache);
439 return RValue::get(Builder.CreateCall2(F, Begin, End));
440 }
441 case Builtin::BI__builtin_trap: {
442 Value *F = CGM.getIntrinsic(Intrinsic::trap);
443 return RValue::get(Builder.CreateCall(F));
444 }
445 case Builtin::BI__debugbreak: {
446 Value *F = CGM.getIntrinsic(Intrinsic::debugtrap);
447 return RValue::get(Builder.CreateCall(F));
448 }
449 case Builtin::BI__builtin_unreachable: {
450 if (SanOpts->Unreachable)
451 EmitCheck(Builder.getFalse(), "builtin_unreachable",
452 EmitCheckSourceLocation(E->getExprLoc()),
453 ArrayRef<llvm::Value *>(), CRK_Unrecoverable);
454 else
455 Builder.CreateUnreachable();
456
457 // We do need to preserve an insertion point.
458 EmitBlock(createBasicBlock("unreachable.cont"));
459
460 return RValue::get(nullptr);
461 }
462
463 case Builtin::BI__builtin_powi:
464 case Builtin::BI__builtin_powif:
465 case Builtin::BI__builtin_powil: {
466 Value *Base = EmitScalarExpr(E->getArg(0));
467 Value *Exponent = EmitScalarExpr(E->getArg(1));
468 llvm::Type *ArgType = Base->getType();
469 Value *F = CGM.getIntrinsic(Intrinsic::powi, ArgType);
470 return RValue::get(Builder.CreateCall2(F, Base, Exponent));
471 }
472
473 case Builtin::BI__builtin_isgreater:
474 case Builtin::BI__builtin_isgreaterequal:
475 case Builtin::BI__builtin_isless:
476 case Builtin::BI__builtin_islessequal:
477 case Builtin::BI__builtin_islessgreater:
478 case Builtin::BI__builtin_isunordered: {
479 // Ordered comparisons: we know the arguments to these are matching scalar
480 // floating point values.
481 Value *LHS = EmitScalarExpr(E->getArg(0));
482 Value *RHS = EmitScalarExpr(E->getArg(1));
483
484 switch (BuiltinID) {
485 default: llvm_unreachable("Unknown ordered comparison");
486 case Builtin::BI__builtin_isgreater:
487 LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
488 break;
489 case Builtin::BI__builtin_isgreaterequal:
490 LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
491 break;
492 case Builtin::BI__builtin_isless:
493 LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
494 break;
495 case Builtin::BI__builtin_islessequal:
496 LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
497 break;
498 case Builtin::BI__builtin_islessgreater:
499 LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
500 break;
501 case Builtin::BI__builtin_isunordered:
502 LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
503 break;
504 }
505 // ZExt bool to int type.
506 return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
507 }
508 case Builtin::BI__builtin_isnan: {
509 Value *V = EmitScalarExpr(E->getArg(0));
510 V = Builder.CreateFCmpUNO(V, V, "cmp");
511 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
512 }
513
514 case Builtin::BI__builtin_isinf: {
515 // isinf(x) --> fabs(x) == infinity
516 Value *V = EmitScalarExpr(E->getArg(0));
517 V = EmitFAbs(*this, V, E->getArg(0)->getType());
518
519 V = Builder.CreateFCmpOEQ(V, ConstantFP::getInfinity(V->getType()),"isinf");
520 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
521 }
522
523 // TODO: BI__builtin_isinf_sign
524 // isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0
525
526 case Builtin::BI__builtin_isnormal: {
527 // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min
528 Value *V = EmitScalarExpr(E->getArg(0));
529 Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
530
531 Value *Abs = EmitFAbs(*this, V, E->getArg(0)->getType());
532 Value *IsLessThanInf =
533 Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
534 APFloat Smallest = APFloat::getSmallestNormalized(
535 getContext().getFloatTypeSemantics(E->getArg(0)->getType()));
536 Value *IsNormal =
537 Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
538 "isnormal");
539 V = Builder.CreateAnd(Eq, IsLessThanInf, "and");
540 V = Builder.CreateAnd(V, IsNormal, "and");
541 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
542 }
543
544 case Builtin::BI__builtin_isfinite: {
545 // isfinite(x) --> x == x && fabs(x) != infinity;
546 Value *V = EmitScalarExpr(E->getArg(0));
547 Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
548
549 Value *Abs = EmitFAbs(*this, V, E->getArg(0)->getType());
550 Value *IsNotInf =
551 Builder.CreateFCmpUNE(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
552
553 V = Builder.CreateAnd(Eq, IsNotInf, "and");
554 return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
555 }
556
557 case Builtin::BI__builtin_fpclassify: {
558 Value *V = EmitScalarExpr(E->getArg(5));
559 llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
560
561 // Create Result
562 BasicBlock *Begin = Builder.GetInsertBlock();
563 BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
564 Builder.SetInsertPoint(End);
565 PHINode *Result =
566 Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
567 "fpclassify_result");
568
569 // if (V==0) return FP_ZERO
570 Builder.SetInsertPoint(Begin);
571 Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
572 "iszero");
573 Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
574 BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
575 Builder.CreateCondBr(IsZero, End, NotZero);
576 Result->addIncoming(ZeroLiteral, Begin);
577
578 // if (V != V) return FP_NAN
579 Builder.SetInsertPoint(NotZero);
580 Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
581 Value *NanLiteral = EmitScalarExpr(E->getArg(0));
582 BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
583 Builder.CreateCondBr(IsNan, End, NotNan);
584 Result->addIncoming(NanLiteral, NotZero);
585
586 // if (fabs(V) == infinity) return FP_INFINITY
587 Builder.SetInsertPoint(NotNan);
588 Value *VAbs = EmitFAbs(*this, V, E->getArg(5)->getType());
589 Value *IsInf =
590 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
591 "isinf");
592 Value *InfLiteral = EmitScalarExpr(E->getArg(1));
593 BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
594 Builder.CreateCondBr(IsInf, End, NotInf);
595 Result->addIncoming(InfLiteral, NotNan);
596
597 // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
598 Builder.SetInsertPoint(NotInf);
599 APFloat Smallest = APFloat::getSmallestNormalized(
600 getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
601 Value *IsNormal =
602 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
603 "isnormal");
604 Value *NormalResult =
605 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
606 EmitScalarExpr(E->getArg(3)));
607 Builder.CreateBr(End);
608 Result->addIncoming(NormalResult, NotInf);
609
610 // return Result
611 Builder.SetInsertPoint(End);
612 return RValue::get(Result);
613 }
614
615 case Builtin::BIalloca:
616 case Builtin::BI_alloca:
617 case Builtin::BI__builtin_alloca: {
618 Value *Size = EmitScalarExpr(E->getArg(0));
619 return RValue::get(Builder.CreateAlloca(Builder.getInt8Ty(), Size));
620 }
621 case Builtin::BIbzero:
622 case Builtin::BI__builtin_bzero: {
623 std::pair<llvm::Value*, unsigned> Dest =
624 EmitPointerWithAlignment(E->getArg(0));
625 Value *SizeVal = EmitScalarExpr(E->getArg(1));
626 Builder.CreateMemSet(Dest.first, Builder.getInt8(0), SizeVal,
627 Dest.second, false);
628 return RValue::get(Dest.first);
629 }
630 case Builtin::BImemcpy:
631 case Builtin::BI__builtin_memcpy: {
632 std::pair<llvm::Value*, unsigned> Dest =
633 EmitPointerWithAlignment(E->getArg(0));
634 std::pair<llvm::Value*, unsigned> Src =
635 EmitPointerWithAlignment(E->getArg(1));
636 Value *SizeVal = EmitScalarExpr(E->getArg(2));
637 unsigned Align = std::min(Dest.second, Src.second);
638 Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false);
639 return RValue::get(Dest.first);
640 }
641
642 case Builtin::BI__builtin___memcpy_chk: {
643 // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
644 llvm::APSInt Size, DstSize;
645 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
646 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
647 break;
648 if (Size.ugt(DstSize))
649 break;
650 std::pair<llvm::Value*, unsigned> Dest =
651 EmitPointerWithAlignment(E->getArg(0));
652 std::pair<llvm::Value*, unsigned> Src =
653 EmitPointerWithAlignment(E->getArg(1));
654 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
655 unsigned Align = std::min(Dest.second, Src.second);
656 Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false);
657 return RValue::get(Dest.first);
658 }
659
660 case Builtin::BI__builtin_objc_memmove_collectable: {
661 Value *Address = EmitScalarExpr(E->getArg(0));
662 Value *SrcAddr = EmitScalarExpr(E->getArg(1));
663 Value *SizeVal = EmitScalarExpr(E->getArg(2));
664 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
665 Address, SrcAddr, SizeVal);
666 return RValue::get(Address);
667 }
668
669 case Builtin::BI__builtin___memmove_chk: {
670 // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
671 llvm::APSInt Size, DstSize;
672 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
673 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
674 break;
675 if (Size.ugt(DstSize))
676 break;
677 std::pair<llvm::Value*, unsigned> Dest =
678 EmitPointerWithAlignment(E->getArg(0));
679 std::pair<llvm::Value*, unsigned> Src =
680 EmitPointerWithAlignment(E->getArg(1));
681 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
682 unsigned Align = std::min(Dest.second, Src.second);
683 Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false);
684 return RValue::get(Dest.first);
685 }
686
687 case Builtin::BImemmove:
688 case Builtin::BI__builtin_memmove: {
689 std::pair<llvm::Value*, unsigned> Dest =
690 EmitPointerWithAlignment(E->getArg(0));
691 std::pair<llvm::Value*, unsigned> Src =
692 EmitPointerWithAlignment(E->getArg(1));
693 Value *SizeVal = EmitScalarExpr(E->getArg(2));
694 unsigned Align = std::min(Dest.second, Src.second);
695 Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false);
696 return RValue::get(Dest.first);
697 }
698 case Builtin::BImemset:
699 case Builtin::BI__builtin_memset: {
700 std::pair<llvm::Value*, unsigned> Dest =
701 EmitPointerWithAlignment(E->getArg(0));
702 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
703 Builder.getInt8Ty());
704 Value *SizeVal = EmitScalarExpr(E->getArg(2));
705 Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false);
706 return RValue::get(Dest.first);
707 }
708 case Builtin::BI__builtin___memset_chk: {
709 // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
710 llvm::APSInt Size, DstSize;
711 if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
712 !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
713 break;
714 if (Size.ugt(DstSize))
715 break;
716 std::pair<llvm::Value*, unsigned> Dest =
717 EmitPointerWithAlignment(E->getArg(0));
718 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
719 Builder.getInt8Ty());
720 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
721 Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false);
722 return RValue::get(Dest.first);
723 }
724 case Builtin::BI__builtin_dwarf_cfa: {
725 // The offset in bytes from the first argument to the CFA.
726 //
727 // Why on earth is this in the frontend? Is there any reason at
728 // all that the backend can't reasonably determine this while
729 // lowering llvm.eh.dwarf.cfa()?
730 //
731 // TODO: If there's a satisfactory reason, add a target hook for
732 // this instead of hard-coding 0, which is correct for most targets.
733 int32_t Offset = 0;
734
735 Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
736 return RValue::get(Builder.CreateCall(F,
737 llvm::ConstantInt::get(Int32Ty, Offset)));
738 }
739 case Builtin::BI__builtin_return_address: {
740 Value *Depth = EmitScalarExpr(E->getArg(0));
741 Depth = Builder.CreateIntCast(Depth, Int32Ty, false);
742 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
743 return RValue::get(Builder.CreateCall(F, Depth));
744 }
745 case Builtin::BI__builtin_frame_address: {
746 Value *Depth = EmitScalarExpr(E->getArg(0));
747 Depth = Builder.CreateIntCast(Depth, Int32Ty, false);
748 Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
749 return RValue::get(Builder.CreateCall(F, Depth));
750 }
751 case Builtin::BI__builtin_extract_return_addr: {
752 Value *Address = EmitScalarExpr(E->getArg(0));
753 Value *Result = getTargetHooks().decodeReturnAddress(*this, Address);
754 return RValue::get(Result);
755 }
756 case Builtin::BI__builtin_frob_return_addr: {
757 Value *Address = EmitScalarExpr(E->getArg(0));
758 Value *Result = getTargetHooks().encodeReturnAddress(*this, Address);
759 return RValue::get(Result);
760 }
761 case Builtin::BI__builtin_dwarf_sp_column: {
762 llvm::IntegerType *Ty
763 = cast<llvm::IntegerType>(ConvertType(E->getType()));
764 int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
765 if (Column == -1) {
766 CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column");
767 return RValue::get(llvm::UndefValue::get(Ty));
768 }
769 return RValue::get(llvm::ConstantInt::get(Ty, Column, true));
770 }
771 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
772 Value *Address = EmitScalarExpr(E->getArg(0));
773 if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address))
774 CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table");
775 return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
776 }
777 case Builtin::BI__builtin_eh_return: {
778 Value *Int = EmitScalarExpr(E->getArg(0));
779 Value *Ptr = EmitScalarExpr(E->getArg(1));
780
781 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
782 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
783 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
784 Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
785 ? Intrinsic::eh_return_i32
786 : Intrinsic::eh_return_i64);
787 Builder.CreateCall2(F, Int, Ptr);
788 Builder.CreateUnreachable();
789
790 // We do need to preserve an insertion point.
791 EmitBlock(createBasicBlock("builtin_eh_return.cont"));
792
793 return RValue::get(nullptr);
794 }
795 case Builtin::BI__builtin_unwind_init: {
796 Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
797 return RValue::get(Builder.CreateCall(F));
798 }
799 case Builtin::BI__builtin_extend_pointer: {
800 // Extends a pointer to the size of an _Unwind_Word, which is
801 // uint64_t on all platforms. Generally this gets poked into a
802 // register and eventually used as an address, so if the
803 // addressing registers are wider than pointers and the platform
804 // doesn't implicitly ignore high-order bits when doing
805 // addressing, we need to make sure we zext / sext based on
806 // the platform's expectations.
807 //
808 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
809
810 // Cast the pointer to intptr_t.
811 Value *Ptr = EmitScalarExpr(E->getArg(0));
812 Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
813
814 // If that's 64 bits, we're done.
815 if (IntPtrTy->getBitWidth() == 64)
816 return RValue::get(Result);
817
818 // Otherwise, ask the codegen data what to do.
819 if (getTargetHooks().extendPointerWithSExt())
820 return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext"));
821 else
822 return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext"));
823 }
824 case Builtin::BI__builtin_setjmp: {
825 // Buffer is a void**.
826 Value *Buf = EmitScalarExpr(E->getArg(0));
827
828 // Store the frame pointer to the setjmp buffer.
829 Value *FrameAddr =
830 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
831 ConstantInt::get(Int32Ty, 0));
832 Builder.CreateStore(FrameAddr, Buf);
833
834 // Store the stack pointer to the setjmp buffer.
835 Value *StackAddr =
836 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
837 Value *StackSaveSlot =
838 Builder.CreateGEP(Buf, ConstantInt::get(Int32Ty, 2));
839 Builder.CreateStore(StackAddr, StackSaveSlot);
840
841 // Call LLVM's EH setjmp, which is lightweight.
842 Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
843 Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
844 return RValue::get(Builder.CreateCall(F, Buf));
845 }
846 case Builtin::BI__builtin_longjmp: {
847 Value *Buf = EmitScalarExpr(E->getArg(0));
848 Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
849
850 // Call LLVM's EH longjmp, which is lightweight.
851 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
852
853 // longjmp doesn't return; mark this as unreachable.
854 Builder.CreateUnreachable();
855
856 // We do need to preserve an insertion point.
857 EmitBlock(createBasicBlock("longjmp.cont"));
858
859 return RValue::get(nullptr);
860 }
861 case Builtin::BI__sync_fetch_and_add:
862 case Builtin::BI__sync_fetch_and_sub:
863 case Builtin::BI__sync_fetch_and_or:
864 case Builtin::BI__sync_fetch_and_and:
865 case Builtin::BI__sync_fetch_and_xor:
866 case Builtin::BI__sync_add_and_fetch:
867 case Builtin::BI__sync_sub_and_fetch:
868 case Builtin::BI__sync_and_and_fetch:
869 case Builtin::BI__sync_or_and_fetch:
870 case Builtin::BI__sync_xor_and_fetch:
871 case Builtin::BI__sync_val_compare_and_swap:
872 case Builtin::BI__sync_bool_compare_and_swap:
873 case Builtin::BI__sync_lock_test_and_set:
874 case Builtin::BI__sync_lock_release:
875 case Builtin::BI__sync_swap:
876 llvm_unreachable("Shouldn't make it through sema");
877 case Builtin::BI__sync_fetch_and_add_1:
878 case Builtin::BI__sync_fetch_and_add_2:
879 case Builtin::BI__sync_fetch_and_add_4:
880 case Builtin::BI__sync_fetch_and_add_8:
881 case Builtin::BI__sync_fetch_and_add_16:
882 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E);
883 case Builtin::BI__sync_fetch_and_sub_1:
884 case Builtin::BI__sync_fetch_and_sub_2:
885 case Builtin::BI__sync_fetch_and_sub_4:
886 case Builtin::BI__sync_fetch_and_sub_8:
887 case Builtin::BI__sync_fetch_and_sub_16:
888 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E);
889 case Builtin::BI__sync_fetch_and_or_1:
890 case Builtin::BI__sync_fetch_and_or_2:
891 case Builtin::BI__sync_fetch_and_or_4:
892 case Builtin::BI__sync_fetch_and_or_8:
893 case Builtin::BI__sync_fetch_and_or_16:
894 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E);
895 case Builtin::BI__sync_fetch_and_and_1:
896 case Builtin::BI__sync_fetch_and_and_2:
897 case Builtin::BI__sync_fetch_and_and_4:
898 case Builtin::BI__sync_fetch_and_and_8:
899 case Builtin::BI__sync_fetch_and_and_16:
900 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E);
901 case Builtin::BI__sync_fetch_and_xor_1:
902 case Builtin::BI__sync_fetch_and_xor_2:
903 case Builtin::BI__sync_fetch_and_xor_4:
904 case Builtin::BI__sync_fetch_and_xor_8:
905 case Builtin::BI__sync_fetch_and_xor_16:
906 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E);
907
908 // Clang extensions: not overloaded yet.
909 case Builtin::BI__sync_fetch_and_min:
910 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E);
911 case Builtin::BI__sync_fetch_and_max:
912 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E);
913 case Builtin::BI__sync_fetch_and_umin:
914 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E);
915 case Builtin::BI__sync_fetch_and_umax:
916 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E);
917
918 case Builtin::BI__sync_add_and_fetch_1:
919 case Builtin::BI__sync_add_and_fetch_2:
920 case Builtin::BI__sync_add_and_fetch_4:
921 case Builtin::BI__sync_add_and_fetch_8:
922 case Builtin::BI__sync_add_and_fetch_16:
923 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E,
924 llvm::Instruction::Add);
925 case Builtin::BI__sync_sub_and_fetch_1:
926 case Builtin::BI__sync_sub_and_fetch_2:
927 case Builtin::BI__sync_sub_and_fetch_4:
928 case Builtin::BI__sync_sub_and_fetch_8:
929 case Builtin::BI__sync_sub_and_fetch_16:
930 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E,
931 llvm::Instruction::Sub);
932 case Builtin::BI__sync_and_and_fetch_1:
933 case Builtin::BI__sync_and_and_fetch_2:
934 case Builtin::BI__sync_and_and_fetch_4:
935 case Builtin::BI__sync_and_and_fetch_8:
936 case Builtin::BI__sync_and_and_fetch_16:
937 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E,
938 llvm::Instruction::And);
939 case Builtin::BI__sync_or_and_fetch_1:
940 case Builtin::BI__sync_or_and_fetch_2:
941 case Builtin::BI__sync_or_and_fetch_4:
942 case Builtin::BI__sync_or_and_fetch_8:
943 case Builtin::BI__sync_or_and_fetch_16:
944 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E,
945 llvm::Instruction::Or);
946 case Builtin::BI__sync_xor_and_fetch_1:
947 case Builtin::BI__sync_xor_and_fetch_2:
948 case Builtin::BI__sync_xor_and_fetch_4:
949 case Builtin::BI__sync_xor_and_fetch_8:
950 case Builtin::BI__sync_xor_and_fetch_16:
951 return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E,
952 llvm::Instruction::Xor);
953
954 case Builtin::BI__sync_val_compare_and_swap_1:
955 case Builtin::BI__sync_val_compare_and_swap_2:
956 case Builtin::BI__sync_val_compare_and_swap_4:
957 case Builtin::BI__sync_val_compare_and_swap_8:
958 case Builtin::BI__sync_val_compare_and_swap_16: {
959 QualType T = E->getType();
960 llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0));
961 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
962
963 llvm::IntegerType *IntType =
964 llvm::IntegerType::get(getLLVMContext(),
965 getContext().getTypeSize(T));
966 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
967
968 Value *Args[3];
969 Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType);
970 Args[1] = EmitScalarExpr(E->getArg(1));
971 llvm::Type *ValueType = Args[1]->getType();
972 Args[1] = EmitToInt(*this, Args[1], T, IntType);
973 Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType);
974
975 Value *Result = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2],
976 llvm::SequentiallyConsistent,
977 llvm::SequentiallyConsistent);
978 Result = Builder.CreateExtractValue(Result, 0);
979 Result = EmitFromInt(*this, Result, T, ValueType);
980 return RValue::get(Result);
981 }
982
983 case Builtin::BI__sync_bool_compare_and_swap_1:
984 case Builtin::BI__sync_bool_compare_and_swap_2:
985 case Builtin::BI__sync_bool_compare_and_swap_4:
986 case Builtin::BI__sync_bool_compare_and_swap_8:
987 case Builtin::BI__sync_bool_compare_and_swap_16: {
988 QualType T = E->getArg(1)->getType();
989 llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0));
990 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
991
992 llvm::IntegerType *IntType =
993 llvm::IntegerType::get(getLLVMContext(),
994 getContext().getTypeSize(T));
995 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
996
997 Value *Args[3];
998 Args[0] = Builder.CreateBitCast(DestPtr, IntPtrType);
999 Args[1] = EmitToInt(*this, EmitScalarExpr(E->getArg(1)), T, IntType);
1000 Args[2] = EmitToInt(*this, EmitScalarExpr(E->getArg(2)), T, IntType);
1001
1002 Value *Pair = Builder.CreateAtomicCmpXchg(Args[0], Args[1], Args[2],
1003 llvm::SequentiallyConsistent,
1004 llvm::SequentiallyConsistent);
1005 Value *Result = Builder.CreateExtractValue(Pair, 1);
1006 // zext bool to int.
1007 Result = Builder.CreateZExt(Result, ConvertType(E->getType()));
1008 return RValue::get(Result);
1009 }
1010
1011 case Builtin::BI__sync_swap_1:
1012 case Builtin::BI__sync_swap_2:
1013 case Builtin::BI__sync_swap_4:
1014 case Builtin::BI__sync_swap_8:
1015 case Builtin::BI__sync_swap_16:
1016 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
1017
1018 case Builtin::BI__sync_lock_test_and_set_1:
1019 case Builtin::BI__sync_lock_test_and_set_2:
1020 case Builtin::BI__sync_lock_test_and_set_4:
1021 case Builtin::BI__sync_lock_test_and_set_8:
1022 case Builtin::BI__sync_lock_test_and_set_16:
1023 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
1024
1025 case Builtin::BI__sync_lock_release_1:
1026 case Builtin::BI__sync_lock_release_2:
1027 case Builtin::BI__sync_lock_release_4:
1028 case Builtin::BI__sync_lock_release_8:
1029 case Builtin::BI__sync_lock_release_16: {
1030 Value *Ptr = EmitScalarExpr(E->getArg(0));
1031 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
1032 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
1033 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
1034 StoreSize.getQuantity() * 8);
1035 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
1036 llvm::StoreInst *Store =
1037 Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr);
1038 Store->setAlignment(StoreSize.getQuantity());
1039 Store->setAtomic(llvm::Release);
1040 return RValue::get(nullptr);
1041 }
1042
1043 case Builtin::BI__sync_synchronize: {
1044 // We assume this is supposed to correspond to a C++0x-style
1045 // sequentially-consistent fence (i.e. this is only usable for
1046 // synchonization, not device I/O or anything like that). This intrinsic
1047 // is really badly designed in the sense that in theory, there isn't
1048 // any way to safely use it... but in practice, it mostly works
1049 // to use it with non-atomic loads and stores to get acquire/release
1050 // semantics.
1051 Builder.CreateFence(llvm::SequentiallyConsistent);
1052 return RValue::get(nullptr);
1053 }
1054
1055 case Builtin::BI__c11_atomic_is_lock_free:
1056 case Builtin::BI__atomic_is_lock_free: {
1057 // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
1058 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
1059 // _Atomic(T) is always properly-aligned.
1060 const char *LibCallName = "__atomic_is_lock_free";
1061 CallArgList Args;
1062 Args.add(RValue::get(EmitScalarExpr(E->getArg(0))),
1063 getContext().getSizeType());
1064 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
1065 Args.add(RValue::get(EmitScalarExpr(E->getArg(1))),
1066 getContext().VoidPtrTy);
1067 else
1068 Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)),
1069 getContext().VoidPtrTy);
1070 const CGFunctionInfo &FuncInfo =
1071 CGM.getTypes().arrangeFreeFunctionCall(E->getType(), Args,
1072 FunctionType::ExtInfo(),
1073 RequiredArgs::All);
1074 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
1075 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
1076 return EmitCall(FuncInfo, Func, ReturnValueSlot(), Args);
1077 }
1078
1079 case Builtin::BI__atomic_test_and_set: {
1080 // Look at the argument type to determine whether this is a volatile
1081 // operation. The parameter type is always volatile.
1082 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
1083 bool Volatile =
1084 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
1085
1086 Value *Ptr = EmitScalarExpr(E->getArg(0));
1087 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
1088 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1089 Value *NewVal = Builder.getInt8(1);
1090 Value *Order = EmitScalarExpr(E->getArg(1));
1091 if (isa<llvm::ConstantInt>(Order)) {
1092 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1093 AtomicRMWInst *Result = nullptr;
1094 switch (ord) {
1095 case 0: // memory_order_relaxed
1096 default: // invalid order
1097 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1098 Ptr, NewVal,
1099 llvm::Monotonic);
1100 break;
1101 case 1: // memory_order_consume
1102 case 2: // memory_order_acquire
1103 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1104 Ptr, NewVal,
1105 llvm::Acquire);
1106 break;
1107 case 3: // memory_order_release
1108 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1109 Ptr, NewVal,
1110 llvm::Release);
1111 break;
1112 case 4: // memory_order_acq_rel
1113 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1114 Ptr, NewVal,
1115 llvm::AcquireRelease);
1116 break;
1117 case 5: // memory_order_seq_cst
1118 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1119 Ptr, NewVal,
1120 llvm::SequentiallyConsistent);
1121 break;
1122 }
1123 Result->setVolatile(Volatile);
1124 return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
1125 }
1126
1127 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1128
1129 llvm::BasicBlock *BBs[5] = {
1130 createBasicBlock("monotonic", CurFn),
1131 createBasicBlock("acquire", CurFn),
1132 createBasicBlock("release", CurFn),
1133 createBasicBlock("acqrel", CurFn),
1134 createBasicBlock("seqcst", CurFn)
1135 };
1136 llvm::AtomicOrdering Orders[5] = {
1137 llvm::Monotonic, llvm::Acquire, llvm::Release,
1138 llvm::AcquireRelease, llvm::SequentiallyConsistent
1139 };
1140
1141 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1142 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
1143
1144 Builder.SetInsertPoint(ContBB);
1145 PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set");
1146
1147 for (unsigned i = 0; i < 5; ++i) {
1148 Builder.SetInsertPoint(BBs[i]);
1149 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
1150 Ptr, NewVal, Orders[i]);
1151 RMW->setVolatile(Volatile);
1152 Result->addIncoming(RMW, BBs[i]);
1153 Builder.CreateBr(ContBB);
1154 }
1155
1156 SI->addCase(Builder.getInt32(0), BBs[0]);
1157 SI->addCase(Builder.getInt32(1), BBs[1]);
1158 SI->addCase(Builder.getInt32(2), BBs[1]);
1159 SI->addCase(Builder.getInt32(3), BBs[2]);
1160 SI->addCase(Builder.getInt32(4), BBs[3]);
1161 SI->addCase(Builder.getInt32(5), BBs[4]);
1162
1163 Builder.SetInsertPoint(ContBB);
1164 return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
1165 }
1166
1167 case Builtin::BI__atomic_clear: {
1168 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
1169 bool Volatile =
1170 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
1171
1172 Value *Ptr = EmitScalarExpr(E->getArg(0));
1173 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
1174 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
1175 Value *NewVal = Builder.getInt8(0);
1176 Value *Order = EmitScalarExpr(E->getArg(1));
1177 if (isa<llvm::ConstantInt>(Order)) {
1178 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1179 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
1180 Store->setAlignment(1);
1181 switch (ord) {
1182 case 0: // memory_order_relaxed
1183 default: // invalid order
1184 Store->setOrdering(llvm::Monotonic);
1185 break;
1186 case 3: // memory_order_release
1187 Store->setOrdering(llvm::Release);
1188 break;
1189 case 5: // memory_order_seq_cst
1190 Store->setOrdering(llvm::SequentiallyConsistent);
1191 break;
1192 }
1193 return RValue::get(nullptr);
1194 }
1195
1196 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1197
1198 llvm::BasicBlock *BBs[3] = {
1199 createBasicBlock("monotonic", CurFn),
1200 createBasicBlock("release", CurFn),
1201 createBasicBlock("seqcst", CurFn)
1202 };
1203 llvm::AtomicOrdering Orders[3] = {
1204 llvm::Monotonic, llvm::Release, llvm::SequentiallyConsistent
1205 };
1206
1207 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1208 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
1209
1210 for (unsigned i = 0; i < 3; ++i) {
1211 Builder.SetInsertPoint(BBs[i]);
1212 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
1213 Store->setAlignment(1);
1214 Store->setOrdering(Orders[i]);
1215 Builder.CreateBr(ContBB);
1216 }
1217
1218 SI->addCase(Builder.getInt32(0), BBs[0]);
1219 SI->addCase(Builder.getInt32(3), BBs[1]);
1220 SI->addCase(Builder.getInt32(5), BBs[2]);
1221
1222 Builder.SetInsertPoint(ContBB);
1223 return RValue::get(nullptr);
1224 }
1225
1226 case Builtin::BI__atomic_thread_fence:
1227 case Builtin::BI__atomic_signal_fence:
1228 case Builtin::BI__c11_atomic_thread_fence:
1229 case Builtin::BI__c11_atomic_signal_fence: {
1230 llvm::SynchronizationScope Scope;
1231 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
1232 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
1233 Scope = llvm::SingleThread;
1234 else
1235 Scope = llvm::CrossThread;
1236 Value *Order = EmitScalarExpr(E->getArg(0));
1237 if (isa<llvm::ConstantInt>(Order)) {
1238 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1239 switch (ord) {
1240 case 0: // memory_order_relaxed
1241 default: // invalid order
1242 break;
1243 case 1: // memory_order_consume
1244 case 2: // memory_order_acquire
1245 Builder.CreateFence(llvm::Acquire, Scope);
1246 break;
1247 case 3: // memory_order_release
1248 Builder.CreateFence(llvm::Release, Scope);
1249 break;
1250 case 4: // memory_order_acq_rel
1251 Builder.CreateFence(llvm::AcquireRelease, Scope);
1252 break;
1253 case 5: // memory_order_seq_cst
1254 Builder.CreateFence(llvm::SequentiallyConsistent, Scope);
1255 break;
1256 }
1257 return RValue::get(nullptr);
1258 }
1259
1260 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
1261 AcquireBB = createBasicBlock("acquire", CurFn);
1262 ReleaseBB = createBasicBlock("release", CurFn);
1263 AcqRelBB = createBasicBlock("acqrel", CurFn);
1264 SeqCstBB = createBasicBlock("seqcst", CurFn);
1265 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1266
1267 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1268 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
1269
1270 Builder.SetInsertPoint(AcquireBB);
1271 Builder.CreateFence(llvm::Acquire, Scope);
1272 Builder.CreateBr(ContBB);
1273 SI->addCase(Builder.getInt32(1), AcquireBB);
1274 SI->addCase(Builder.getInt32(2), AcquireBB);
1275
1276 Builder.SetInsertPoint(ReleaseBB);
1277 Builder.CreateFence(llvm::Release, Scope);
1278 Builder.CreateBr(ContBB);
1279 SI->addCase(Builder.getInt32(3), ReleaseBB);
1280
1281 Builder.SetInsertPoint(AcqRelBB);
1282 Builder.CreateFence(llvm::AcquireRelease, Scope);
1283 Builder.CreateBr(ContBB);
1284 SI->addCase(Builder.getInt32(4), AcqRelBB);
1285
1286 Builder.SetInsertPoint(SeqCstBB);
1287 Builder.CreateFence(llvm::SequentiallyConsistent, Scope);
1288 Builder.CreateBr(ContBB);
1289 SI->addCase(Builder.getInt32(5), SeqCstBB);
1290
1291 Builder.SetInsertPoint(ContBB);
1292 return RValue::get(nullptr);
1293 }
1294
1295 // Library functions with special handling.
1296 case Builtin::BIsqrt:
1297 case Builtin::BIsqrtf:
1298 case Builtin::BIsqrtl: {
1299 // Transform a call to sqrt* into a @llvm.sqrt.* intrinsic call, but only
1300 // in finite- or unsafe-math mode (the intrinsic has different semantics
1301 // for handling negative numbers compared to the library function, so
1302 // -fmath-errno=0 is not enough).
1303 if (!FD->hasAttr<ConstAttr>())
1304 break;
1305 if (!(CGM.getCodeGenOpts().UnsafeFPMath ||
1306 CGM.getCodeGenOpts().NoNaNsFPMath))
1307 break;
1308 Value *Arg0 = EmitScalarExpr(E->getArg(0));
1309 llvm::Type *ArgType = Arg0->getType();
1310 Value *F = CGM.getIntrinsic(Intrinsic::sqrt, ArgType);
1311 return RValue::get(Builder.CreateCall(F, Arg0));
1312 }
1313
1314 case Builtin::BIpow:
1315 case Builtin::BIpowf:
1316 case Builtin::BIpowl: {
1317 // Transform a call to pow* into a @llvm.pow.* intrinsic call.
1318 if (!FD->hasAttr<ConstAttr>())
1319 break;
1320 Value *Base = EmitScalarExpr(E->getArg(0));
1321 Value *Exponent = EmitScalarExpr(E->getArg(1));
1322 llvm::Type *ArgType = Base->getType();
1323 Value *F = CGM.getIntrinsic(Intrinsic::pow, ArgType);
1324 return RValue::get(Builder.CreateCall2(F, Base, Exponent));
1325 }
1326
1327 case Builtin::BIfma:
1328 case Builtin::BIfmaf:
1329 case Builtin::BIfmal:
1330 case Builtin::BI__builtin_fma:
1331 case Builtin::BI__builtin_fmaf:
1332 case Builtin::BI__builtin_fmal: {
1333 // Rewrite fma to intrinsic.
1334 Value *FirstArg = EmitScalarExpr(E->getArg(0));
1335 llvm::Type *ArgType = FirstArg->getType();
1336 Value *F = CGM.getIntrinsic(Intrinsic::fma, ArgType);
1337 return RValue::get(Builder.CreateCall3(F, FirstArg,
1338 EmitScalarExpr(E->getArg(1)),
1339 EmitScalarExpr(E->getArg(2))));
1340 }
1341
1342 case Builtin::BI__builtin_signbit:
1343 case Builtin::BI__builtin_signbitf:
1344 case Builtin::BI__builtin_signbitl: {
1345 LLVMContext &C = CGM.getLLVMContext();
1346
1347 Value *Arg = EmitScalarExpr(E->getArg(0));
1348 llvm::Type *ArgTy = Arg->getType();
1349 if (ArgTy->isPPC_FP128Ty())
1350 break; // FIXME: I'm not sure what the right implementation is here.
1351 int ArgWidth = ArgTy->getPrimitiveSizeInBits();
1352 llvm::Type *ArgIntTy = llvm::IntegerType::get(C, ArgWidth);
1353 Value *BCArg = Builder.CreateBitCast(Arg, ArgIntTy);
1354 Value *ZeroCmp = llvm::Constant::getNullValue(ArgIntTy);
1355 Value *Result = Builder.CreateICmpSLT(BCArg, ZeroCmp);
1356 return RValue::get(Builder.CreateZExt(Result, ConvertType(E->getType())));
1357 }
1358 case Builtin::BI__builtin_annotation: {
1359 llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
1360 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
1361 AnnVal->getType());
1362
1363 // Get the annotation string, go through casts. Sema requires this to be a
1364 // non-wide string literal, potentially casted, so the cast<> is safe.
1365 const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts();
1366 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
1367 return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc()));
1368 }
1369 case Builtin::BI__builtin_addcb:
1370 case Builtin::BI__builtin_addcs:
1371 case Builtin::BI__builtin_addc:
1372 case Builtin::BI__builtin_addcl:
1373 case Builtin::BI__builtin_addcll:
1374 case Builtin::BI__builtin_subcb:
1375 case Builtin::BI__builtin_subcs:
1376 case Builtin::BI__builtin_subc:
1377 case Builtin::BI__builtin_subcl:
1378 case Builtin::BI__builtin_subcll: {
1379
1380 // We translate all of these builtins from expressions of the form:
1381 // int x = ..., y = ..., carryin = ..., carryout, result;
1382 // result = __builtin_addc(x, y, carryin, &carryout);
1383 //
1384 // to LLVM IR of the form:
1385 //
1386 // %tmp1 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
1387 // %tmpsum1 = extractvalue {i32, i1} %tmp1, 0
1388 // %carry1 = extractvalue {i32, i1} %tmp1, 1
1389 // %tmp2 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %tmpsum1,
1390 // i32 %carryin)
1391 // %result = extractvalue {i32, i1} %tmp2, 0
1392 // %carry2 = extractvalue {i32, i1} %tmp2, 1
1393 // %tmp3 = or i1 %carry1, %carry2
1394 // %tmp4 = zext i1 %tmp3 to i32
1395 // store i32 %tmp4, i32* %carryout
1396
1397 // Scalarize our inputs.
1398 llvm::Value *X = EmitScalarExpr(E->getArg(0));
1399 llvm::Value *Y = EmitScalarExpr(E->getArg(1));
1400 llvm::Value *Carryin = EmitScalarExpr(E->getArg(2));
1401 std::pair<llvm::Value*, unsigned> CarryOutPtr =
1402 EmitPointerWithAlignment(E->getArg(3));
1403
1404 // Decide if we are lowering to a uadd.with.overflow or usub.with.overflow.
1405 llvm::Intrinsic::ID IntrinsicId;
1406 switch (BuiltinID) {
1407 default: llvm_unreachable("Unknown multiprecision builtin id.");
1408 case Builtin::BI__builtin_addcb:
1409 case Builtin::BI__builtin_addcs:
1410 case Builtin::BI__builtin_addc:
1411 case Builtin::BI__builtin_addcl:
1412 case Builtin::BI__builtin_addcll:
1413 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
1414 break;
1415 case Builtin::BI__builtin_subcb:
1416 case Builtin::BI__builtin_subcs:
1417 case Builtin::BI__builtin_subc:
1418 case Builtin::BI__builtin_subcl:
1419 case Builtin::BI__builtin_subcll:
1420 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
1421 break;
1422 }
1423
1424 // Construct our resulting LLVM IR expression.
1425 llvm::Value *Carry1;
1426 llvm::Value *Sum1 = EmitOverflowIntrinsic(*this, IntrinsicId,
1427 X, Y, Carry1);
1428 llvm::Value *Carry2;
1429 llvm::Value *Sum2 = EmitOverflowIntrinsic(*this, IntrinsicId,
1430 Sum1, Carryin, Carry2);
1431 llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
1432 X->getType());
1433 llvm::StoreInst *CarryOutStore = Builder.CreateStore(CarryOut,
1434 CarryOutPtr.first);
1435 CarryOutStore->setAlignment(CarryOutPtr.second);
1436 return RValue::get(Sum2);
1437 }
1438 case Builtin::BI__builtin_uadd_overflow:
1439 case Builtin::BI__builtin_uaddl_overflow:
1440 case Builtin::BI__builtin_uaddll_overflow:
1441 case Builtin::BI__builtin_usub_overflow:
1442 case Builtin::BI__builtin_usubl_overflow:
1443 case Builtin::BI__builtin_usubll_overflow:
1444 case Builtin::BI__builtin_umul_overflow:
1445 case Builtin::BI__builtin_umull_overflow:
1446 case Builtin::BI__builtin_umulll_overflow:
1447 case Builtin::BI__builtin_sadd_overflow:
1448 case Builtin::BI__builtin_saddl_overflow:
1449 case Builtin::BI__builtin_saddll_overflow:
1450 case Builtin::BI__builtin_ssub_overflow:
1451 case Builtin::BI__builtin_ssubl_overflow:
1452 case Builtin::BI__builtin_ssubll_overflow:
1453 case Builtin::BI__builtin_smul_overflow:
1454 case Builtin::BI__builtin_smull_overflow:
1455 case Builtin::BI__builtin_smulll_overflow: {
1456
1457 // We translate all of these builtins directly to the relevant llvm IR node.
1458
1459 // Scalarize our inputs.
1460 llvm::Value *X = EmitScalarExpr(E->getArg(0));
1461 llvm::Value *Y = EmitScalarExpr(E->getArg(1));
1462 std::pair<llvm::Value *, unsigned> SumOutPtr =
1463 EmitPointerWithAlignment(E->getArg(2));
1464
1465 // Decide which of the overflow intrinsics we are lowering to:
1466 llvm::Intrinsic::ID IntrinsicId;
1467 switch (BuiltinID) {
1468 default: llvm_unreachable("Unknown security overflow builtin id.");
1469 case Builtin::BI__builtin_uadd_overflow:
1470 case Builtin::BI__builtin_uaddl_overflow:
1471 case Builtin::BI__builtin_uaddll_overflow:
1472 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
1473 break;
1474 case Builtin::BI__builtin_usub_overflow:
1475 case Builtin::BI__builtin_usubl_overflow:
1476 case Builtin::BI__builtin_usubll_overflow:
1477 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
1478 break;
1479 case Builtin::BI__builtin_umul_overflow:
1480 case Builtin::BI__builtin_umull_overflow:
1481 case Builtin::BI__builtin_umulll_overflow:
1482 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
1483 break;
1484 case Builtin::BI__builtin_sadd_overflow:
1485 case Builtin::BI__builtin_saddl_overflow:
1486 case Builtin::BI__builtin_saddll_overflow:
1487 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
1488 break;
1489 case Builtin::BI__builtin_ssub_overflow:
1490 case Builtin::BI__builtin_ssubl_overflow:
1491 case Builtin::BI__builtin_ssubll_overflow:
1492 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
1493 break;
1494 case Builtin::BI__builtin_smul_overflow:
1495 case Builtin::BI__builtin_smull_overflow:
1496 case Builtin::BI__builtin_smulll_overflow:
1497 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
1498 break;
1499 }
1500
1501
1502 llvm::Value *Carry;
1503 llvm::Value *Sum = EmitOverflowIntrinsic(*this, IntrinsicId, X, Y, Carry);
1504 llvm::StoreInst *SumOutStore = Builder.CreateStore(Sum, SumOutPtr.first);
1505 SumOutStore->setAlignment(SumOutPtr.second);
1506
1507 return RValue::get(Carry);
1508 }
1509 case Builtin::BI__builtin_addressof:
1510 return RValue::get(EmitLValue(E->getArg(0)).getAddress());
1511 case Builtin::BI__builtin_operator_new:
1512 return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(),
1513 E->getArg(0), false);
1514 case Builtin::BI__builtin_operator_delete:
1515 return EmitBuiltinNewDeleteCall(FD->getType()->castAs<FunctionProtoType>(),
1516 E->getArg(0), true);
1517 case Builtin::BI__noop:
1518 return RValue::get(nullptr);
1519 case Builtin::BI_InterlockedExchange:
1520 case Builtin::BI_InterlockedExchangePointer:
1521 return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
1522 case Builtin::BI_InterlockedCompareExchangePointer: {
1523 llvm::Type *RTy;
1524 llvm::IntegerType *IntType =
1525 IntegerType::get(getLLVMContext(),
1526 getContext().getTypeSize(E->getType()));
1527 llvm::Type *IntPtrType = IntType->getPointerTo();
1528
1529 llvm::Value *Destination =
1530 Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), IntPtrType);
1531
1532 llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
1533 RTy = Exchange->getType();
1534 Exchange = Builder.CreatePtrToInt(Exchange, IntType);
1535
1536 llvm::Value *Comparand =
1537 Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
1538
1539 auto Result = Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
1540 SequentiallyConsistent,
1541 SequentiallyConsistent);
1542 Result->setVolatile(true);
1543
1544 return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
1545 0),
1546 RTy));
1547 }
1548 case Builtin::BI_InterlockedCompareExchange: {
1549 AtomicCmpXchgInst *CXI = Builder.CreateAtomicCmpXchg(
1550 EmitScalarExpr(E->getArg(0)),
1551 EmitScalarExpr(E->getArg(2)),
1552 EmitScalarExpr(E->getArg(1)),
1553 SequentiallyConsistent,
1554 SequentiallyConsistent);
1555 CXI->setVolatile(true);
1556 return RValue::get(Builder.CreateExtractValue(CXI, 0));
1557 }
1558 case Builtin::BI_InterlockedIncrement: {
1559 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
1560 AtomicRMWInst::Add,
1561 EmitScalarExpr(E->getArg(0)),
1562 ConstantInt::get(Int32Ty, 1),
1563 llvm::SequentiallyConsistent);
1564 RMWI->setVolatile(true);
1565 return RValue::get(Builder.CreateAdd(RMWI, ConstantInt::get(Int32Ty, 1)));
1566 }
1567 case Builtin::BI_InterlockedDecrement: {
1568 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
1569 AtomicRMWInst::Sub,
1570 EmitScalarExpr(E->getArg(0)),
1571 ConstantInt::get(Int32Ty, 1),
1572 llvm::SequentiallyConsistent);
1573 RMWI->setVolatile(true);
1574 return RValue::get(Builder.CreateSub(RMWI, ConstantInt::get(Int32Ty, 1)));
1575 }
1576 case Builtin::BI_InterlockedExchangeAdd: {
1577 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
1578 AtomicRMWInst::Add,
1579 EmitScalarExpr(E->getArg(0)),
1580 EmitScalarExpr(E->getArg(1)),
1581 llvm::SequentiallyConsistent);
1582 RMWI->setVolatile(true);
1583 return RValue::get(RMWI);
1584 }
1585 }
1586
1587 // If this is an alias for a lib function (e.g. __builtin_sin), emit
1588 // the call using the normal call path, but using the unmangled
1589 // version of the function name.
1590 if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
1591 return emitLibraryCall(*this, FD, E,
1592 CGM.getBuiltinLibFunction(FD, BuiltinID));
1593
1594 // If this is a predefined lib function (e.g. malloc), emit the call
1595 // using exactly the normal call path.
1596 if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
1597 return emitLibraryCall(*this, FD, E, EmitScalarExpr(E->getCallee()));
1598
1599 // See if we have a target specific intrinsic.
1600 const char *Name = getContext().BuiltinInfo.GetName(BuiltinID);
1601 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
1602 if (const char *Prefix =
1603 llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch())) {
1604 IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix, Name);
1605 // NOTE we dont need to perform a compatibility flag check here since the
1606 // intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
1607 // MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
1608 if (IntrinsicID == Intrinsic::not_intrinsic)
1609 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix, Name);
1610 }
1611
1612 if (IntrinsicID != Intrinsic::not_intrinsic) {
1613 SmallVector<Value*, 16> Args;
1614
1615 // Find out if any arguments are required to be integer constant
1616 // expressions.
1617 unsigned ICEArguments = 0;
1618 ASTContext::GetBuiltinTypeError Error;
1619 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
1620 assert(Error == ASTContext::GE_None && "Should not codegen an error");
1621
1622 Function *F = CGM.getIntrinsic(IntrinsicID);
1623 llvm::FunctionType *FTy = F->getFunctionType();
1624
1625 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
1626 Value *ArgValue;
1627 // If this is a normal argument, just emit it as a scalar.
1628 if ((ICEArguments & (1 << i)) == 0) {
1629 ArgValue = EmitScalarExpr(E->getArg(i));
1630 } else {
1631 // If this is required to be a constant, constant fold it so that we
1632 // know that the generated intrinsic gets a ConstantInt.
1633 llvm::APSInt Result;
1634 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext());
1635 assert(IsConst && "Constant arg isn't actually constant?");
1636 (void)IsConst;
1637 ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
1638 }
1639
1640 // If the intrinsic arg type is different from the builtin arg type
1641 // we need to do a bit cast.
1642 llvm::Type *PTy = FTy->getParamType(i);
1643 if (PTy != ArgValue->getType()) {
1644 assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
1645 "Must be able to losslessly bit cast to param");
1646 ArgValue = Builder.CreateBitCast(ArgValue, PTy);
1647 }
1648
1649 Args.push_back(ArgValue);
1650 }
1651
1652 Value *V = Builder.CreateCall(F, Args);
1653 QualType BuiltinRetType = E->getType();
1654
1655 llvm::Type *RetTy = VoidTy;
1656 if (!BuiltinRetType->isVoidType())
1657 RetTy = ConvertType(BuiltinRetType);
1658
1659 if (RetTy != V->getType()) {
1660 assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&
1661 "Must be able to losslessly bit cast result type");
1662 V = Builder.CreateBitCast(V, RetTy);
1663 }
1664
1665 return RValue::get(V);
1666 }
1667
1668 // See if we have a target specific builtin that needs to be lowered.
1669 if (Value *V = EmitTargetBuiltinExpr(BuiltinID, E))
1670 return RValue::get(V);
1671
1672 ErrorUnsupported(E, "builtin function");
1673
1674 // Unknown builtin, for now just dump it out and return undef.
1675 return GetUndefRValue(E->getType());
1676 }
1677
EmitTargetBuiltinExpr(unsigned BuiltinID,const CallExpr * E)1678 Value *CodeGenFunction::EmitTargetBuiltinExpr(unsigned BuiltinID,
1679 const CallExpr *E) {
1680 switch (getTarget().getTriple().getArch()) {
1681 case llvm::Triple::arm:
1682 case llvm::Triple::armeb:
1683 case llvm::Triple::thumb:
1684 case llvm::Triple::thumbeb:
1685 return EmitARMBuiltinExpr(BuiltinID, E);
1686 case llvm::Triple::aarch64:
1687 case llvm::Triple::aarch64_be:
1688 case llvm::Triple::arm64:
1689 case llvm::Triple::arm64_be:
1690 return EmitAArch64BuiltinExpr(BuiltinID, E);
1691 case llvm::Triple::x86:
1692 case llvm::Triple::x86_64:
1693 return EmitX86BuiltinExpr(BuiltinID, E);
1694 case llvm::Triple::ppc:
1695 case llvm::Triple::ppc64:
1696 case llvm::Triple::ppc64le:
1697 return EmitPPCBuiltinExpr(BuiltinID, E);
1698 case llvm::Triple::r600:
1699 return EmitR600BuiltinExpr(BuiltinID, E);
1700 default:
1701 return nullptr;
1702 }
1703 }
1704
GetNeonType(CodeGenFunction * CGF,NeonTypeFlags TypeFlags,bool V1Ty=false)1705 static llvm::VectorType *GetNeonType(CodeGenFunction *CGF,
1706 NeonTypeFlags TypeFlags,
1707 bool V1Ty=false) {
1708 int IsQuad = TypeFlags.isQuad();
1709 switch (TypeFlags.getEltType()) {
1710 case NeonTypeFlags::Int8:
1711 case NeonTypeFlags::Poly8:
1712 return llvm::VectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
1713 case NeonTypeFlags::Int16:
1714 case NeonTypeFlags::Poly16:
1715 case NeonTypeFlags::Float16:
1716 return llvm::VectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
1717 case NeonTypeFlags::Int32:
1718 return llvm::VectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad));
1719 case NeonTypeFlags::Int64:
1720 case NeonTypeFlags::Poly64:
1721 return llvm::VectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad));
1722 case NeonTypeFlags::Poly128:
1723 // FIXME: i128 and f128 doesn't get fully support in Clang and llvm.
1724 // There is a lot of i128 and f128 API missing.
1725 // so we use v16i8 to represent poly128 and get pattern matched.
1726 return llvm::VectorType::get(CGF->Int8Ty, 16);
1727 case NeonTypeFlags::Float32:
1728 return llvm::VectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad));
1729 case NeonTypeFlags::Float64:
1730 return llvm::VectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad));
1731 }
1732 llvm_unreachable("Unknown vector element type!");
1733 }
1734
EmitNeonSplat(Value * V,Constant * C)1735 Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
1736 unsigned nElts = cast<llvm::VectorType>(V->getType())->getNumElements();
1737 Value* SV = llvm::ConstantVector::getSplat(nElts, C);
1738 return Builder.CreateShuffleVector(V, V, SV, "lane");
1739 }
1740
EmitNeonCall(Function * F,SmallVectorImpl<Value * > & Ops,const char * name,unsigned shift,bool rightshift)1741 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
1742 const char *name,
1743 unsigned shift, bool rightshift) {
1744 unsigned j = 0;
1745 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
1746 ai != ae; ++ai, ++j)
1747 if (shift > 0 && shift == j)
1748 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
1749 else
1750 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
1751
1752 return Builder.CreateCall(F, Ops, name);
1753 }
1754
EmitNeonShiftVector(Value * V,llvm::Type * Ty,bool neg)1755 Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
1756 bool neg) {
1757 int SV = cast<ConstantInt>(V)->getSExtValue();
1758
1759 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
1760 llvm::Constant *C = ConstantInt::get(VTy->getElementType(), neg ? -SV : SV);
1761 return llvm::ConstantVector::getSplat(VTy->getNumElements(), C);
1762 }
1763
1764 // \brief Right-shift a vector by a constant.
EmitNeonRShiftImm(Value * Vec,Value * Shift,llvm::Type * Ty,bool usgn,const char * name)1765 Value *CodeGenFunction::EmitNeonRShiftImm(Value *Vec, Value *Shift,
1766 llvm::Type *Ty, bool usgn,
1767 const char *name) {
1768 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
1769
1770 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
1771 int EltSize = VTy->getScalarSizeInBits();
1772
1773 Vec = Builder.CreateBitCast(Vec, Ty);
1774
1775 // lshr/ashr are undefined when the shift amount is equal to the vector
1776 // element size.
1777 if (ShiftAmt == EltSize) {
1778 if (usgn) {
1779 // Right-shifting an unsigned value by its size yields 0.
1780 llvm::Constant *Zero = ConstantInt::get(VTy->getElementType(), 0);
1781 return llvm::ConstantVector::getSplat(VTy->getNumElements(), Zero);
1782 } else {
1783 // Right-shifting a signed value by its size is equivalent
1784 // to a shift of size-1.
1785 --ShiftAmt;
1786 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
1787 }
1788 }
1789
1790 Shift = EmitNeonShiftVector(Shift, Ty, false);
1791 if (usgn)
1792 return Builder.CreateLShr(Vec, Shift, name);
1793 else
1794 return Builder.CreateAShr(Vec, Shift, name);
1795 }
1796
1797 /// GetPointeeAlignment - Given an expression with a pointer type, find the
1798 /// alignment of the type referenced by the pointer. Skip over implicit
1799 /// casts.
1800 std::pair<llvm::Value*, unsigned>
EmitPointerWithAlignment(const Expr * Addr)1801 CodeGenFunction::EmitPointerWithAlignment(const Expr *Addr) {
1802 assert(Addr->getType()->isPointerType());
1803 Addr = Addr->IgnoreParens();
1804 if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Addr)) {
1805 if ((ICE->getCastKind() == CK_BitCast || ICE->getCastKind() == CK_NoOp) &&
1806 ICE->getSubExpr()->getType()->isPointerType()) {
1807 std::pair<llvm::Value*, unsigned> Ptr =
1808 EmitPointerWithAlignment(ICE->getSubExpr());
1809 Ptr.first = Builder.CreateBitCast(Ptr.first,
1810 ConvertType(Addr->getType()));
1811 return Ptr;
1812 } else if (ICE->getCastKind() == CK_ArrayToPointerDecay) {
1813 LValue LV = EmitLValue(ICE->getSubExpr());
1814 unsigned Align = LV.getAlignment().getQuantity();
1815 if (!Align) {
1816 // FIXME: Once LValues are fixed to always set alignment,
1817 // zap this code.
1818 QualType PtTy = ICE->getSubExpr()->getType();
1819 if (!PtTy->isIncompleteType())
1820 Align = getContext().getTypeAlignInChars(PtTy).getQuantity();
1821 else
1822 Align = 1;
1823 }
1824 return std::make_pair(LV.getAddress(), Align);
1825 }
1826 }
1827 if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Addr)) {
1828 if (UO->getOpcode() == UO_AddrOf) {
1829 LValue LV = EmitLValue(UO->getSubExpr());
1830 unsigned Align = LV.getAlignment().getQuantity();
1831 if (!Align) {
1832 // FIXME: Once LValues are fixed to always set alignment,
1833 // zap this code.
1834 QualType PtTy = UO->getSubExpr()->getType();
1835 if (!PtTy->isIncompleteType())
1836 Align = getContext().getTypeAlignInChars(PtTy).getQuantity();
1837 else
1838 Align = 1;
1839 }
1840 return std::make_pair(LV.getAddress(), Align);
1841 }
1842 }
1843
1844 unsigned Align = 1;
1845 QualType PtTy = Addr->getType()->getPointeeType();
1846 if (!PtTy->isIncompleteType())
1847 Align = getContext().getTypeAlignInChars(PtTy).getQuantity();
1848
1849 return std::make_pair(EmitScalarExpr(Addr), Align);
1850 }
1851
1852 enum {
1853 AddRetType = (1 << 0),
1854 Add1ArgType = (1 << 1),
1855 Add2ArgTypes = (1 << 2),
1856
1857 VectorizeRetType = (1 << 3),
1858 VectorizeArgTypes = (1 << 4),
1859
1860 InventFloatType = (1 << 5),
1861 UnsignedAlts = (1 << 6),
1862
1863 Use64BitVectors = (1 << 7),
1864 Use128BitVectors = (1 << 8),
1865
1866 Vectorize1ArgType = Add1ArgType | VectorizeArgTypes,
1867 VectorRet = AddRetType | VectorizeRetType,
1868 VectorRetGetArgs01 =
1869 AddRetType | Add2ArgTypes | VectorizeRetType | VectorizeArgTypes,
1870 FpCmpzModifiers =
1871 AddRetType | VectorizeRetType | Add1ArgType | InventFloatType
1872 };
1873
1874 struct NeonIntrinsicInfo {
1875 unsigned BuiltinID;
1876 unsigned LLVMIntrinsic;
1877 unsigned AltLLVMIntrinsic;
1878 const char *NameHint;
1879 unsigned TypeModifier;
1880
operator <NeonIntrinsicInfo1881 bool operator<(unsigned RHSBuiltinID) const {
1882 return BuiltinID < RHSBuiltinID;
1883 }
1884 };
1885
1886 #define NEONMAP0(NameBase) \
1887 { NEON::BI__builtin_neon_ ## NameBase, 0, 0, #NameBase, 0 }
1888
1889 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
1890 { NEON:: BI__builtin_neon_ ## NameBase, \
1891 Intrinsic::LLVMIntrinsic, 0, #NameBase, TypeModifier }
1892
1893 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
1894 { NEON:: BI__builtin_neon_ ## NameBase, \
1895 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
1896 #NameBase, TypeModifier }
1897
1898 static NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = {
1899 NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
1900 NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts),
1901 NEONMAP1(vabs_v, arm_neon_vabs, 0),
1902 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
1903 NEONMAP0(vaddhn_v),
1904 NEONMAP1(vaesdq_v, arm_neon_aesd, 0),
1905 NEONMAP1(vaeseq_v, arm_neon_aese, 0),
1906 NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0),
1907 NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0),
1908 NEONMAP1(vbsl_v, arm_neon_vbsl, AddRetType),
1909 NEONMAP1(vbslq_v, arm_neon_vbsl, AddRetType),
1910 NEONMAP1(vcage_v, arm_neon_vacge, 0),
1911 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
1912 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
1913 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
1914 NEONMAP1(vcale_v, arm_neon_vacge, 0),
1915 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
1916 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
1917 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
1918 NEONMAP1(vcls_v, arm_neon_vcls, Add1ArgType),
1919 NEONMAP1(vclsq_v, arm_neon_vcls, Add1ArgType),
1920 NEONMAP1(vclz_v, ctlz, Add1ArgType),
1921 NEONMAP1(vclzq_v, ctlz, Add1ArgType),
1922 NEONMAP1(vcnt_v, ctpop, Add1ArgType),
1923 NEONMAP1(vcntq_v, ctpop, Add1ArgType),
1924 NEONMAP1(vcvt_f16_v, arm_neon_vcvtfp2hf, 0),
1925 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
1926 NEONMAP0(vcvt_f32_v),
1927 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
1928 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
1929 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
1930 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
1931 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
1932 NEONMAP0(vcvt_s32_v),
1933 NEONMAP0(vcvt_s64_v),
1934 NEONMAP0(vcvt_u32_v),
1935 NEONMAP0(vcvt_u64_v),
1936 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
1937 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
1938 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
1939 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
1940 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
1941 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
1942 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
1943 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
1944 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
1945 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
1946 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
1947 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
1948 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
1949 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
1950 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
1951 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
1952 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
1953 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
1954 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
1955 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
1956 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
1957 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
1958 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
1959 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
1960 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
1961 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
1962 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
1963 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
1964 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
1965 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
1966 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
1967 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
1968 NEONMAP0(vcvtq_f32_v),
1969 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
1970 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
1971 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
1972 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
1973 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
1974 NEONMAP0(vcvtq_s32_v),
1975 NEONMAP0(vcvtq_s64_v),
1976 NEONMAP0(vcvtq_u32_v),
1977 NEONMAP0(vcvtq_u64_v),
1978 NEONMAP0(vext_v),
1979 NEONMAP0(vextq_v),
1980 NEONMAP0(vfma_v),
1981 NEONMAP0(vfmaq_v),
1982 NEONMAP2(vhadd_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
1983 NEONMAP2(vhaddq_v, arm_neon_vhaddu, arm_neon_vhadds, Add1ArgType | UnsignedAlts),
1984 NEONMAP2(vhsub_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
1985 NEONMAP2(vhsubq_v, arm_neon_vhsubu, arm_neon_vhsubs, Add1ArgType | UnsignedAlts),
1986 NEONMAP0(vld1_dup_v),
1987 NEONMAP1(vld1_v, arm_neon_vld1, 0),
1988 NEONMAP0(vld1q_dup_v),
1989 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
1990 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
1991 NEONMAP1(vld2_v, arm_neon_vld2, 0),
1992 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
1993 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
1994 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
1995 NEONMAP1(vld3_v, arm_neon_vld3, 0),
1996 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
1997 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
1998 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
1999 NEONMAP1(vld4_v, arm_neon_vld4, 0),
2000 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
2001 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
2002 NEONMAP2(vmax_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
2003 NEONMAP2(vmaxq_v, arm_neon_vmaxu, arm_neon_vmaxs, Add1ArgType | UnsignedAlts),
2004 NEONMAP2(vmin_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
2005 NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
2006 NEONMAP0(vmovl_v),
2007 NEONMAP0(vmovn_v),
2008 NEONMAP1(vmul_v, arm_neon_vmulp, Add1ArgType),
2009 NEONMAP0(vmull_v),
2010 NEONMAP1(vmulq_v, arm_neon_vmulp, Add1ArgType),
2011 NEONMAP2(vpadal_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
2012 NEONMAP2(vpadalq_v, arm_neon_vpadalu, arm_neon_vpadals, UnsignedAlts),
2013 NEONMAP1(vpadd_v, arm_neon_vpadd, Add1ArgType),
2014 NEONMAP2(vpaddl_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
2015 NEONMAP2(vpaddlq_v, arm_neon_vpaddlu, arm_neon_vpaddls, UnsignedAlts),
2016 NEONMAP1(vpaddq_v, arm_neon_vpadd, Add1ArgType),
2017 NEONMAP2(vpmax_v, arm_neon_vpmaxu, arm_neon_vpmaxs, Add1ArgType | UnsignedAlts),
2018 NEONMAP2(vpmin_v, arm_neon_vpminu, arm_neon_vpmins, Add1ArgType | UnsignedAlts),
2019 NEONMAP1(vqabs_v, arm_neon_vqabs, Add1ArgType),
2020 NEONMAP1(vqabsq_v, arm_neon_vqabs, Add1ArgType),
2021 NEONMAP2(vqadd_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts),
2022 NEONMAP2(vqaddq_v, arm_neon_vqaddu, arm_neon_vqadds, Add1ArgType | UnsignedAlts),
2023 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0),
2024 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0),
2025 NEONMAP1(vqdmulh_v, arm_neon_vqdmulh, Add1ArgType),
2026 NEONMAP1(vqdmulhq_v, arm_neon_vqdmulh, Add1ArgType),
2027 NEONMAP1(vqdmull_v, arm_neon_vqdmull, Add1ArgType),
2028 NEONMAP2(vqmovn_v, arm_neon_vqmovnu, arm_neon_vqmovns, Add1ArgType | UnsignedAlts),
2029 NEONMAP1(vqmovun_v, arm_neon_vqmovnsu, Add1ArgType),
2030 NEONMAP1(vqneg_v, arm_neon_vqneg, Add1ArgType),
2031 NEONMAP1(vqnegq_v, arm_neon_vqneg, Add1ArgType),
2032 NEONMAP1(vqrdmulh_v, arm_neon_vqrdmulh, Add1ArgType),
2033 NEONMAP1(vqrdmulhq_v, arm_neon_vqrdmulh, Add1ArgType),
2034 NEONMAP2(vqrshl_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
2035 NEONMAP2(vqrshlq_v, arm_neon_vqrshiftu, arm_neon_vqrshifts, Add1ArgType | UnsignedAlts),
2036 NEONMAP2(vqshl_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
2037 NEONMAP2(vqshl_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
2038 NEONMAP2(vqshlq_n_v, arm_neon_vqshiftu, arm_neon_vqshifts, UnsignedAlts),
2039 NEONMAP2(vqshlq_v, arm_neon_vqshiftu, arm_neon_vqshifts, Add1ArgType | UnsignedAlts),
2040 NEONMAP2(vqsub_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts),
2041 NEONMAP2(vqsubq_v, arm_neon_vqsubu, arm_neon_vqsubs, Add1ArgType | UnsignedAlts),
2042 NEONMAP1(vraddhn_v, arm_neon_vraddhn, Add1ArgType),
2043 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
2044 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
2045 NEONMAP1(vrecps_v, arm_neon_vrecps, Add1ArgType),
2046 NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
2047 NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
2048 NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
2049 NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
2050 NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
2051 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
2052 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
2053 NEONMAP1(vrsqrts_v, arm_neon_vrsqrts, Add1ArgType),
2054 NEONMAP1(vrsqrtsq_v, arm_neon_vrsqrts, Add1ArgType),
2055 NEONMAP1(vrsubhn_v, arm_neon_vrsubhn, Add1ArgType),
2056 NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0),
2057 NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0),
2058 NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0),
2059 NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0),
2060 NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0),
2061 NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0),
2062 NEONMAP0(vshl_n_v),
2063 NEONMAP2(vshl_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
2064 NEONMAP0(vshll_n_v),
2065 NEONMAP0(vshlq_n_v),
2066 NEONMAP2(vshlq_v, arm_neon_vshiftu, arm_neon_vshifts, Add1ArgType | UnsignedAlts),
2067 NEONMAP0(vshr_n_v),
2068 NEONMAP0(vshrn_n_v),
2069 NEONMAP0(vshrq_n_v),
2070 NEONMAP1(vst1_v, arm_neon_vst1, 0),
2071 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
2072 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
2073 NEONMAP1(vst2_v, arm_neon_vst2, 0),
2074 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
2075 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
2076 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
2077 NEONMAP1(vst3_v, arm_neon_vst3, 0),
2078 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
2079 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
2080 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
2081 NEONMAP1(vst4_v, arm_neon_vst4, 0),
2082 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
2083 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
2084 NEONMAP0(vsubhn_v),
2085 NEONMAP0(vtrn_v),
2086 NEONMAP0(vtrnq_v),
2087 NEONMAP0(vtst_v),
2088 NEONMAP0(vtstq_v),
2089 NEONMAP0(vuzp_v),
2090 NEONMAP0(vuzpq_v),
2091 NEONMAP0(vzip_v),
2092 NEONMAP0(vzipq_v)
2093 };
2094
2095 static NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
2096 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
2097 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
2098 NEONMAP0(vaddhn_v),
2099 NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0),
2100 NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0),
2101 NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0),
2102 NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0),
2103 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
2104 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
2105 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
2106 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
2107 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
2108 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
2109 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
2110 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
2111 NEONMAP1(vcls_v, aarch64_neon_cls, Add1ArgType),
2112 NEONMAP1(vclsq_v, aarch64_neon_cls, Add1ArgType),
2113 NEONMAP1(vclz_v, ctlz, Add1ArgType),
2114 NEONMAP1(vclzq_v, ctlz, Add1ArgType),
2115 NEONMAP1(vcnt_v, ctpop, Add1ArgType),
2116 NEONMAP1(vcntq_v, ctpop, Add1ArgType),
2117 NEONMAP1(vcvt_f16_v, aarch64_neon_vcvtfp2hf, 0),
2118 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
2119 NEONMAP0(vcvt_f32_v),
2120 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
2121 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
2122 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
2123 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
2124 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
2125 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
2126 NEONMAP0(vcvtq_f32_v),
2127 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
2128 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
2129 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
2130 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
2131 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
2132 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
2133 NEONMAP1(vcvtx_f32_v, aarch64_neon_fcvtxn, AddRetType | Add1ArgType),
2134 NEONMAP0(vext_v),
2135 NEONMAP0(vextq_v),
2136 NEONMAP0(vfma_v),
2137 NEONMAP0(vfmaq_v),
2138 NEONMAP2(vhadd_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
2139 NEONMAP2(vhaddq_v, aarch64_neon_uhadd, aarch64_neon_shadd, Add1ArgType | UnsignedAlts),
2140 NEONMAP2(vhsub_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
2141 NEONMAP2(vhsubq_v, aarch64_neon_uhsub, aarch64_neon_shsub, Add1ArgType | UnsignedAlts),
2142 NEONMAP0(vmovl_v),
2143 NEONMAP0(vmovn_v),
2144 NEONMAP1(vmul_v, aarch64_neon_pmul, Add1ArgType),
2145 NEONMAP1(vmulq_v, aarch64_neon_pmul, Add1ArgType),
2146 NEONMAP1(vpadd_v, aarch64_neon_addp, Add1ArgType),
2147 NEONMAP2(vpaddl_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
2148 NEONMAP2(vpaddlq_v, aarch64_neon_uaddlp, aarch64_neon_saddlp, UnsignedAlts),
2149 NEONMAP1(vpaddq_v, aarch64_neon_addp, Add1ArgType),
2150 NEONMAP1(vqabs_v, aarch64_neon_sqabs, Add1ArgType),
2151 NEONMAP1(vqabsq_v, aarch64_neon_sqabs, Add1ArgType),
2152 NEONMAP2(vqadd_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
2153 NEONMAP2(vqaddq_v, aarch64_neon_uqadd, aarch64_neon_sqadd, Add1ArgType | UnsignedAlts),
2154 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
2155 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
2156 NEONMAP1(vqdmulh_v, aarch64_neon_sqdmulh, Add1ArgType),
2157 NEONMAP1(vqdmulhq_v, aarch64_neon_sqdmulh, Add1ArgType),
2158 NEONMAP1(vqdmull_v, aarch64_neon_sqdmull, Add1ArgType),
2159 NEONMAP2(vqmovn_v, aarch64_neon_uqxtn, aarch64_neon_sqxtn, Add1ArgType | UnsignedAlts),
2160 NEONMAP1(vqmovun_v, aarch64_neon_sqxtun, Add1ArgType),
2161 NEONMAP1(vqneg_v, aarch64_neon_sqneg, Add1ArgType),
2162 NEONMAP1(vqnegq_v, aarch64_neon_sqneg, Add1ArgType),
2163 NEONMAP1(vqrdmulh_v, aarch64_neon_sqrdmulh, Add1ArgType),
2164 NEONMAP1(vqrdmulhq_v, aarch64_neon_sqrdmulh, Add1ArgType),
2165 NEONMAP2(vqrshl_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
2166 NEONMAP2(vqrshlq_v, aarch64_neon_uqrshl, aarch64_neon_sqrshl, Add1ArgType | UnsignedAlts),
2167 NEONMAP2(vqshl_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl, UnsignedAlts),
2168 NEONMAP2(vqshl_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
2169 NEONMAP2(vqshlq_n_v, aarch64_neon_uqshl, aarch64_neon_sqshl,UnsignedAlts),
2170 NEONMAP2(vqshlq_v, aarch64_neon_uqshl, aarch64_neon_sqshl, Add1ArgType | UnsignedAlts),
2171 NEONMAP2(vqsub_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
2172 NEONMAP2(vqsubq_v, aarch64_neon_uqsub, aarch64_neon_sqsub, Add1ArgType | UnsignedAlts),
2173 NEONMAP1(vraddhn_v, aarch64_neon_raddhn, Add1ArgType),
2174 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
2175 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
2176 NEONMAP1(vrecps_v, aarch64_neon_frecps, Add1ArgType),
2177 NEONMAP1(vrecpsq_v, aarch64_neon_frecps, Add1ArgType),
2178 NEONMAP2(vrhadd_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
2179 NEONMAP2(vrhaddq_v, aarch64_neon_urhadd, aarch64_neon_srhadd, Add1ArgType | UnsignedAlts),
2180 NEONMAP2(vrshl_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
2181 NEONMAP2(vrshlq_v, aarch64_neon_urshl, aarch64_neon_srshl, Add1ArgType | UnsignedAlts),
2182 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
2183 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
2184 NEONMAP1(vrsqrts_v, aarch64_neon_frsqrts, Add1ArgType),
2185 NEONMAP1(vrsqrtsq_v, aarch64_neon_frsqrts, Add1ArgType),
2186 NEONMAP1(vrsubhn_v, aarch64_neon_rsubhn, Add1ArgType),
2187 NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0),
2188 NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0),
2189 NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0),
2190 NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0),
2191 NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0),
2192 NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0),
2193 NEONMAP0(vshl_n_v),
2194 NEONMAP2(vshl_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
2195 NEONMAP0(vshll_n_v),
2196 NEONMAP0(vshlq_n_v),
2197 NEONMAP2(vshlq_v, aarch64_neon_ushl, aarch64_neon_sshl, Add1ArgType | UnsignedAlts),
2198 NEONMAP0(vshr_n_v),
2199 NEONMAP0(vshrn_n_v),
2200 NEONMAP0(vshrq_n_v),
2201 NEONMAP0(vsubhn_v),
2202 NEONMAP0(vtst_v),
2203 NEONMAP0(vtstq_v),
2204 };
2205
2206 static NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = {
2207 NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType),
2208 NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType),
2209 NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType),
2210 NEONMAP1(vaddlv_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
2211 NEONMAP1(vaddlv_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
2212 NEONMAP1(vaddlvq_s32, aarch64_neon_saddlv, AddRetType | Add1ArgType),
2213 NEONMAP1(vaddlvq_u32, aarch64_neon_uaddlv, AddRetType | Add1ArgType),
2214 NEONMAP1(vaddv_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
2215 NEONMAP1(vaddv_s32, aarch64_neon_saddv, AddRetType | Add1ArgType),
2216 NEONMAP1(vaddv_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType),
2217 NEONMAP1(vaddvq_f32, aarch64_neon_faddv, AddRetType | Add1ArgType),
2218 NEONMAP1(vaddvq_f64, aarch64_neon_faddv, AddRetType | Add1ArgType),
2219 NEONMAP1(vaddvq_s32, aarch64_neon_saddv, AddRetType | Add1ArgType),
2220 NEONMAP1(vaddvq_s64, aarch64_neon_saddv, AddRetType | Add1ArgType),
2221 NEONMAP1(vaddvq_u32, aarch64_neon_uaddv, AddRetType | Add1ArgType),
2222 NEONMAP1(vaddvq_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
2223 NEONMAP1(vcaged_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
2224 NEONMAP1(vcages_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
2225 NEONMAP1(vcagtd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
2226 NEONMAP1(vcagts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
2227 NEONMAP1(vcaled_f64, aarch64_neon_facge, AddRetType | Add1ArgType),
2228 NEONMAP1(vcales_f32, aarch64_neon_facge, AddRetType | Add1ArgType),
2229 NEONMAP1(vcaltd_f64, aarch64_neon_facgt, AddRetType | Add1ArgType),
2230 NEONMAP1(vcalts_f32, aarch64_neon_facgt, AddRetType | Add1ArgType),
2231 NEONMAP1(vcvtad_s64_f64, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
2232 NEONMAP1(vcvtad_u64_f64, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
2233 NEONMAP1(vcvtas_s32_f32, aarch64_neon_fcvtas, AddRetType | Add1ArgType),
2234 NEONMAP1(vcvtas_u32_f32, aarch64_neon_fcvtau, AddRetType | Add1ArgType),
2235 NEONMAP1(vcvtd_n_f64_s64, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
2236 NEONMAP1(vcvtd_n_f64_u64, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
2237 NEONMAP1(vcvtd_n_s64_f64, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
2238 NEONMAP1(vcvtd_n_u64_f64, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
2239 NEONMAP1(vcvtmd_s64_f64, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
2240 NEONMAP1(vcvtmd_u64_f64, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
2241 NEONMAP1(vcvtms_s32_f32, aarch64_neon_fcvtms, AddRetType | Add1ArgType),
2242 NEONMAP1(vcvtms_u32_f32, aarch64_neon_fcvtmu, AddRetType | Add1ArgType),
2243 NEONMAP1(vcvtnd_s64_f64, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
2244 NEONMAP1(vcvtnd_u64_f64, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
2245 NEONMAP1(vcvtns_s32_f32, aarch64_neon_fcvtns, AddRetType | Add1ArgType),
2246 NEONMAP1(vcvtns_u32_f32, aarch64_neon_fcvtnu, AddRetType | Add1ArgType),
2247 NEONMAP1(vcvtpd_s64_f64, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
2248 NEONMAP1(vcvtpd_u64_f64, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
2249 NEONMAP1(vcvtps_s32_f32, aarch64_neon_fcvtps, AddRetType | Add1ArgType),
2250 NEONMAP1(vcvtps_u32_f32, aarch64_neon_fcvtpu, AddRetType | Add1ArgType),
2251 NEONMAP1(vcvts_n_f32_s32, aarch64_neon_vcvtfxs2fp, AddRetType | Add1ArgType),
2252 NEONMAP1(vcvts_n_f32_u32, aarch64_neon_vcvtfxu2fp, AddRetType | Add1ArgType),
2253 NEONMAP1(vcvts_n_s32_f32, aarch64_neon_vcvtfp2fxs, AddRetType | Add1ArgType),
2254 NEONMAP1(vcvts_n_u32_f32, aarch64_neon_vcvtfp2fxu, AddRetType | Add1ArgType),
2255 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
2256 NEONMAP1(vmaxnmv_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
2257 NEONMAP1(vmaxnmvq_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
2258 NEONMAP1(vmaxnmvq_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
2259 NEONMAP1(vmaxv_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
2260 NEONMAP1(vmaxv_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType),
2261 NEONMAP1(vmaxv_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType),
2262 NEONMAP1(vmaxvq_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
2263 NEONMAP1(vmaxvq_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
2264 NEONMAP1(vmaxvq_s32, aarch64_neon_smaxv, AddRetType | Add1ArgType),
2265 NEONMAP1(vmaxvq_u32, aarch64_neon_umaxv, AddRetType | Add1ArgType),
2266 NEONMAP1(vminnmv_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
2267 NEONMAP1(vminnmvq_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
2268 NEONMAP1(vminnmvq_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
2269 NEONMAP1(vminv_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
2270 NEONMAP1(vminv_s32, aarch64_neon_sminv, AddRetType | Add1ArgType),
2271 NEONMAP1(vminv_u32, aarch64_neon_uminv, AddRetType | Add1ArgType),
2272 NEONMAP1(vminvq_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
2273 NEONMAP1(vminvq_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
2274 NEONMAP1(vminvq_s32, aarch64_neon_sminv, AddRetType | Add1ArgType),
2275 NEONMAP1(vminvq_u32, aarch64_neon_uminv, AddRetType | Add1ArgType),
2276 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
2277 NEONMAP1(vmulxd_f64, aarch64_neon_fmulx, Add1ArgType),
2278 NEONMAP1(vmulxs_f32, aarch64_neon_fmulx, Add1ArgType),
2279 NEONMAP1(vpaddd_s64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
2280 NEONMAP1(vpaddd_u64, aarch64_neon_uaddv, AddRetType | Add1ArgType),
2281 NEONMAP1(vpmaxnmqd_f64, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
2282 NEONMAP1(vpmaxnms_f32, aarch64_neon_fmaxnmv, AddRetType | Add1ArgType),
2283 NEONMAP1(vpmaxqd_f64, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
2284 NEONMAP1(vpmaxs_f32, aarch64_neon_fmaxv, AddRetType | Add1ArgType),
2285 NEONMAP1(vpminnmqd_f64, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
2286 NEONMAP1(vpminnms_f32, aarch64_neon_fminnmv, AddRetType | Add1ArgType),
2287 NEONMAP1(vpminqd_f64, aarch64_neon_fminv, AddRetType | Add1ArgType),
2288 NEONMAP1(vpmins_f32, aarch64_neon_fminv, AddRetType | Add1ArgType),
2289 NEONMAP1(vqabsb_s8, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
2290 NEONMAP1(vqabsd_s64, aarch64_neon_sqabs, Add1ArgType),
2291 NEONMAP1(vqabsh_s16, aarch64_neon_sqabs, Vectorize1ArgType | Use64BitVectors),
2292 NEONMAP1(vqabss_s32, aarch64_neon_sqabs, Add1ArgType),
2293 NEONMAP1(vqaddb_s8, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
2294 NEONMAP1(vqaddb_u8, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
2295 NEONMAP1(vqaddd_s64, aarch64_neon_sqadd, Add1ArgType),
2296 NEONMAP1(vqaddd_u64, aarch64_neon_uqadd, Add1ArgType),
2297 NEONMAP1(vqaddh_s16, aarch64_neon_sqadd, Vectorize1ArgType | Use64BitVectors),
2298 NEONMAP1(vqaddh_u16, aarch64_neon_uqadd, Vectorize1ArgType | Use64BitVectors),
2299 NEONMAP1(vqadds_s32, aarch64_neon_sqadd, Add1ArgType),
2300 NEONMAP1(vqadds_u32, aarch64_neon_uqadd, Add1ArgType),
2301 NEONMAP1(vqdmulhh_s16, aarch64_neon_sqdmulh, Vectorize1ArgType | Use64BitVectors),
2302 NEONMAP1(vqdmulhs_s32, aarch64_neon_sqdmulh, Add1ArgType),
2303 NEONMAP1(vqdmullh_s16, aarch64_neon_sqdmull, VectorRet | Use128BitVectors),
2304 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
2305 NEONMAP1(vqmovnd_s64, aarch64_neon_scalar_sqxtn, AddRetType | Add1ArgType),
2306 NEONMAP1(vqmovnd_u64, aarch64_neon_scalar_uqxtn, AddRetType | Add1ArgType),
2307 NEONMAP1(vqmovnh_s16, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
2308 NEONMAP1(vqmovnh_u16, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
2309 NEONMAP1(vqmovns_s32, aarch64_neon_sqxtn, VectorRet | Use64BitVectors),
2310 NEONMAP1(vqmovns_u32, aarch64_neon_uqxtn, VectorRet | Use64BitVectors),
2311 NEONMAP1(vqmovund_s64, aarch64_neon_scalar_sqxtun, AddRetType | Add1ArgType),
2312 NEONMAP1(vqmovunh_s16, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
2313 NEONMAP1(vqmovuns_s32, aarch64_neon_sqxtun, VectorRet | Use64BitVectors),
2314 NEONMAP1(vqnegb_s8, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
2315 NEONMAP1(vqnegd_s64, aarch64_neon_sqneg, Add1ArgType),
2316 NEONMAP1(vqnegh_s16, aarch64_neon_sqneg, Vectorize1ArgType | Use64BitVectors),
2317 NEONMAP1(vqnegs_s32, aarch64_neon_sqneg, Add1ArgType),
2318 NEONMAP1(vqrdmulhh_s16, aarch64_neon_sqrdmulh, Vectorize1ArgType | Use64BitVectors),
2319 NEONMAP1(vqrdmulhs_s32, aarch64_neon_sqrdmulh, Add1ArgType),
2320 NEONMAP1(vqrshlb_s8, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
2321 NEONMAP1(vqrshlb_u8, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
2322 NEONMAP1(vqrshld_s64, aarch64_neon_sqrshl, Add1ArgType),
2323 NEONMAP1(vqrshld_u64, aarch64_neon_uqrshl, Add1ArgType),
2324 NEONMAP1(vqrshlh_s16, aarch64_neon_sqrshl, Vectorize1ArgType | Use64BitVectors),
2325 NEONMAP1(vqrshlh_u16, aarch64_neon_uqrshl, Vectorize1ArgType | Use64BitVectors),
2326 NEONMAP1(vqrshls_s32, aarch64_neon_sqrshl, Add1ArgType),
2327 NEONMAP1(vqrshls_u32, aarch64_neon_uqrshl, Add1ArgType),
2328 NEONMAP1(vqrshrnd_n_s64, aarch64_neon_sqrshrn, AddRetType),
2329 NEONMAP1(vqrshrnd_n_u64, aarch64_neon_uqrshrn, AddRetType),
2330 NEONMAP1(vqrshrnh_n_s16, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
2331 NEONMAP1(vqrshrnh_n_u16, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
2332 NEONMAP1(vqrshrns_n_s32, aarch64_neon_sqrshrn, VectorRet | Use64BitVectors),
2333 NEONMAP1(vqrshrns_n_u32, aarch64_neon_uqrshrn, VectorRet | Use64BitVectors),
2334 NEONMAP1(vqrshrund_n_s64, aarch64_neon_sqrshrun, AddRetType),
2335 NEONMAP1(vqrshrunh_n_s16, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
2336 NEONMAP1(vqrshruns_n_s32, aarch64_neon_sqrshrun, VectorRet | Use64BitVectors),
2337 NEONMAP1(vqshlb_n_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
2338 NEONMAP1(vqshlb_n_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
2339 NEONMAP1(vqshlb_s8, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
2340 NEONMAP1(vqshlb_u8, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
2341 NEONMAP1(vqshld_s64, aarch64_neon_sqshl, Add1ArgType),
2342 NEONMAP1(vqshld_u64, aarch64_neon_uqshl, Add1ArgType),
2343 NEONMAP1(vqshlh_n_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
2344 NEONMAP1(vqshlh_n_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
2345 NEONMAP1(vqshlh_s16, aarch64_neon_sqshl, Vectorize1ArgType | Use64BitVectors),
2346 NEONMAP1(vqshlh_u16, aarch64_neon_uqshl, Vectorize1ArgType | Use64BitVectors),
2347 NEONMAP1(vqshls_n_s32, aarch64_neon_sqshl, Add1ArgType),
2348 NEONMAP1(vqshls_n_u32, aarch64_neon_uqshl, Add1ArgType),
2349 NEONMAP1(vqshls_s32, aarch64_neon_sqshl, Add1ArgType),
2350 NEONMAP1(vqshls_u32, aarch64_neon_uqshl, Add1ArgType),
2351 NEONMAP1(vqshlub_n_s8, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
2352 NEONMAP1(vqshluh_n_s16, aarch64_neon_sqshlu, Vectorize1ArgType | Use64BitVectors),
2353 NEONMAP1(vqshlus_n_s32, aarch64_neon_sqshlu, Add1ArgType),
2354 NEONMAP1(vqshrnd_n_s64, aarch64_neon_sqshrn, AddRetType),
2355 NEONMAP1(vqshrnd_n_u64, aarch64_neon_uqshrn, AddRetType),
2356 NEONMAP1(vqshrnh_n_s16, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
2357 NEONMAP1(vqshrnh_n_u16, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
2358 NEONMAP1(vqshrns_n_s32, aarch64_neon_sqshrn, VectorRet | Use64BitVectors),
2359 NEONMAP1(vqshrns_n_u32, aarch64_neon_uqshrn, VectorRet | Use64BitVectors),
2360 NEONMAP1(vqshrund_n_s64, aarch64_neon_sqshrun, AddRetType),
2361 NEONMAP1(vqshrunh_n_s16, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
2362 NEONMAP1(vqshruns_n_s32, aarch64_neon_sqshrun, VectorRet | Use64BitVectors),
2363 NEONMAP1(vqsubb_s8, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
2364 NEONMAP1(vqsubb_u8, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
2365 NEONMAP1(vqsubd_s64, aarch64_neon_sqsub, Add1ArgType),
2366 NEONMAP1(vqsubd_u64, aarch64_neon_uqsub, Add1ArgType),
2367 NEONMAP1(vqsubh_s16, aarch64_neon_sqsub, Vectorize1ArgType | Use64BitVectors),
2368 NEONMAP1(vqsubh_u16, aarch64_neon_uqsub, Vectorize1ArgType | Use64BitVectors),
2369 NEONMAP1(vqsubs_s32, aarch64_neon_sqsub, Add1ArgType),
2370 NEONMAP1(vqsubs_u32, aarch64_neon_uqsub, Add1ArgType),
2371 NEONMAP1(vrecped_f64, aarch64_neon_frecpe, Add1ArgType),
2372 NEONMAP1(vrecpes_f32, aarch64_neon_frecpe, Add1ArgType),
2373 NEONMAP1(vrecpxd_f64, aarch64_neon_frecpx, Add1ArgType),
2374 NEONMAP1(vrecpxs_f32, aarch64_neon_frecpx, Add1ArgType),
2375 NEONMAP1(vrshld_s64, aarch64_neon_srshl, Add1ArgType),
2376 NEONMAP1(vrshld_u64, aarch64_neon_urshl, Add1ArgType),
2377 NEONMAP1(vrsqrted_f64, aarch64_neon_frsqrte, Add1ArgType),
2378 NEONMAP1(vrsqrtes_f32, aarch64_neon_frsqrte, Add1ArgType),
2379 NEONMAP1(vrsqrtsd_f64, aarch64_neon_frsqrts, Add1ArgType),
2380 NEONMAP1(vrsqrtss_f32, aarch64_neon_frsqrts, Add1ArgType),
2381 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
2382 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
2383 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
2384 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
2385 NEONMAP1(vshld_s64, aarch64_neon_sshl, Add1ArgType),
2386 NEONMAP1(vshld_u64, aarch64_neon_ushl, Add1ArgType),
2387 NEONMAP1(vslid_n_s64, aarch64_neon_vsli, Vectorize1ArgType),
2388 NEONMAP1(vslid_n_u64, aarch64_neon_vsli, Vectorize1ArgType),
2389 NEONMAP1(vsqaddb_u8, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
2390 NEONMAP1(vsqaddd_u64, aarch64_neon_usqadd, Add1ArgType),
2391 NEONMAP1(vsqaddh_u16, aarch64_neon_usqadd, Vectorize1ArgType | Use64BitVectors),
2392 NEONMAP1(vsqadds_u32, aarch64_neon_usqadd, Add1ArgType),
2393 NEONMAP1(vsrid_n_s64, aarch64_neon_vsri, Vectorize1ArgType),
2394 NEONMAP1(vsrid_n_u64, aarch64_neon_vsri, Vectorize1ArgType),
2395 NEONMAP1(vuqaddb_s8, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
2396 NEONMAP1(vuqaddd_s64, aarch64_neon_suqadd, Add1ArgType),
2397 NEONMAP1(vuqaddh_s16, aarch64_neon_suqadd, Vectorize1ArgType | Use64BitVectors),
2398 NEONMAP1(vuqadds_s32, aarch64_neon_suqadd, Add1ArgType),
2399 };
2400
2401 #undef NEONMAP0
2402 #undef NEONMAP1
2403 #undef NEONMAP2
2404
2405 static bool NEONSIMDIntrinsicsProvenSorted = false;
2406
2407 static bool AArch64SIMDIntrinsicsProvenSorted = false;
2408 static bool AArch64SISDIntrinsicsProvenSorted = false;
2409
2410
2411 static const NeonIntrinsicInfo *
findNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap,unsigned BuiltinID,bool & MapProvenSorted)2412 findNeonIntrinsicInMap(ArrayRef<NeonIntrinsicInfo> IntrinsicMap,
2413 unsigned BuiltinID, bool &MapProvenSorted) {
2414
2415 #ifndef NDEBUG
2416 if (!MapProvenSorted) {
2417 // FIXME: use std::is_sorted once C++11 is allowed
2418 for (unsigned i = 0; i < IntrinsicMap.size() - 1; ++i)
2419 assert(IntrinsicMap[i].BuiltinID <= IntrinsicMap[i + 1].BuiltinID);
2420 MapProvenSorted = true;
2421 }
2422 #endif
2423
2424 const NeonIntrinsicInfo *Builtin =
2425 std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID);
2426
2427 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
2428 return Builtin;
2429
2430 return nullptr;
2431 }
2432
LookupNeonLLVMIntrinsic(unsigned IntrinsicID,unsigned Modifier,llvm::Type * ArgType,const CallExpr * E)2433 Function *CodeGenFunction::LookupNeonLLVMIntrinsic(unsigned IntrinsicID,
2434 unsigned Modifier,
2435 llvm::Type *ArgType,
2436 const CallExpr *E) {
2437 int VectorSize = 0;
2438 if (Modifier & Use64BitVectors)
2439 VectorSize = 64;
2440 else if (Modifier & Use128BitVectors)
2441 VectorSize = 128;
2442
2443 // Return type.
2444 SmallVector<llvm::Type *, 3> Tys;
2445 if (Modifier & AddRetType) {
2446 llvm::Type *Ty = ConvertType(E->getCallReturnType());
2447 if (Modifier & VectorizeRetType)
2448 Ty = llvm::VectorType::get(
2449 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
2450
2451 Tys.push_back(Ty);
2452 }
2453
2454 // Arguments.
2455 if (Modifier & VectorizeArgTypes) {
2456 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
2457 ArgType = llvm::VectorType::get(ArgType, Elts);
2458 }
2459
2460 if (Modifier & (Add1ArgType | Add2ArgTypes))
2461 Tys.push_back(ArgType);
2462
2463 if (Modifier & Add2ArgTypes)
2464 Tys.push_back(ArgType);
2465
2466 if (Modifier & InventFloatType)
2467 Tys.push_back(FloatTy);
2468
2469 return CGM.getIntrinsic(IntrinsicID, Tys);
2470 }
2471
EmitCommonNeonSISDBuiltinExpr(CodeGenFunction & CGF,const NeonIntrinsicInfo & SISDInfo,SmallVectorImpl<Value * > & Ops,const CallExpr * E)2472 static Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF,
2473 const NeonIntrinsicInfo &SISDInfo,
2474 SmallVectorImpl<Value *> &Ops,
2475 const CallExpr *E) {
2476 unsigned BuiltinID = SISDInfo.BuiltinID;
2477 unsigned int Int = SISDInfo.LLVMIntrinsic;
2478 unsigned Modifier = SISDInfo.TypeModifier;
2479 const char *s = SISDInfo.NameHint;
2480
2481 switch (BuiltinID) {
2482 case NEON::BI__builtin_neon_vcled_s64:
2483 case NEON::BI__builtin_neon_vcled_u64:
2484 case NEON::BI__builtin_neon_vcles_f32:
2485 case NEON::BI__builtin_neon_vcled_f64:
2486 case NEON::BI__builtin_neon_vcltd_s64:
2487 case NEON::BI__builtin_neon_vcltd_u64:
2488 case NEON::BI__builtin_neon_vclts_f32:
2489 case NEON::BI__builtin_neon_vcltd_f64:
2490 case NEON::BI__builtin_neon_vcales_f32:
2491 case NEON::BI__builtin_neon_vcaled_f64:
2492 case NEON::BI__builtin_neon_vcalts_f32:
2493 case NEON::BI__builtin_neon_vcaltd_f64:
2494 // Only one direction of comparisons actually exist, cmle is actually a cmge
2495 // with swapped operands. The table gives us the right intrinsic but we
2496 // still need to do the swap.
2497 std::swap(Ops[0], Ops[1]);
2498 break;
2499 }
2500
2501 assert(Int && "Generic code assumes a valid intrinsic");
2502
2503 // Determine the type(s) of this overloaded AArch64 intrinsic.
2504 const Expr *Arg = E->getArg(0);
2505 llvm::Type *ArgTy = CGF.ConvertType(Arg->getType());
2506 Function *F = CGF.LookupNeonLLVMIntrinsic(Int, Modifier, ArgTy, E);
2507
2508 int j = 0;
2509 ConstantInt *C0 = ConstantInt::get(CGF.SizeTy, 0);
2510 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
2511 ai != ae; ++ai, ++j) {
2512 llvm::Type *ArgTy = ai->getType();
2513 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
2514 ArgTy->getPrimitiveSizeInBits())
2515 continue;
2516
2517 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
2518 // The constant argument to an _n_ intrinsic always has Int32Ty, so truncate
2519 // it before inserting.
2520 Ops[j] =
2521 CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType());
2522 Ops[j] =
2523 CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0);
2524 }
2525
2526 Value *Result = CGF.EmitNeonCall(F, Ops, s);
2527 llvm::Type *ResultType = CGF.ConvertType(E->getType());
2528 if (ResultType->getPrimitiveSizeInBits() <
2529 Result->getType()->getPrimitiveSizeInBits())
2530 return CGF.Builder.CreateExtractElement(Result, C0);
2531
2532 return CGF.Builder.CreateBitCast(Result, ResultType, s);
2533 }
2534
EmitCommonNeonBuiltinExpr(unsigned BuiltinID,unsigned LLVMIntrinsic,unsigned AltLLVMIntrinsic,const char * NameHint,unsigned Modifier,const CallExpr * E,SmallVectorImpl<llvm::Value * > & Ops,llvm::Value * Align)2535 Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
2536 unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic,
2537 const char *NameHint, unsigned Modifier, const CallExpr *E,
2538 SmallVectorImpl<llvm::Value *> &Ops, llvm::Value *Align) {
2539 // Get the last argument, which specifies the vector type.
2540 llvm::APSInt NeonTypeConst;
2541 const Expr *Arg = E->getArg(E->getNumArgs() - 1);
2542 if (!Arg->isIntegerConstantExpr(NeonTypeConst, getContext()))
2543 return nullptr;
2544
2545 // Determine the type of this overloaded NEON intrinsic.
2546 NeonTypeFlags Type(NeonTypeConst.getZExtValue());
2547 bool Usgn = Type.isUnsigned();
2548 bool Quad = Type.isQuad();
2549
2550 llvm::VectorType *VTy = GetNeonType(this, Type);
2551 llvm::Type *Ty = VTy;
2552 if (!Ty)
2553 return nullptr;
2554
2555 unsigned Int = LLVMIntrinsic;
2556 if ((Modifier & UnsignedAlts) && !Usgn)
2557 Int = AltLLVMIntrinsic;
2558
2559 switch (BuiltinID) {
2560 default: break;
2561 case NEON::BI__builtin_neon_vabs_v:
2562 case NEON::BI__builtin_neon_vabsq_v:
2563 if (VTy->getElementType()->isFloatingPointTy())
2564 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs");
2565 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs");
2566 case NEON::BI__builtin_neon_vaddhn_v: {
2567 llvm::VectorType *SrcTy =
2568 llvm::VectorType::getExtendedElementVectorType(VTy);
2569
2570 // %sum = add <4 x i32> %lhs, %rhs
2571 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
2572 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
2573 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
2574
2575 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
2576 Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(),
2577 SrcTy->getScalarSizeInBits() / 2);
2578 ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt);
2579 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
2580
2581 // %res = trunc <4 x i32> %high to <4 x i16>
2582 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn");
2583 }
2584 case NEON::BI__builtin_neon_vcale_v:
2585 case NEON::BI__builtin_neon_vcaleq_v:
2586 case NEON::BI__builtin_neon_vcalt_v:
2587 case NEON::BI__builtin_neon_vcaltq_v:
2588 std::swap(Ops[0], Ops[1]);
2589 case NEON::BI__builtin_neon_vcage_v:
2590 case NEON::BI__builtin_neon_vcageq_v:
2591 case NEON::BI__builtin_neon_vcagt_v:
2592 case NEON::BI__builtin_neon_vcagtq_v: {
2593 llvm::Type *VecFlt = llvm::VectorType::get(
2594 VTy->getScalarSizeInBits() == 32 ? FloatTy : DoubleTy,
2595 VTy->getNumElements());
2596 llvm::Type *Tys[] = { VTy, VecFlt };
2597 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
2598 return EmitNeonCall(F, Ops, NameHint);
2599 }
2600 case NEON::BI__builtin_neon_vclz_v:
2601 case NEON::BI__builtin_neon_vclzq_v:
2602 // We generate target-independent intrinsic, which needs a second argument
2603 // for whether or not clz of zero is undefined; on ARM it isn't.
2604 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
2605 break;
2606 case NEON::BI__builtin_neon_vcvt_f32_v:
2607 case NEON::BI__builtin_neon_vcvtq_f32_v:
2608 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2609 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad));
2610 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
2611 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
2612 case NEON::BI__builtin_neon_vcvt_n_f32_v:
2613 case NEON::BI__builtin_neon_vcvt_n_f64_v:
2614 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
2615 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
2616 bool Double =
2617 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
2618 llvm::Type *FloatTy =
2619 GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64
2620 : NeonTypeFlags::Float32,
2621 false, Quad));
2622 llvm::Type *Tys[2] = { FloatTy, Ty };
2623 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
2624 Function *F = CGM.getIntrinsic(Int, Tys);
2625 return EmitNeonCall(F, Ops, "vcvt_n");
2626 }
2627 case NEON::BI__builtin_neon_vcvt_n_s32_v:
2628 case NEON::BI__builtin_neon_vcvt_n_u32_v:
2629 case NEON::BI__builtin_neon_vcvt_n_s64_v:
2630 case NEON::BI__builtin_neon_vcvt_n_u64_v:
2631 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
2632 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
2633 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
2634 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
2635 bool Double =
2636 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
2637 llvm::Type *FloatTy =
2638 GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64
2639 : NeonTypeFlags::Float32,
2640 false, Quad));
2641 llvm::Type *Tys[2] = { Ty, FloatTy };
2642 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
2643 return EmitNeonCall(F, Ops, "vcvt_n");
2644 }
2645 case NEON::BI__builtin_neon_vcvt_s32_v:
2646 case NEON::BI__builtin_neon_vcvt_u32_v:
2647 case NEON::BI__builtin_neon_vcvt_s64_v:
2648 case NEON::BI__builtin_neon_vcvt_u64_v:
2649 case NEON::BI__builtin_neon_vcvtq_s32_v:
2650 case NEON::BI__builtin_neon_vcvtq_u32_v:
2651 case NEON::BI__builtin_neon_vcvtq_s64_v:
2652 case NEON::BI__builtin_neon_vcvtq_u64_v: {
2653 bool Double =
2654 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
2655 llvm::Type *FloatTy =
2656 GetNeonType(this, NeonTypeFlags(Double ? NeonTypeFlags::Float64
2657 : NeonTypeFlags::Float32,
2658 false, Quad));
2659 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
2660 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
2661 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
2662 }
2663 case NEON::BI__builtin_neon_vcvta_s32_v:
2664 case NEON::BI__builtin_neon_vcvta_s64_v:
2665 case NEON::BI__builtin_neon_vcvta_u32_v:
2666 case NEON::BI__builtin_neon_vcvta_u64_v:
2667 case NEON::BI__builtin_neon_vcvtaq_s32_v:
2668 case NEON::BI__builtin_neon_vcvtaq_s64_v:
2669 case NEON::BI__builtin_neon_vcvtaq_u32_v:
2670 case NEON::BI__builtin_neon_vcvtaq_u64_v:
2671 case NEON::BI__builtin_neon_vcvtn_s32_v:
2672 case NEON::BI__builtin_neon_vcvtn_s64_v:
2673 case NEON::BI__builtin_neon_vcvtn_u32_v:
2674 case NEON::BI__builtin_neon_vcvtn_u64_v:
2675 case NEON::BI__builtin_neon_vcvtnq_s32_v:
2676 case NEON::BI__builtin_neon_vcvtnq_s64_v:
2677 case NEON::BI__builtin_neon_vcvtnq_u32_v:
2678 case NEON::BI__builtin_neon_vcvtnq_u64_v:
2679 case NEON::BI__builtin_neon_vcvtp_s32_v:
2680 case NEON::BI__builtin_neon_vcvtp_s64_v:
2681 case NEON::BI__builtin_neon_vcvtp_u32_v:
2682 case NEON::BI__builtin_neon_vcvtp_u64_v:
2683 case NEON::BI__builtin_neon_vcvtpq_s32_v:
2684 case NEON::BI__builtin_neon_vcvtpq_s64_v:
2685 case NEON::BI__builtin_neon_vcvtpq_u32_v:
2686 case NEON::BI__builtin_neon_vcvtpq_u64_v:
2687 case NEON::BI__builtin_neon_vcvtm_s32_v:
2688 case NEON::BI__builtin_neon_vcvtm_s64_v:
2689 case NEON::BI__builtin_neon_vcvtm_u32_v:
2690 case NEON::BI__builtin_neon_vcvtm_u64_v:
2691 case NEON::BI__builtin_neon_vcvtmq_s32_v:
2692 case NEON::BI__builtin_neon_vcvtmq_s64_v:
2693 case NEON::BI__builtin_neon_vcvtmq_u32_v:
2694 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
2695 bool Double =
2696 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
2697 llvm::Type *InTy =
2698 GetNeonType(this,
2699 NeonTypeFlags(Double ? NeonTypeFlags::Float64
2700 : NeonTypeFlags::Float32, false, Quad));
2701 llvm::Type *Tys[2] = { Ty, InTy };
2702 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
2703 }
2704 case NEON::BI__builtin_neon_vext_v:
2705 case NEON::BI__builtin_neon_vextq_v: {
2706 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
2707 SmallVector<Constant*, 16> Indices;
2708 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
2709 Indices.push_back(ConstantInt::get(Int32Ty, i+CV));
2710
2711 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2712 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2713 Value *SV = llvm::ConstantVector::get(Indices);
2714 return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext");
2715 }
2716 case NEON::BI__builtin_neon_vfma_v:
2717 case NEON::BI__builtin_neon_vfmaq_v: {
2718 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
2719 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2720 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2721 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2722
2723 // NEON intrinsic puts accumulator first, unlike the LLVM fma.
2724 return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
2725 }
2726 case NEON::BI__builtin_neon_vld1_v:
2727 case NEON::BI__builtin_neon_vld1q_v:
2728 Ops.push_back(Align);
2729 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vld1");
2730 case NEON::BI__builtin_neon_vld2_v:
2731 case NEON::BI__builtin_neon_vld2q_v:
2732 case NEON::BI__builtin_neon_vld3_v:
2733 case NEON::BI__builtin_neon_vld3q_v:
2734 case NEON::BI__builtin_neon_vld4_v:
2735 case NEON::BI__builtin_neon_vld4q_v: {
2736 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty);
2737 Ops[1] = Builder.CreateCall2(F, Ops[1], Align, NameHint);
2738 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
2739 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2740 return Builder.CreateStore(Ops[1], Ops[0]);
2741 }
2742 case NEON::BI__builtin_neon_vld1_dup_v:
2743 case NEON::BI__builtin_neon_vld1q_dup_v: {
2744 Value *V = UndefValue::get(Ty);
2745 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
2746 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2747 LoadInst *Ld = Builder.CreateLoad(Ops[0]);
2748 Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
2749 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
2750 Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
2751 return EmitNeonSplat(Ops[0], CI);
2752 }
2753 case NEON::BI__builtin_neon_vld2_lane_v:
2754 case NEON::BI__builtin_neon_vld2q_lane_v:
2755 case NEON::BI__builtin_neon_vld3_lane_v:
2756 case NEON::BI__builtin_neon_vld3q_lane_v:
2757 case NEON::BI__builtin_neon_vld4_lane_v:
2758 case NEON::BI__builtin_neon_vld4q_lane_v: {
2759 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Ty);
2760 for (unsigned I = 2; I < Ops.size() - 1; ++I)
2761 Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
2762 Ops.push_back(Align);
2763 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
2764 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
2765 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2766 return Builder.CreateStore(Ops[1], Ops[0]);
2767 }
2768 case NEON::BI__builtin_neon_vmovl_v: {
2769 llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
2770 Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
2771 if (Usgn)
2772 return Builder.CreateZExt(Ops[0], Ty, "vmovl");
2773 return Builder.CreateSExt(Ops[0], Ty, "vmovl");
2774 }
2775 case NEON::BI__builtin_neon_vmovn_v: {
2776 llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
2777 Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
2778 return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
2779 }
2780 case NEON::BI__builtin_neon_vmull_v:
2781 // FIXME: the integer vmull operations could be emitted in terms of pure
2782 // LLVM IR (2 exts followed by a mul). Unfortunately LLVM has a habit of
2783 // hoisting the exts outside loops. Until global ISel comes along that can
2784 // see through such movement this leads to bad CodeGen. So we need an
2785 // intrinsic for now.
2786 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
2787 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
2788 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
2789 case NEON::BI__builtin_neon_vpadal_v:
2790 case NEON::BI__builtin_neon_vpadalq_v: {
2791 // The source operand type has twice as many elements of half the size.
2792 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
2793 llvm::Type *EltTy =
2794 llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
2795 llvm::Type *NarrowTy =
2796 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
2797 llvm::Type *Tys[2] = { Ty, NarrowTy };
2798 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
2799 }
2800 case NEON::BI__builtin_neon_vpaddl_v:
2801 case NEON::BI__builtin_neon_vpaddlq_v: {
2802 // The source operand type has twice as many elements of half the size.
2803 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
2804 llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
2805 llvm::Type *NarrowTy =
2806 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
2807 llvm::Type *Tys[2] = { Ty, NarrowTy };
2808 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl");
2809 }
2810 case NEON::BI__builtin_neon_vqdmlal_v:
2811 case NEON::BI__builtin_neon_vqdmlsl_v: {
2812 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end());
2813 Value *Mul = EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty),
2814 MulOps, "vqdmlal");
2815
2816 SmallVector<Value *, 2> AccumOps;
2817 AccumOps.push_back(Ops[0]);
2818 AccumOps.push_back(Mul);
2819 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty),
2820 AccumOps, NameHint);
2821 }
2822 case NEON::BI__builtin_neon_vqshl_n_v:
2823 case NEON::BI__builtin_neon_vqshlq_n_v:
2824 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n",
2825 1, false);
2826 case NEON::BI__builtin_neon_vrecpe_v:
2827 case NEON::BI__builtin_neon_vrecpeq_v:
2828 case NEON::BI__builtin_neon_vrsqrte_v:
2829 case NEON::BI__builtin_neon_vrsqrteq_v:
2830 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
2831 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
2832
2833 case NEON::BI__builtin_neon_vshl_n_v:
2834 case NEON::BI__builtin_neon_vshlq_n_v:
2835 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
2836 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
2837 "vshl_n");
2838 case NEON::BI__builtin_neon_vshll_n_v: {
2839 llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy);
2840 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
2841 if (Usgn)
2842 Ops[0] = Builder.CreateZExt(Ops[0], VTy);
2843 else
2844 Ops[0] = Builder.CreateSExt(Ops[0], VTy);
2845 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false);
2846 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n");
2847 }
2848 case NEON::BI__builtin_neon_vshrn_n_v: {
2849 llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy);
2850 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
2851 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false);
2852 if (Usgn)
2853 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
2854 else
2855 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
2856 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n");
2857 }
2858 case NEON::BI__builtin_neon_vshr_n_v:
2859 case NEON::BI__builtin_neon_vshrq_n_v:
2860 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n");
2861 case NEON::BI__builtin_neon_vst1_v:
2862 case NEON::BI__builtin_neon_vst1q_v:
2863 case NEON::BI__builtin_neon_vst2_v:
2864 case NEON::BI__builtin_neon_vst2q_v:
2865 case NEON::BI__builtin_neon_vst3_v:
2866 case NEON::BI__builtin_neon_vst3q_v:
2867 case NEON::BI__builtin_neon_vst4_v:
2868 case NEON::BI__builtin_neon_vst4q_v:
2869 case NEON::BI__builtin_neon_vst2_lane_v:
2870 case NEON::BI__builtin_neon_vst2q_lane_v:
2871 case NEON::BI__builtin_neon_vst3_lane_v:
2872 case NEON::BI__builtin_neon_vst3q_lane_v:
2873 case NEON::BI__builtin_neon_vst4_lane_v:
2874 case NEON::BI__builtin_neon_vst4q_lane_v:
2875 Ops.push_back(Align);
2876 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "");
2877 case NEON::BI__builtin_neon_vsubhn_v: {
2878 llvm::VectorType *SrcTy =
2879 llvm::VectorType::getExtendedElementVectorType(VTy);
2880
2881 // %sum = add <4 x i32> %lhs, %rhs
2882 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
2883 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
2884 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
2885
2886 // %high = lshr <4 x i32> %sum, <i32 16, i32 16, i32 16, i32 16>
2887 Constant *ShiftAmt = ConstantInt::get(SrcTy->getElementType(),
2888 SrcTy->getScalarSizeInBits() / 2);
2889 ShiftAmt = ConstantVector::getSplat(VTy->getNumElements(), ShiftAmt);
2890 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
2891
2892 // %res = trunc <4 x i32> %high to <4 x i16>
2893 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn");
2894 }
2895 case NEON::BI__builtin_neon_vtrn_v:
2896 case NEON::BI__builtin_neon_vtrnq_v: {
2897 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
2898 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2899 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2900 Value *SV = nullptr;
2901
2902 for (unsigned vi = 0; vi != 2; ++vi) {
2903 SmallVector<Constant*, 16> Indices;
2904 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
2905 Indices.push_back(Builder.getInt32(i+vi));
2906 Indices.push_back(Builder.getInt32(i+e+vi));
2907 }
2908 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
2909 SV = llvm::ConstantVector::get(Indices);
2910 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn");
2911 SV = Builder.CreateStore(SV, Addr);
2912 }
2913 return SV;
2914 }
2915 case NEON::BI__builtin_neon_vtst_v:
2916 case NEON::BI__builtin_neon_vtstq_v: {
2917 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
2918 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2919 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
2920 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
2921 ConstantAggregateZero::get(Ty));
2922 return Builder.CreateSExt(Ops[0], Ty, "vtst");
2923 }
2924 case NEON::BI__builtin_neon_vuzp_v:
2925 case NEON::BI__builtin_neon_vuzpq_v: {
2926 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
2927 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2928 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2929 Value *SV = nullptr;
2930
2931 for (unsigned vi = 0; vi != 2; ++vi) {
2932 SmallVector<Constant*, 16> Indices;
2933 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
2934 Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi));
2935
2936 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
2937 SV = llvm::ConstantVector::get(Indices);
2938 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp");
2939 SV = Builder.CreateStore(SV, Addr);
2940 }
2941 return SV;
2942 }
2943 case NEON::BI__builtin_neon_vzip_v:
2944 case NEON::BI__builtin_neon_vzipq_v: {
2945 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
2946 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
2947 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
2948 Value *SV = nullptr;
2949
2950 for (unsigned vi = 0; vi != 2; ++vi) {
2951 SmallVector<Constant*, 16> Indices;
2952 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
2953 Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1));
2954 Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e));
2955 }
2956 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
2957 SV = llvm::ConstantVector::get(Indices);
2958 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip");
2959 SV = Builder.CreateStore(SV, Addr);
2960 }
2961 return SV;
2962 }
2963 }
2964
2965 assert(Int && "Expected valid intrinsic number");
2966
2967 // Determine the type(s) of this overloaded AArch64 intrinsic.
2968 Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E);
2969
2970 Value *Result = EmitNeonCall(F, Ops, NameHint);
2971 llvm::Type *ResultType = ConvertType(E->getType());
2972 // AArch64 intrinsic one-element vector type cast to
2973 // scalar type expected by the builtin
2974 return Builder.CreateBitCast(Result, ResultType, NameHint);
2975 }
2976
EmitAArch64CompareBuiltinExpr(Value * Op,llvm::Type * Ty,const CmpInst::Predicate Fp,const CmpInst::Predicate Ip,const Twine & Name)2977 Value *CodeGenFunction::EmitAArch64CompareBuiltinExpr(
2978 Value *Op, llvm::Type *Ty, const CmpInst::Predicate Fp,
2979 const CmpInst::Predicate Ip, const Twine &Name) {
2980 llvm::Type *OTy = Op->getType();
2981
2982 // FIXME: this is utterly horrific. We should not be looking at previous
2983 // codegen context to find out what needs doing. Unfortunately TableGen
2984 // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32
2985 // (etc).
2986 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
2987 OTy = BI->getOperand(0)->getType();
2988
2989 Op = Builder.CreateBitCast(Op, OTy);
2990 if (OTy->getScalarType()->isFloatingPointTy()) {
2991 Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
2992 } else {
2993 Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
2994 }
2995 return Builder.CreateSExt(Op, Ty, Name);
2996 }
2997
packTBLDVectorList(CodeGenFunction & CGF,ArrayRef<Value * > Ops,Value * ExtOp,Value * IndexOp,llvm::Type * ResTy,unsigned IntID,const char * Name)2998 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops,
2999 Value *ExtOp, Value *IndexOp,
3000 llvm::Type *ResTy, unsigned IntID,
3001 const char *Name) {
3002 SmallVector<Value *, 2> TblOps;
3003 if (ExtOp)
3004 TblOps.push_back(ExtOp);
3005
3006 // Build a vector containing sequential number like (0, 1, 2, ..., 15)
3007 SmallVector<Constant*, 16> Indices;
3008 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType());
3009 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
3010 Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i));
3011 Indices.push_back(ConstantInt::get(CGF.Int32Ty, 2*i+1));
3012 }
3013 Value *SV = llvm::ConstantVector::get(Indices);
3014
3015 int PairPos = 0, End = Ops.size() - 1;
3016 while (PairPos < End) {
3017 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
3018 Ops[PairPos+1], SV, Name));
3019 PairPos += 2;
3020 }
3021
3022 // If there's an odd number of 64-bit lookup table, fill the high 64-bit
3023 // of the 128-bit lookup table with zero.
3024 if (PairPos == End) {
3025 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
3026 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
3027 ZeroTbl, SV, Name));
3028 }
3029
3030 Function *TblF;
3031 TblOps.push_back(IndexOp);
3032 TblF = CGF.CGM.getIntrinsic(IntID, ResTy);
3033
3034 return CGF.EmitNeonCall(TblF, TblOps, Name);
3035 }
3036
EmitARMBuiltinExpr(unsigned BuiltinID,const CallExpr * E)3037 Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
3038 const CallExpr *E) {
3039 unsigned HintID = static_cast<unsigned>(-1);
3040 switch (BuiltinID) {
3041 default: break;
3042 case ARM::BI__builtin_arm_yield:
3043 case ARM::BI__yield:
3044 HintID = 1;
3045 break;
3046 case ARM::BI__builtin_arm_wfe:
3047 case ARM::BI__wfe:
3048 HintID = 2;
3049 break;
3050 case ARM::BI__builtin_arm_wfi:
3051 case ARM::BI__wfi:
3052 HintID = 3;
3053 break;
3054 case ARM::BI__builtin_arm_sev:
3055 case ARM::BI__sev:
3056 HintID = 4;
3057 break;
3058 case ARM::BI__builtin_arm_sevl:
3059 case ARM::BI__sevl:
3060 HintID = 5;
3061 break;
3062 }
3063
3064 if (HintID != static_cast<unsigned>(-1)) {
3065 Function *F = CGM.getIntrinsic(Intrinsic::arm_hint);
3066 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
3067 }
3068
3069 if (BuiltinID == ARM::BI__builtin_arm_rbit) {
3070 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_rbit),
3071 EmitScalarExpr(E->getArg(0)),
3072 "rbit");
3073 }
3074
3075 if (BuiltinID == ARM::BI__clear_cache) {
3076 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
3077 const FunctionDecl *FD = E->getDirectCallee();
3078 SmallVector<Value*, 2> Ops;
3079 for (unsigned i = 0; i < 2; i++)
3080 Ops.push_back(EmitScalarExpr(E->getArg(i)));
3081 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
3082 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
3083 StringRef Name = FD->getName();
3084 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
3085 }
3086
3087 if (BuiltinID == ARM::BI__builtin_arm_ldrexd ||
3088 ((BuiltinID == ARM::BI__builtin_arm_ldrex ||
3089 BuiltinID == ARM::BI__builtin_arm_ldaex) &&
3090 getContext().getTypeSize(E->getType()) == 64) ||
3091 BuiltinID == ARM::BI__ldrexd) {
3092 Function *F;
3093
3094 switch (BuiltinID) {
3095 default: llvm_unreachable("unexpected builtin");
3096 case ARM::BI__builtin_arm_ldaex:
3097 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
3098 break;
3099 case ARM::BI__builtin_arm_ldrexd:
3100 case ARM::BI__builtin_arm_ldrex:
3101 case ARM::BI__ldrexd:
3102 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
3103 break;
3104 }
3105
3106 Value *LdPtr = EmitScalarExpr(E->getArg(0));
3107 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
3108 "ldrexd");
3109
3110 Value *Val0 = Builder.CreateExtractValue(Val, 1);
3111 Value *Val1 = Builder.CreateExtractValue(Val, 0);
3112 Val0 = Builder.CreateZExt(Val0, Int64Ty);
3113 Val1 = Builder.CreateZExt(Val1, Int64Ty);
3114
3115 Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
3116 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
3117 Val = Builder.CreateOr(Val, Val1);
3118 return Builder.CreateBitCast(Val, ConvertType(E->getType()));
3119 }
3120
3121 if (BuiltinID == ARM::BI__builtin_arm_ldrex ||
3122 BuiltinID == ARM::BI__builtin_arm_ldaex) {
3123 Value *LoadAddr = EmitScalarExpr(E->getArg(0));
3124
3125 QualType Ty = E->getType();
3126 llvm::Type *RealResTy = ConvertType(Ty);
3127 llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(),
3128 getContext().getTypeSize(Ty));
3129 LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo());
3130
3131 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex
3132 ? Intrinsic::arm_ldaex
3133 : Intrinsic::arm_ldrex,
3134 LoadAddr->getType());
3135 Value *Val = Builder.CreateCall(F, LoadAddr, "ldrex");
3136
3137 if (RealResTy->isPointerTy())
3138 return Builder.CreateIntToPtr(Val, RealResTy);
3139 else {
3140 Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
3141 return Builder.CreateBitCast(Val, RealResTy);
3142 }
3143 }
3144
3145 if (BuiltinID == ARM::BI__builtin_arm_strexd ||
3146 ((BuiltinID == ARM::BI__builtin_arm_stlex ||
3147 BuiltinID == ARM::BI__builtin_arm_strex) &&
3148 getContext().getTypeSize(E->getArg(0)->getType()) == 64)) {
3149 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
3150 ? Intrinsic::arm_stlexd
3151 : Intrinsic::arm_strexd);
3152 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, NULL);
3153
3154 Value *Tmp = CreateMemTemp(E->getArg(0)->getType());
3155 Value *Val = EmitScalarExpr(E->getArg(0));
3156 Builder.CreateStore(Val, Tmp);
3157
3158 Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
3159 Val = Builder.CreateLoad(LdPtr);
3160
3161 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
3162 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
3163 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), Int8PtrTy);
3164 return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "strexd");
3165 }
3166
3167 if (BuiltinID == ARM::BI__builtin_arm_strex ||
3168 BuiltinID == ARM::BI__builtin_arm_stlex) {
3169 Value *StoreVal = EmitScalarExpr(E->getArg(0));
3170 Value *StoreAddr = EmitScalarExpr(E->getArg(1));
3171
3172 QualType Ty = E->getArg(0)->getType();
3173 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
3174 getContext().getTypeSize(Ty));
3175 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
3176
3177 if (StoreVal->getType()->isPointerTy())
3178 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
3179 else {
3180 StoreVal = Builder.CreateBitCast(StoreVal, StoreTy);
3181 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
3182 }
3183
3184 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
3185 ? Intrinsic::arm_stlex
3186 : Intrinsic::arm_strex,
3187 StoreAddr->getType());
3188 return Builder.CreateCall2(F, StoreVal, StoreAddr, "strex");
3189 }
3190
3191 if (BuiltinID == ARM::BI__builtin_arm_clrex) {
3192 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
3193 return Builder.CreateCall(F);
3194 }
3195
3196 // CRC32
3197 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
3198 switch (BuiltinID) {
3199 case ARM::BI__builtin_arm_crc32b:
3200 CRCIntrinsicID = Intrinsic::arm_crc32b; break;
3201 case ARM::BI__builtin_arm_crc32cb:
3202 CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
3203 case ARM::BI__builtin_arm_crc32h:
3204 CRCIntrinsicID = Intrinsic::arm_crc32h; break;
3205 case ARM::BI__builtin_arm_crc32ch:
3206 CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
3207 case ARM::BI__builtin_arm_crc32w:
3208 case ARM::BI__builtin_arm_crc32d:
3209 CRCIntrinsicID = Intrinsic::arm_crc32w; break;
3210 case ARM::BI__builtin_arm_crc32cw:
3211 case ARM::BI__builtin_arm_crc32cd:
3212 CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
3213 }
3214
3215 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
3216 Value *Arg0 = EmitScalarExpr(E->getArg(0));
3217 Value *Arg1 = EmitScalarExpr(E->getArg(1));
3218
3219 // crc32{c,}d intrinsics are implemnted as two calls to crc32{c,}w
3220 // intrinsics, hence we need different codegen for these cases.
3221 if (BuiltinID == ARM::BI__builtin_arm_crc32d ||
3222 BuiltinID == ARM::BI__builtin_arm_crc32cd) {
3223 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
3224 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
3225 Value *Arg1b = Builder.CreateLShr(Arg1, C1);
3226 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
3227
3228 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
3229 Value *Res = Builder.CreateCall2(F, Arg0, Arg1a);
3230 return Builder.CreateCall2(F, Res, Arg1b);
3231 } else {
3232 Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
3233
3234 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
3235 return Builder.CreateCall2(F, Arg0, Arg1);
3236 }
3237 }
3238
3239 SmallVector<Value*, 4> Ops;
3240 llvm::Value *Align = nullptr;
3241 for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) {
3242 if (i == 0) {
3243 switch (BuiltinID) {
3244 case NEON::BI__builtin_neon_vld1_v:
3245 case NEON::BI__builtin_neon_vld1q_v:
3246 case NEON::BI__builtin_neon_vld1q_lane_v:
3247 case NEON::BI__builtin_neon_vld1_lane_v:
3248 case NEON::BI__builtin_neon_vld1_dup_v:
3249 case NEON::BI__builtin_neon_vld1q_dup_v:
3250 case NEON::BI__builtin_neon_vst1_v:
3251 case NEON::BI__builtin_neon_vst1q_v:
3252 case NEON::BI__builtin_neon_vst1q_lane_v:
3253 case NEON::BI__builtin_neon_vst1_lane_v:
3254 case NEON::BI__builtin_neon_vst2_v:
3255 case NEON::BI__builtin_neon_vst2q_v:
3256 case NEON::BI__builtin_neon_vst2_lane_v:
3257 case NEON::BI__builtin_neon_vst2q_lane_v:
3258 case NEON::BI__builtin_neon_vst3_v:
3259 case NEON::BI__builtin_neon_vst3q_v:
3260 case NEON::BI__builtin_neon_vst3_lane_v:
3261 case NEON::BI__builtin_neon_vst3q_lane_v:
3262 case NEON::BI__builtin_neon_vst4_v:
3263 case NEON::BI__builtin_neon_vst4q_v:
3264 case NEON::BI__builtin_neon_vst4_lane_v:
3265 case NEON::BI__builtin_neon_vst4q_lane_v:
3266 // Get the alignment for the argument in addition to the value;
3267 // we'll use it later.
3268 std::pair<llvm::Value*, unsigned> Src =
3269 EmitPointerWithAlignment(E->getArg(0));
3270 Ops.push_back(Src.first);
3271 Align = Builder.getInt32(Src.second);
3272 continue;
3273 }
3274 }
3275 if (i == 1) {
3276 switch (BuiltinID) {
3277 case NEON::BI__builtin_neon_vld2_v:
3278 case NEON::BI__builtin_neon_vld2q_v:
3279 case NEON::BI__builtin_neon_vld3_v:
3280 case NEON::BI__builtin_neon_vld3q_v:
3281 case NEON::BI__builtin_neon_vld4_v:
3282 case NEON::BI__builtin_neon_vld4q_v:
3283 case NEON::BI__builtin_neon_vld2_lane_v:
3284 case NEON::BI__builtin_neon_vld2q_lane_v:
3285 case NEON::BI__builtin_neon_vld3_lane_v:
3286 case NEON::BI__builtin_neon_vld3q_lane_v:
3287 case NEON::BI__builtin_neon_vld4_lane_v:
3288 case NEON::BI__builtin_neon_vld4q_lane_v:
3289 case NEON::BI__builtin_neon_vld2_dup_v:
3290 case NEON::BI__builtin_neon_vld3_dup_v:
3291 case NEON::BI__builtin_neon_vld4_dup_v:
3292 // Get the alignment for the argument in addition to the value;
3293 // we'll use it later.
3294 std::pair<llvm::Value*, unsigned> Src =
3295 EmitPointerWithAlignment(E->getArg(1));
3296 Ops.push_back(Src.first);
3297 Align = Builder.getInt32(Src.second);
3298 continue;
3299 }
3300 }
3301 Ops.push_back(EmitScalarExpr(E->getArg(i)));
3302 }
3303
3304 switch (BuiltinID) {
3305 default: break;
3306 // vget_lane and vset_lane are not overloaded and do not have an extra
3307 // argument that specifies the vector type.
3308 case NEON::BI__builtin_neon_vget_lane_i8:
3309 case NEON::BI__builtin_neon_vget_lane_i16:
3310 case NEON::BI__builtin_neon_vget_lane_i32:
3311 case NEON::BI__builtin_neon_vget_lane_i64:
3312 case NEON::BI__builtin_neon_vget_lane_f32:
3313 case NEON::BI__builtin_neon_vgetq_lane_i8:
3314 case NEON::BI__builtin_neon_vgetq_lane_i16:
3315 case NEON::BI__builtin_neon_vgetq_lane_i32:
3316 case NEON::BI__builtin_neon_vgetq_lane_i64:
3317 case NEON::BI__builtin_neon_vgetq_lane_f32:
3318 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
3319 "vget_lane");
3320 case NEON::BI__builtin_neon_vset_lane_i8:
3321 case NEON::BI__builtin_neon_vset_lane_i16:
3322 case NEON::BI__builtin_neon_vset_lane_i32:
3323 case NEON::BI__builtin_neon_vset_lane_i64:
3324 case NEON::BI__builtin_neon_vset_lane_f32:
3325 case NEON::BI__builtin_neon_vsetq_lane_i8:
3326 case NEON::BI__builtin_neon_vsetq_lane_i16:
3327 case NEON::BI__builtin_neon_vsetq_lane_i32:
3328 case NEON::BI__builtin_neon_vsetq_lane_i64:
3329 case NEON::BI__builtin_neon_vsetq_lane_f32:
3330 Ops.push_back(EmitScalarExpr(E->getArg(2)));
3331 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
3332
3333 // Non-polymorphic crypto instructions also not overloaded
3334 case NEON::BI__builtin_neon_vsha1h_u32:
3335 Ops.push_back(EmitScalarExpr(E->getArg(0)));
3336 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
3337 "vsha1h");
3338 case NEON::BI__builtin_neon_vsha1cq_u32:
3339 Ops.push_back(EmitScalarExpr(E->getArg(2)));
3340 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
3341 "vsha1h");
3342 case NEON::BI__builtin_neon_vsha1pq_u32:
3343 Ops.push_back(EmitScalarExpr(E->getArg(2)));
3344 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
3345 "vsha1h");
3346 case NEON::BI__builtin_neon_vsha1mq_u32:
3347 Ops.push_back(EmitScalarExpr(E->getArg(2)));
3348 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
3349 "vsha1h");
3350 }
3351
3352 // Get the last argument, which specifies the vector type.
3353 llvm::APSInt Result;
3354 const Expr *Arg = E->getArg(E->getNumArgs()-1);
3355 if (!Arg->isIntegerConstantExpr(Result, getContext()))
3356 return nullptr;
3357
3358 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
3359 BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
3360 // Determine the overloaded type of this builtin.
3361 llvm::Type *Ty;
3362 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
3363 Ty = FloatTy;
3364 else
3365 Ty = DoubleTy;
3366
3367 // Determine whether this is an unsigned conversion or not.
3368 bool usgn = Result.getZExtValue() == 1;
3369 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
3370
3371 // Call the appropriate intrinsic.
3372 Function *F = CGM.getIntrinsic(Int, Ty);
3373 return Builder.CreateCall(F, Ops, "vcvtr");
3374 }
3375
3376 // Determine the type of this overloaded NEON intrinsic.
3377 NeonTypeFlags Type(Result.getZExtValue());
3378 bool usgn = Type.isUnsigned();
3379 bool rightShift = false;
3380
3381 llvm::VectorType *VTy = GetNeonType(this, Type);
3382 llvm::Type *Ty = VTy;
3383 if (!Ty)
3384 return nullptr;
3385
3386 // Many NEON builtins have identical semantics and uses in ARM and
3387 // AArch64. Emit these in a single function.
3388 ArrayRef<NeonIntrinsicInfo> IntrinsicMap(ARMSIMDIntrinsicMap);
3389 const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap(
3390 IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted);
3391 if (Builtin)
3392 return EmitCommonNeonBuiltinExpr(
3393 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
3394 Builtin->NameHint, Builtin->TypeModifier, E, Ops, Align);
3395
3396 unsigned Int;
3397 switch (BuiltinID) {
3398 default: return nullptr;
3399 case NEON::BI__builtin_neon_vld1q_lane_v:
3400 // Handle 64-bit integer elements as a special case. Use shuffles of
3401 // one-element vectors to avoid poor code for i64 in the backend.
3402 if (VTy->getElementType()->isIntegerTy(64)) {
3403 // Extract the other lane.
3404 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3405 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
3406 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
3407 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
3408 // Load the value as a one-element vector.
3409 Ty = llvm::VectorType::get(VTy->getElementType(), 1);
3410 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty);
3411 Value *Ld = Builder.CreateCall2(F, Ops[0], Align);
3412 // Combine them.
3413 SmallVector<Constant*, 2> Indices;
3414 Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane));
3415 Indices.push_back(ConstantInt::get(Int32Ty, Lane));
3416 SV = llvm::ConstantVector::get(Indices);
3417 return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane");
3418 }
3419 // fall through
3420 case NEON::BI__builtin_neon_vld1_lane_v: {
3421 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3422 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
3423 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3424 LoadInst *Ld = Builder.CreateLoad(Ops[0]);
3425 Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
3426 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
3427 }
3428 case NEON::BI__builtin_neon_vld2_dup_v:
3429 case NEON::BI__builtin_neon_vld3_dup_v:
3430 case NEON::BI__builtin_neon_vld4_dup_v: {
3431 // Handle 64-bit elements as a special-case. There is no "dup" needed.
3432 if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) {
3433 switch (BuiltinID) {
3434 case NEON::BI__builtin_neon_vld2_dup_v:
3435 Int = Intrinsic::arm_neon_vld2;
3436 break;
3437 case NEON::BI__builtin_neon_vld3_dup_v:
3438 Int = Intrinsic::arm_neon_vld3;
3439 break;
3440 case NEON::BI__builtin_neon_vld4_dup_v:
3441 Int = Intrinsic::arm_neon_vld4;
3442 break;
3443 default: llvm_unreachable("unknown vld_dup intrinsic?");
3444 }
3445 Function *F = CGM.getIntrinsic(Int, Ty);
3446 Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup");
3447 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3448 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3449 return Builder.CreateStore(Ops[1], Ops[0]);
3450 }
3451 switch (BuiltinID) {
3452 case NEON::BI__builtin_neon_vld2_dup_v:
3453 Int = Intrinsic::arm_neon_vld2lane;
3454 break;
3455 case NEON::BI__builtin_neon_vld3_dup_v:
3456 Int = Intrinsic::arm_neon_vld3lane;
3457 break;
3458 case NEON::BI__builtin_neon_vld4_dup_v:
3459 Int = Intrinsic::arm_neon_vld4lane;
3460 break;
3461 default: llvm_unreachable("unknown vld_dup intrinsic?");
3462 }
3463 Function *F = CGM.getIntrinsic(Int, Ty);
3464 llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType());
3465
3466 SmallVector<Value*, 6> Args;
3467 Args.push_back(Ops[1]);
3468 Args.append(STy->getNumElements(), UndefValue::get(Ty));
3469
3470 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
3471 Args.push_back(CI);
3472 Args.push_back(Align);
3473
3474 Ops[1] = Builder.CreateCall(F, Args, "vld_dup");
3475 // splat lane 0 to all elts in each vector of the result.
3476 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
3477 Value *Val = Builder.CreateExtractValue(Ops[1], i);
3478 Value *Elt = Builder.CreateBitCast(Val, Ty);
3479 Elt = EmitNeonSplat(Elt, CI);
3480 Elt = Builder.CreateBitCast(Elt, Val->getType());
3481 Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i);
3482 }
3483 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3484 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3485 return Builder.CreateStore(Ops[1], Ops[0]);
3486 }
3487 case NEON::BI__builtin_neon_vqrshrn_n_v:
3488 Int =
3489 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
3490 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n",
3491 1, true);
3492 case NEON::BI__builtin_neon_vqrshrun_n_v:
3493 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
3494 Ops, "vqrshrun_n", 1, true);
3495 case NEON::BI__builtin_neon_vqshlu_n_v:
3496 case NEON::BI__builtin_neon_vqshluq_n_v:
3497 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftsu, Ty),
3498 Ops, "vqshlu", 1, false);
3499 case NEON::BI__builtin_neon_vqshrn_n_v:
3500 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
3501 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n",
3502 1, true);
3503 case NEON::BI__builtin_neon_vqshrun_n_v:
3504 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
3505 Ops, "vqshrun_n", 1, true);
3506 case NEON::BI__builtin_neon_vrecpe_v:
3507 case NEON::BI__builtin_neon_vrecpeq_v:
3508 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
3509 Ops, "vrecpe");
3510 case NEON::BI__builtin_neon_vrshrn_n_v:
3511 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
3512 Ops, "vrshrn_n", 1, true);
3513 case NEON::BI__builtin_neon_vrshr_n_v:
3514 case NEON::BI__builtin_neon_vrshrq_n_v:
3515 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
3516 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 1, true);
3517 case NEON::BI__builtin_neon_vrsra_n_v:
3518 case NEON::BI__builtin_neon_vrsraq_n_v:
3519 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3520 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3521 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true);
3522 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
3523 Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]);
3524 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
3525 case NEON::BI__builtin_neon_vsri_n_v:
3526 case NEON::BI__builtin_neon_vsriq_n_v:
3527 rightShift = true;
3528 case NEON::BI__builtin_neon_vsli_n_v:
3529 case NEON::BI__builtin_neon_vsliq_n_v:
3530 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
3531 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
3532 Ops, "vsli_n");
3533 case NEON::BI__builtin_neon_vsra_n_v:
3534 case NEON::BI__builtin_neon_vsraq_n_v:
3535 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
3536 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
3537 return Builder.CreateAdd(Ops[0], Ops[1]);
3538 case NEON::BI__builtin_neon_vst1q_lane_v:
3539 // Handle 64-bit integer elements as a special case. Use a shuffle to get
3540 // a one-element vector and avoid poor code for i64 in the backend.
3541 if (VTy->getElementType()->isIntegerTy(64)) {
3542 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3543 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
3544 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
3545 Ops[2] = Align;
3546 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
3547 Ops[1]->getType()), Ops);
3548 }
3549 // fall through
3550 case NEON::BI__builtin_neon_vst1_lane_v: {
3551 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
3552 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
3553 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
3554 StoreInst *St = Builder.CreateStore(Ops[1],
3555 Builder.CreateBitCast(Ops[0], Ty));
3556 St->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
3557 return St;
3558 }
3559 case NEON::BI__builtin_neon_vtbl1_v:
3560 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
3561 Ops, "vtbl1");
3562 case NEON::BI__builtin_neon_vtbl2_v:
3563 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
3564 Ops, "vtbl2");
3565 case NEON::BI__builtin_neon_vtbl3_v:
3566 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
3567 Ops, "vtbl3");
3568 case NEON::BI__builtin_neon_vtbl4_v:
3569 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
3570 Ops, "vtbl4");
3571 case NEON::BI__builtin_neon_vtbx1_v:
3572 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
3573 Ops, "vtbx1");
3574 case NEON::BI__builtin_neon_vtbx2_v:
3575 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
3576 Ops, "vtbx2");
3577 case NEON::BI__builtin_neon_vtbx3_v:
3578 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
3579 Ops, "vtbx3");
3580 case NEON::BI__builtin_neon_vtbx4_v:
3581 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
3582 Ops, "vtbx4");
3583 }
3584 }
3585
EmitAArch64TblBuiltinExpr(CodeGenFunction & CGF,unsigned BuiltinID,const CallExpr * E,SmallVectorImpl<Value * > & Ops)3586 static Value *EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID,
3587 const CallExpr *E,
3588 SmallVectorImpl<Value *> &Ops) {
3589 unsigned int Int = 0;
3590 const char *s = nullptr;
3591
3592 switch (BuiltinID) {
3593 default:
3594 return nullptr;
3595 case NEON::BI__builtin_neon_vtbl1_v:
3596 case NEON::BI__builtin_neon_vqtbl1_v:
3597 case NEON::BI__builtin_neon_vqtbl1q_v:
3598 case NEON::BI__builtin_neon_vtbl2_v:
3599 case NEON::BI__builtin_neon_vqtbl2_v:
3600 case NEON::BI__builtin_neon_vqtbl2q_v:
3601 case NEON::BI__builtin_neon_vtbl3_v:
3602 case NEON::BI__builtin_neon_vqtbl3_v:
3603 case NEON::BI__builtin_neon_vqtbl3q_v:
3604 case NEON::BI__builtin_neon_vtbl4_v:
3605 case NEON::BI__builtin_neon_vqtbl4_v:
3606 case NEON::BI__builtin_neon_vqtbl4q_v:
3607 break;
3608 case NEON::BI__builtin_neon_vtbx1_v:
3609 case NEON::BI__builtin_neon_vqtbx1_v:
3610 case NEON::BI__builtin_neon_vqtbx1q_v:
3611 case NEON::BI__builtin_neon_vtbx2_v:
3612 case NEON::BI__builtin_neon_vqtbx2_v:
3613 case NEON::BI__builtin_neon_vqtbx2q_v:
3614 case NEON::BI__builtin_neon_vtbx3_v:
3615 case NEON::BI__builtin_neon_vqtbx3_v:
3616 case NEON::BI__builtin_neon_vqtbx3q_v:
3617 case NEON::BI__builtin_neon_vtbx4_v:
3618 case NEON::BI__builtin_neon_vqtbx4_v:
3619 case NEON::BI__builtin_neon_vqtbx4q_v:
3620 break;
3621 }
3622
3623 assert(E->getNumArgs() >= 3);
3624
3625 // Get the last argument, which specifies the vector type.
3626 llvm::APSInt Result;
3627 const Expr *Arg = E->getArg(E->getNumArgs() - 1);
3628 if (!Arg->isIntegerConstantExpr(Result, CGF.getContext()))
3629 return nullptr;
3630
3631 // Determine the type of this overloaded NEON intrinsic.
3632 NeonTypeFlags Type(Result.getZExtValue());
3633 llvm::VectorType *VTy = GetNeonType(&CGF, Type);
3634 llvm::Type *Ty = VTy;
3635 if (!Ty)
3636 return nullptr;
3637
3638 unsigned nElts = VTy->getNumElements();
3639
3640 CodeGen::CGBuilderTy &Builder = CGF.Builder;
3641
3642 // AArch64 scalar builtins are not overloaded, they do not have an extra
3643 // argument that specifies the vector type, need to handle each case.
3644 SmallVector<Value *, 2> TblOps;
3645 switch (BuiltinID) {
3646 case NEON::BI__builtin_neon_vtbl1_v: {
3647 TblOps.push_back(Ops[0]);
3648 return packTBLDVectorList(CGF, TblOps, nullptr, Ops[1], Ty,
3649 Intrinsic::aarch64_neon_tbl1, "vtbl1");
3650 }
3651 case NEON::BI__builtin_neon_vtbl2_v: {
3652 TblOps.push_back(Ops[0]);
3653 TblOps.push_back(Ops[1]);
3654 return packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty,
3655 Intrinsic::aarch64_neon_tbl1, "vtbl1");
3656 }
3657 case NEON::BI__builtin_neon_vtbl3_v: {
3658 TblOps.push_back(Ops[0]);
3659 TblOps.push_back(Ops[1]);
3660 TblOps.push_back(Ops[2]);
3661 return packTBLDVectorList(CGF, TblOps, nullptr, Ops[3], Ty,
3662 Intrinsic::aarch64_neon_tbl2, "vtbl2");
3663 }
3664 case NEON::BI__builtin_neon_vtbl4_v: {
3665 TblOps.push_back(Ops[0]);
3666 TblOps.push_back(Ops[1]);
3667 TblOps.push_back(Ops[2]);
3668 TblOps.push_back(Ops[3]);
3669 return packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty,
3670 Intrinsic::aarch64_neon_tbl2, "vtbl2");
3671 }
3672 case NEON::BI__builtin_neon_vtbx1_v: {
3673 TblOps.push_back(Ops[1]);
3674 Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[2], Ty,
3675 Intrinsic::aarch64_neon_tbl1, "vtbl1");
3676
3677 llvm::Constant *Eight = ConstantInt::get(VTy->getElementType(), 8);
3678 Value* EightV = llvm::ConstantVector::getSplat(nElts, Eight);
3679 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
3680 CmpRes = Builder.CreateSExt(CmpRes, Ty);
3681
3682 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
3683 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
3684 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
3685 }
3686 case NEON::BI__builtin_neon_vtbx2_v: {
3687 TblOps.push_back(Ops[1]);
3688 TblOps.push_back(Ops[2]);
3689 return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[3], Ty,
3690 Intrinsic::aarch64_neon_tbx1, "vtbx1");
3691 }
3692 case NEON::BI__builtin_neon_vtbx3_v: {
3693 TblOps.push_back(Ops[1]);
3694 TblOps.push_back(Ops[2]);
3695 TblOps.push_back(Ops[3]);
3696 Value *TblRes = packTBLDVectorList(CGF, TblOps, nullptr, Ops[4], Ty,
3697 Intrinsic::aarch64_neon_tbl2, "vtbl2");
3698
3699 llvm::Constant *TwentyFour = ConstantInt::get(VTy->getElementType(), 24);
3700 Value* TwentyFourV = llvm::ConstantVector::getSplat(nElts, TwentyFour);
3701 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
3702 TwentyFourV);
3703 CmpRes = Builder.CreateSExt(CmpRes, Ty);
3704
3705 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
3706 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
3707 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
3708 }
3709 case NEON::BI__builtin_neon_vtbx4_v: {
3710 TblOps.push_back(Ops[1]);
3711 TblOps.push_back(Ops[2]);
3712 TblOps.push_back(Ops[3]);
3713 TblOps.push_back(Ops[4]);
3714 return packTBLDVectorList(CGF, TblOps, Ops[0], Ops[5], Ty,
3715 Intrinsic::aarch64_neon_tbx2, "vtbx2");
3716 }
3717 case NEON::BI__builtin_neon_vqtbl1_v:
3718 case NEON::BI__builtin_neon_vqtbl1q_v:
3719 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break;
3720 case NEON::BI__builtin_neon_vqtbl2_v:
3721 case NEON::BI__builtin_neon_vqtbl2q_v: {
3722 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break;
3723 case NEON::BI__builtin_neon_vqtbl3_v:
3724 case NEON::BI__builtin_neon_vqtbl3q_v:
3725 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break;
3726 case NEON::BI__builtin_neon_vqtbl4_v:
3727 case NEON::BI__builtin_neon_vqtbl4q_v:
3728 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break;
3729 case NEON::BI__builtin_neon_vqtbx1_v:
3730 case NEON::BI__builtin_neon_vqtbx1q_v:
3731 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break;
3732 case NEON::BI__builtin_neon_vqtbx2_v:
3733 case NEON::BI__builtin_neon_vqtbx2q_v:
3734 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break;
3735 case NEON::BI__builtin_neon_vqtbx3_v:
3736 case NEON::BI__builtin_neon_vqtbx3q_v:
3737 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break;
3738 case NEON::BI__builtin_neon_vqtbx4_v:
3739 case NEON::BI__builtin_neon_vqtbx4q_v:
3740 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break;
3741 }
3742 }
3743
3744 if (!Int)
3745 return nullptr;
3746
3747 Function *F = CGF.CGM.getIntrinsic(Int, Ty);
3748 return CGF.EmitNeonCall(F, Ops, s);
3749 }
3750
vectorWrapScalar16(Value * Op)3751 Value *CodeGenFunction::vectorWrapScalar16(Value *Op) {
3752 llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4);
3753 Op = Builder.CreateBitCast(Op, Int16Ty);
3754 Value *V = UndefValue::get(VTy);
3755 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
3756 Op = Builder.CreateInsertElement(V, Op, CI);
3757 return Op;
3758 }
3759
vectorWrapScalar8(Value * Op)3760 Value *CodeGenFunction::vectorWrapScalar8(Value *Op) {
3761 llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8);
3762 Op = Builder.CreateBitCast(Op, Int8Ty);
3763 Value *V = UndefValue::get(VTy);
3764 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
3765 Op = Builder.CreateInsertElement(V, Op, CI);
3766 return Op;
3767 }
3768
3769 Value *CodeGenFunction::
emitVectorWrappedScalar8Intrinsic(unsigned Int,SmallVectorImpl<Value * > & Ops,const char * Name)3770 emitVectorWrappedScalar8Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops,
3771 const char *Name) {
3772 // i8 is not a legal types for AArch64, so we can't just use
3773 // a normal overloaded intrinsic call for these scalar types. Instead
3774 // we'll build 64-bit vectors w/ lane zero being our input values and
3775 // perform the operation on that. The back end can pattern match directly
3776 // to the scalar instruction.
3777 Ops[0] = vectorWrapScalar8(Ops[0]);
3778 Ops[1] = vectorWrapScalar8(Ops[1]);
3779 llvm::Type *VTy = llvm::VectorType::get(Int8Ty, 8);
3780 Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name);
3781 Constant *CI = ConstantInt::get(SizeTy, 0);
3782 return Builder.CreateExtractElement(V, CI, "lane0");
3783 }
3784
3785 Value *CodeGenFunction::
emitVectorWrappedScalar16Intrinsic(unsigned Int,SmallVectorImpl<Value * > & Ops,const char * Name)3786 emitVectorWrappedScalar16Intrinsic(unsigned Int, SmallVectorImpl<Value*> &Ops,
3787 const char *Name) {
3788 // i16 is not a legal types for AArch64, so we can't just use
3789 // a normal overloaded intrinsic call for these scalar types. Instead
3790 // we'll build 64-bit vectors w/ lane zero being our input values and
3791 // perform the operation on that. The back end can pattern match directly
3792 // to the scalar instruction.
3793 Ops[0] = vectorWrapScalar16(Ops[0]);
3794 Ops[1] = vectorWrapScalar16(Ops[1]);
3795 llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4);
3796 Value *V = EmitNeonCall(CGM.getIntrinsic(Int, VTy), Ops, Name);
3797 Constant *CI = ConstantInt::get(SizeTy, 0);
3798 return Builder.CreateExtractElement(V, CI, "lane0");
3799 }
3800
EmitAArch64BuiltinExpr(unsigned BuiltinID,const CallExpr * E)3801 Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
3802 const CallExpr *E) {
3803 if (BuiltinID == AArch64::BI__builtin_arm_rbit) {
3804 assert((getContext().getTypeSize(E->getType()) == 32) &&
3805 "rbit of unusual size!");
3806 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
3807 return Builder.CreateCall(
3808 CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit");
3809 }
3810 if (BuiltinID == AArch64::BI__builtin_arm_rbit64) {
3811 assert((getContext().getTypeSize(E->getType()) == 64) &&
3812 "rbit of unusual size!");
3813 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
3814 return Builder.CreateCall(
3815 CGM.getIntrinsic(Intrinsic::aarch64_rbit, Arg->getType()), Arg, "rbit");
3816 }
3817
3818 if (BuiltinID == AArch64::BI__clear_cache) {
3819 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
3820 const FunctionDecl *FD = E->getDirectCallee();
3821 SmallVector<Value*, 2> Ops;
3822 for (unsigned i = 0; i < 2; i++)
3823 Ops.push_back(EmitScalarExpr(E->getArg(i)));
3824 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
3825 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
3826 StringRef Name = FD->getName();
3827 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
3828 }
3829
3830 if ((BuiltinID == AArch64::BI__builtin_arm_ldrex ||
3831 BuiltinID == AArch64::BI__builtin_arm_ldaex) &&
3832 getContext().getTypeSize(E->getType()) == 128) {
3833 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
3834 ? Intrinsic::aarch64_ldaxp
3835 : Intrinsic::aarch64_ldxp);
3836
3837 Value *LdPtr = EmitScalarExpr(E->getArg(0));
3838 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
3839 "ldxp");
3840
3841 Value *Val0 = Builder.CreateExtractValue(Val, 1);
3842 Value *Val1 = Builder.CreateExtractValue(Val, 0);
3843 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
3844 Val0 = Builder.CreateZExt(Val0, Int128Ty);
3845 Val1 = Builder.CreateZExt(Val1, Int128Ty);
3846
3847 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
3848 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
3849 Val = Builder.CreateOr(Val, Val1);
3850 return Builder.CreateBitCast(Val, ConvertType(E->getType()));
3851 } else if (BuiltinID == AArch64::BI__builtin_arm_ldrex ||
3852 BuiltinID == AArch64::BI__builtin_arm_ldaex) {
3853 Value *LoadAddr = EmitScalarExpr(E->getArg(0));
3854
3855 QualType Ty = E->getType();
3856 llvm::Type *RealResTy = ConvertType(Ty);
3857 llvm::Type *IntResTy = llvm::IntegerType::get(getLLVMContext(),
3858 getContext().getTypeSize(Ty));
3859 LoadAddr = Builder.CreateBitCast(LoadAddr, IntResTy->getPointerTo());
3860
3861 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
3862 ? Intrinsic::aarch64_ldaxr
3863 : Intrinsic::aarch64_ldxr,
3864 LoadAddr->getType());
3865 Value *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
3866
3867 if (RealResTy->isPointerTy())
3868 return Builder.CreateIntToPtr(Val, RealResTy);
3869
3870 Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
3871 return Builder.CreateBitCast(Val, RealResTy);
3872 }
3873
3874 if ((BuiltinID == AArch64::BI__builtin_arm_strex ||
3875 BuiltinID == AArch64::BI__builtin_arm_stlex) &&
3876 getContext().getTypeSize(E->getArg(0)->getType()) == 128) {
3877 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
3878 ? Intrinsic::aarch64_stlxp
3879 : Intrinsic::aarch64_stxp);
3880 llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty, NULL);
3881
3882 Value *One = llvm::ConstantInt::get(Int32Ty, 1);
3883 Value *Tmp = Builder.CreateAlloca(ConvertType(E->getArg(0)->getType()),
3884 One);
3885 Value *Val = EmitScalarExpr(E->getArg(0));
3886 Builder.CreateStore(Val, Tmp);
3887
3888 Value *LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
3889 Val = Builder.CreateLoad(LdPtr);
3890
3891 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
3892 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
3893 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)),
3894 Int8PtrTy);
3895 return Builder.CreateCall3(F, Arg0, Arg1, StPtr, "stxp");
3896 } else if (BuiltinID == AArch64::BI__builtin_arm_strex ||
3897 BuiltinID == AArch64::BI__builtin_arm_stlex) {
3898 Value *StoreVal = EmitScalarExpr(E->getArg(0));
3899 Value *StoreAddr = EmitScalarExpr(E->getArg(1));
3900
3901 QualType Ty = E->getArg(0)->getType();
3902 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
3903 getContext().getTypeSize(Ty));
3904 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
3905
3906 if (StoreVal->getType()->isPointerTy())
3907 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty);
3908 else {
3909 StoreVal = Builder.CreateBitCast(StoreVal, StoreTy);
3910 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
3911 }
3912
3913 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
3914 ? Intrinsic::aarch64_stlxr
3915 : Intrinsic::aarch64_stxr,
3916 StoreAddr->getType());
3917 return Builder.CreateCall2(F, StoreVal, StoreAddr, "stxr");
3918 }
3919
3920 if (BuiltinID == AArch64::BI__builtin_arm_clrex) {
3921 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
3922 return Builder.CreateCall(F);
3923 }
3924
3925 // CRC32
3926 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
3927 switch (BuiltinID) {
3928 case AArch64::BI__builtin_arm_crc32b:
3929 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
3930 case AArch64::BI__builtin_arm_crc32cb:
3931 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
3932 case AArch64::BI__builtin_arm_crc32h:
3933 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
3934 case AArch64::BI__builtin_arm_crc32ch:
3935 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
3936 case AArch64::BI__builtin_arm_crc32w:
3937 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
3938 case AArch64::BI__builtin_arm_crc32cw:
3939 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
3940 case AArch64::BI__builtin_arm_crc32d:
3941 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
3942 case AArch64::BI__builtin_arm_crc32cd:
3943 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
3944 }
3945
3946 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
3947 Value *Arg0 = EmitScalarExpr(E->getArg(0));
3948 Value *Arg1 = EmitScalarExpr(E->getArg(1));
3949 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
3950
3951 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
3952 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
3953
3954 return Builder.CreateCall2(F, Arg0, Arg1);
3955 }
3956
3957 llvm::SmallVector<Value*, 4> Ops;
3958 for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++)
3959 Ops.push_back(EmitScalarExpr(E->getArg(i)));
3960
3961 ArrayRef<NeonIntrinsicInfo> SISDMap(AArch64SISDIntrinsicMap);
3962 const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap(
3963 SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted);
3964
3965 if (Builtin) {
3966 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1)));
3967 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E);
3968 assert(Result && "SISD intrinsic should have been handled");
3969 return Result;
3970 }
3971
3972 llvm::APSInt Result;
3973 const Expr *Arg = E->getArg(E->getNumArgs()-1);
3974 NeonTypeFlags Type(0);
3975 if (Arg->isIntegerConstantExpr(Result, getContext()))
3976 // Determine the type of this overloaded NEON intrinsic.
3977 Type = NeonTypeFlags(Result.getZExtValue());
3978
3979 bool usgn = Type.isUnsigned();
3980 bool quad = Type.isQuad();
3981
3982 // Handle non-overloaded intrinsics first.
3983 switch (BuiltinID) {
3984 default: break;
3985 case NEON::BI__builtin_neon_vldrq_p128: {
3986 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128);
3987 Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)), Int128PTy);
3988 return Builder.CreateLoad(Ptr);
3989 }
3990 case NEON::BI__builtin_neon_vstrq_p128: {
3991 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128);
3992 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
3993 return Builder.CreateStore(EmitScalarExpr(E->getArg(1)), Ptr);
3994 }
3995 case NEON::BI__builtin_neon_vcvts_u32_f32:
3996 case NEON::BI__builtin_neon_vcvtd_u64_f64:
3997 usgn = true;
3998 // FALL THROUGH
3999 case NEON::BI__builtin_neon_vcvts_s32_f32:
4000 case NEON::BI__builtin_neon_vcvtd_s64_f64: {
4001 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4002 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
4003 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
4004 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
4005 Ops[0] = Builder.CreateBitCast(Ops[0], FTy);
4006 if (usgn)
4007 return Builder.CreateFPToUI(Ops[0], InTy);
4008 return Builder.CreateFPToSI(Ops[0], InTy);
4009 }
4010 case NEON::BI__builtin_neon_vcvts_f32_u32:
4011 case NEON::BI__builtin_neon_vcvtd_f64_u64:
4012 usgn = true;
4013 // FALL THROUGH
4014 case NEON::BI__builtin_neon_vcvts_f32_s32:
4015 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
4016 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4017 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
4018 llvm::Type *InTy = Is64 ? Int64Ty : Int32Ty;
4019 llvm::Type *FTy = Is64 ? DoubleTy : FloatTy;
4020 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
4021 if (usgn)
4022 return Builder.CreateUIToFP(Ops[0], FTy);
4023 return Builder.CreateSIToFP(Ops[0], FTy);
4024 }
4025 case NEON::BI__builtin_neon_vpaddd_s64: {
4026 llvm::Type *Ty =
4027 llvm::VectorType::get(llvm::Type::getInt64Ty(getLLVMContext()), 2);
4028 Value *Vec = EmitScalarExpr(E->getArg(0));
4029 // The vector is v2f64, so make sure it's bitcast to that.
4030 Vec = Builder.CreateBitCast(Vec, Ty, "v2i64");
4031 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
4032 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
4033 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
4034 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
4035 // Pairwise addition of a v2f64 into a scalar f64.
4036 return Builder.CreateAdd(Op0, Op1, "vpaddd");
4037 }
4038 case NEON::BI__builtin_neon_vpaddd_f64: {
4039 llvm::Type *Ty =
4040 llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2);
4041 Value *Vec = EmitScalarExpr(E->getArg(0));
4042 // The vector is v2f64, so make sure it's bitcast to that.
4043 Vec = Builder.CreateBitCast(Vec, Ty, "v2f64");
4044 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
4045 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
4046 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
4047 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
4048 // Pairwise addition of a v2f64 into a scalar f64.
4049 return Builder.CreateFAdd(Op0, Op1, "vpaddd");
4050 }
4051 case NEON::BI__builtin_neon_vpadds_f32: {
4052 llvm::Type *Ty =
4053 llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2);
4054 Value *Vec = EmitScalarExpr(E->getArg(0));
4055 // The vector is v2f32, so make sure it's bitcast to that.
4056 Vec = Builder.CreateBitCast(Vec, Ty, "v2f32");
4057 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
4058 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
4059 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
4060 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
4061 // Pairwise addition of a v2f32 into a scalar f32.
4062 return Builder.CreateFAdd(Op0, Op1, "vpaddd");
4063 }
4064 case NEON::BI__builtin_neon_vceqzd_s64:
4065 case NEON::BI__builtin_neon_vceqzd_f64:
4066 case NEON::BI__builtin_neon_vceqzs_f32:
4067 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4068 return EmitAArch64CompareBuiltinExpr(
4069 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OEQ,
4070 ICmpInst::ICMP_EQ, "vceqz");
4071 case NEON::BI__builtin_neon_vcgezd_s64:
4072 case NEON::BI__builtin_neon_vcgezd_f64:
4073 case NEON::BI__builtin_neon_vcgezs_f32:
4074 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4075 return EmitAArch64CompareBuiltinExpr(
4076 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGE,
4077 ICmpInst::ICMP_SGE, "vcgez");
4078 case NEON::BI__builtin_neon_vclezd_s64:
4079 case NEON::BI__builtin_neon_vclezd_f64:
4080 case NEON::BI__builtin_neon_vclezs_f32:
4081 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4082 return EmitAArch64CompareBuiltinExpr(
4083 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLE,
4084 ICmpInst::ICMP_SLE, "vclez");
4085 case NEON::BI__builtin_neon_vcgtzd_s64:
4086 case NEON::BI__builtin_neon_vcgtzd_f64:
4087 case NEON::BI__builtin_neon_vcgtzs_f32:
4088 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4089 return EmitAArch64CompareBuiltinExpr(
4090 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OGT,
4091 ICmpInst::ICMP_SGT, "vcgtz");
4092 case NEON::BI__builtin_neon_vcltzd_s64:
4093 case NEON::BI__builtin_neon_vcltzd_f64:
4094 case NEON::BI__builtin_neon_vcltzs_f32:
4095 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4096 return EmitAArch64CompareBuiltinExpr(
4097 Ops[0], ConvertType(E->getCallReturnType()), ICmpInst::FCMP_OLT,
4098 ICmpInst::ICMP_SLT, "vcltz");
4099
4100 case NEON::BI__builtin_neon_vceqzd_u64: {
4101 llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext());
4102 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4103 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4104 Ops[0] = Builder.CreateICmp(llvm::ICmpInst::ICMP_EQ, Ops[0],
4105 llvm::Constant::getNullValue(Ty));
4106 return Builder.CreateSExt(Ops[0], Ty, "vceqzd");
4107 }
4108 case NEON::BI__builtin_neon_vceqd_f64:
4109 case NEON::BI__builtin_neon_vcled_f64:
4110 case NEON::BI__builtin_neon_vcltd_f64:
4111 case NEON::BI__builtin_neon_vcged_f64:
4112 case NEON::BI__builtin_neon_vcgtd_f64: {
4113 llvm::CmpInst::Predicate P;
4114 switch (BuiltinID) {
4115 default: llvm_unreachable("missing builtin ID in switch!");
4116 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ; break;
4117 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE; break;
4118 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT; break;
4119 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE; break;
4120 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT; break;
4121 }
4122 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4123 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
4124 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
4125 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
4126 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd");
4127 }
4128 case NEON::BI__builtin_neon_vceqs_f32:
4129 case NEON::BI__builtin_neon_vcles_f32:
4130 case NEON::BI__builtin_neon_vclts_f32:
4131 case NEON::BI__builtin_neon_vcges_f32:
4132 case NEON::BI__builtin_neon_vcgts_f32: {
4133 llvm::CmpInst::Predicate P;
4134 switch (BuiltinID) {
4135 default: llvm_unreachable("missing builtin ID in switch!");
4136 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ; break;
4137 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE; break;
4138 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT; break;
4139 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE; break;
4140 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT; break;
4141 }
4142 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4143 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
4144 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
4145 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
4146 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd");
4147 }
4148 case NEON::BI__builtin_neon_vceqd_s64:
4149 case NEON::BI__builtin_neon_vceqd_u64:
4150 case NEON::BI__builtin_neon_vcgtd_s64:
4151 case NEON::BI__builtin_neon_vcgtd_u64:
4152 case NEON::BI__builtin_neon_vcltd_s64:
4153 case NEON::BI__builtin_neon_vcltd_u64:
4154 case NEON::BI__builtin_neon_vcged_u64:
4155 case NEON::BI__builtin_neon_vcged_s64:
4156 case NEON::BI__builtin_neon_vcled_u64:
4157 case NEON::BI__builtin_neon_vcled_s64: {
4158 llvm::CmpInst::Predicate P;
4159 switch (BuiltinID) {
4160 default: llvm_unreachable("missing builtin ID in switch!");
4161 case NEON::BI__builtin_neon_vceqd_s64:
4162 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;break;
4163 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;break;
4164 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;break;
4165 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;break;
4166 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;break;
4167 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;break;
4168 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;break;
4169 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;break;
4170 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;break;
4171 }
4172 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4173 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
4174 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
4175 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
4176 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd");
4177 }
4178 case NEON::BI__builtin_neon_vtstd_s64:
4179 case NEON::BI__builtin_neon_vtstd_u64: {
4180 llvm::Type *Ty = llvm::Type::getInt64Ty(getLLVMContext());
4181 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4182 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4183 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4184 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
4185 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
4186 llvm::Constant::getNullValue(Ty));
4187 return Builder.CreateSExt(Ops[0], Ty, "vtstd");
4188 }
4189 case NEON::BI__builtin_neon_vset_lane_i8:
4190 case NEON::BI__builtin_neon_vset_lane_i16:
4191 case NEON::BI__builtin_neon_vset_lane_i32:
4192 case NEON::BI__builtin_neon_vset_lane_i64:
4193 case NEON::BI__builtin_neon_vset_lane_f32:
4194 case NEON::BI__builtin_neon_vsetq_lane_i8:
4195 case NEON::BI__builtin_neon_vsetq_lane_i16:
4196 case NEON::BI__builtin_neon_vsetq_lane_i32:
4197 case NEON::BI__builtin_neon_vsetq_lane_i64:
4198 case NEON::BI__builtin_neon_vsetq_lane_f32:
4199 Ops.push_back(EmitScalarExpr(E->getArg(2)));
4200 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
4201 case NEON::BI__builtin_neon_vset_lane_f64:
4202 // The vector type needs a cast for the v1f64 variant.
4203 Ops[1] = Builder.CreateBitCast(Ops[1],
4204 llvm::VectorType::get(DoubleTy, 1));
4205 Ops.push_back(EmitScalarExpr(E->getArg(2)));
4206 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
4207 case NEON::BI__builtin_neon_vsetq_lane_f64:
4208 // The vector type needs a cast for the v2f64 variant.
4209 Ops[1] = Builder.CreateBitCast(Ops[1],
4210 llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2));
4211 Ops.push_back(EmitScalarExpr(E->getArg(2)));
4212 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
4213
4214 case NEON::BI__builtin_neon_vget_lane_i8:
4215 case NEON::BI__builtin_neon_vdupb_lane_i8:
4216 Ops[0] = Builder.CreateBitCast(Ops[0],
4217 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8));
4218 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4219 "vget_lane");
4220 case NEON::BI__builtin_neon_vgetq_lane_i8:
4221 case NEON::BI__builtin_neon_vdupb_laneq_i8:
4222 Ops[0] = Builder.CreateBitCast(Ops[0],
4223 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16));
4224 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4225 "vgetq_lane");
4226 case NEON::BI__builtin_neon_vget_lane_i16:
4227 case NEON::BI__builtin_neon_vduph_lane_i16:
4228 Ops[0] = Builder.CreateBitCast(Ops[0],
4229 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4));
4230 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4231 "vget_lane");
4232 case NEON::BI__builtin_neon_vgetq_lane_i16:
4233 case NEON::BI__builtin_neon_vduph_laneq_i16:
4234 Ops[0] = Builder.CreateBitCast(Ops[0],
4235 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8));
4236 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4237 "vgetq_lane");
4238 case NEON::BI__builtin_neon_vget_lane_i32:
4239 case NEON::BI__builtin_neon_vdups_lane_i32:
4240 Ops[0] = Builder.CreateBitCast(
4241 Ops[0],
4242 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 2));
4243 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4244 "vget_lane");
4245 case NEON::BI__builtin_neon_vdups_lane_f32:
4246 Ops[0] = Builder.CreateBitCast(Ops[0],
4247 llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2));
4248 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4249 "vdups_lane");
4250 case NEON::BI__builtin_neon_vgetq_lane_i32:
4251 case NEON::BI__builtin_neon_vdups_laneq_i32:
4252 Ops[0] = Builder.CreateBitCast(Ops[0],
4253 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 32), 4));
4254 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4255 "vgetq_lane");
4256 case NEON::BI__builtin_neon_vget_lane_i64:
4257 case NEON::BI__builtin_neon_vdupd_lane_i64:
4258 Ops[0] = Builder.CreateBitCast(Ops[0],
4259 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 1));
4260 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4261 "vget_lane");
4262 case NEON::BI__builtin_neon_vdupd_lane_f64:
4263 Ops[0] = Builder.CreateBitCast(Ops[0],
4264 llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1));
4265 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4266 "vdupd_lane");
4267 case NEON::BI__builtin_neon_vgetq_lane_i64:
4268 case NEON::BI__builtin_neon_vdupd_laneq_i64:
4269 Ops[0] = Builder.CreateBitCast(Ops[0],
4270 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 64), 2));
4271 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4272 "vgetq_lane");
4273 case NEON::BI__builtin_neon_vget_lane_f32:
4274 Ops[0] = Builder.CreateBitCast(Ops[0],
4275 llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 2));
4276 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4277 "vget_lane");
4278 case NEON::BI__builtin_neon_vget_lane_f64:
4279 Ops[0] = Builder.CreateBitCast(Ops[0],
4280 llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 1));
4281 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4282 "vget_lane");
4283 case NEON::BI__builtin_neon_vgetq_lane_f32:
4284 case NEON::BI__builtin_neon_vdups_laneq_f32:
4285 Ops[0] = Builder.CreateBitCast(Ops[0],
4286 llvm::VectorType::get(llvm::Type::getFloatTy(getLLVMContext()), 4));
4287 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4288 "vgetq_lane");
4289 case NEON::BI__builtin_neon_vgetq_lane_f64:
4290 case NEON::BI__builtin_neon_vdupd_laneq_f64:
4291 Ops[0] = Builder.CreateBitCast(Ops[0],
4292 llvm::VectorType::get(llvm::Type::getDoubleTy(getLLVMContext()), 2));
4293 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
4294 "vgetq_lane");
4295 case NEON::BI__builtin_neon_vaddd_s64:
4296 case NEON::BI__builtin_neon_vaddd_u64:
4297 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd");
4298 case NEON::BI__builtin_neon_vsubd_s64:
4299 case NEON::BI__builtin_neon_vsubd_u64:
4300 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd");
4301 case NEON::BI__builtin_neon_vqdmlalh_s16:
4302 case NEON::BI__builtin_neon_vqdmlslh_s16: {
4303 SmallVector<Value *, 2> ProductOps;
4304 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
4305 ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2))));
4306 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4);
4307 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
4308 ProductOps, "vqdmlXl");
4309 Constant *CI = ConstantInt::get(SizeTy, 0);
4310 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
4311
4312 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
4313 ? Intrinsic::aarch64_neon_sqadd
4314 : Intrinsic::aarch64_neon_sqsub;
4315 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl");
4316 }
4317 case NEON::BI__builtin_neon_vqshlud_n_s64: {
4318 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4319 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
4320 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty),
4321 Ops, "vqshlu_n");
4322 }
4323 case NEON::BI__builtin_neon_vqshld_n_u64:
4324 case NEON::BI__builtin_neon_vqshld_n_s64: {
4325 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
4326 ? Intrinsic::aarch64_neon_uqshl
4327 : Intrinsic::aarch64_neon_sqshl;
4328 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4329 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
4330 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n");
4331 }
4332 case NEON::BI__builtin_neon_vrshrd_n_u64:
4333 case NEON::BI__builtin_neon_vrshrd_n_s64: {
4334 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
4335 ? Intrinsic::aarch64_neon_urshl
4336 : Intrinsic::aarch64_neon_srshl;
4337 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4338 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
4339 Ops[1] = ConstantInt::get(Int64Ty, -SV);
4340 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n");
4341 }
4342 case NEON::BI__builtin_neon_vrsrad_n_u64:
4343 case NEON::BI__builtin_neon_vrsrad_n_s64: {
4344 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
4345 ? Intrinsic::aarch64_neon_urshl
4346 : Intrinsic::aarch64_neon_srshl;
4347 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
4348 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2))));
4349 Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Int64Ty), Ops[1],
4350 Builder.CreateSExt(Ops[2], Int64Ty));
4351 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
4352 }
4353 case NEON::BI__builtin_neon_vshld_n_s64:
4354 case NEON::BI__builtin_neon_vshld_n_u64: {
4355 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
4356 return Builder.CreateShl(
4357 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n");
4358 }
4359 case NEON::BI__builtin_neon_vshrd_n_s64: {
4360 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
4361 return Builder.CreateAShr(
4362 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
4363 Amt->getZExtValue())),
4364 "shrd_n");
4365 }
4366 case NEON::BI__builtin_neon_vshrd_n_u64: {
4367 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
4368 uint64_t ShiftAmt = Amt->getZExtValue();
4369 // Right-shifting an unsigned value by its size yields 0.
4370 if (ShiftAmt == 64)
4371 return ConstantInt::get(Int64Ty, 0);
4372 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
4373 "shrd_n");
4374 }
4375 case NEON::BI__builtin_neon_vsrad_n_s64: {
4376 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2)));
4377 Ops[1] = Builder.CreateAShr(
4378 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63),
4379 Amt->getZExtValue())),
4380 "shrd_n");
4381 return Builder.CreateAdd(Ops[0], Ops[1]);
4382 }
4383 case NEON::BI__builtin_neon_vsrad_n_u64: {
4384 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2)));
4385 uint64_t ShiftAmt = Amt->getZExtValue();
4386 // Right-shifting an unsigned value by its size yields 0.
4387 // As Op + 0 = Op, return Ops[0] directly.
4388 if (ShiftAmt == 64)
4389 return Ops[0];
4390 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
4391 "shrd_n");
4392 return Builder.CreateAdd(Ops[0], Ops[1]);
4393 }
4394 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
4395 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
4396 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
4397 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
4398 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
4399 "lane");
4400 SmallVector<Value *, 2> ProductOps;
4401 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
4402 ProductOps.push_back(vectorWrapScalar16(Ops[2]));
4403 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4);
4404 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
4405 ProductOps, "vqdmlXl");
4406 Constant *CI = ConstantInt::get(SizeTy, 0);
4407 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
4408 Ops.pop_back();
4409
4410 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
4411 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
4412 ? Intrinsic::aarch64_neon_sqadd
4413 : Intrinsic::aarch64_neon_sqsub;
4414 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl");
4415 }
4416 case NEON::BI__builtin_neon_vqdmlals_s32:
4417 case NEON::BI__builtin_neon_vqdmlsls_s32: {
4418 SmallVector<Value *, 2> ProductOps;
4419 ProductOps.push_back(Ops[1]);
4420 ProductOps.push_back(EmitScalarExpr(E->getArg(2)));
4421 Ops[1] =
4422 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
4423 ProductOps, "vqdmlXl");
4424
4425 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
4426 ? Intrinsic::aarch64_neon_sqadd
4427 : Intrinsic::aarch64_neon_sqsub;
4428 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl");
4429 }
4430 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
4431 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
4432 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
4433 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
4434 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
4435 "lane");
4436 SmallVector<Value *, 2> ProductOps;
4437 ProductOps.push_back(Ops[1]);
4438 ProductOps.push_back(Ops[2]);
4439 Ops[1] =
4440 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
4441 ProductOps, "vqdmlXl");
4442 Ops.pop_back();
4443
4444 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
4445 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
4446 ? Intrinsic::aarch64_neon_sqadd
4447 : Intrinsic::aarch64_neon_sqsub;
4448 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl");
4449 }
4450 }
4451
4452 llvm::VectorType *VTy = GetNeonType(this, Type);
4453 llvm::Type *Ty = VTy;
4454 if (!Ty)
4455 return nullptr;
4456
4457 // Not all intrinsics handled by the common case work for AArch64 yet, so only
4458 // defer to common code if it's been added to our special map.
4459 Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID,
4460 AArch64SIMDIntrinsicsProvenSorted);
4461
4462 if (Builtin)
4463 return EmitCommonNeonBuiltinExpr(
4464 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
4465 Builtin->NameHint, Builtin->TypeModifier, E, Ops, nullptr);
4466
4467 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops))
4468 return V;
4469
4470 unsigned Int;
4471 switch (BuiltinID) {
4472 default: return nullptr;
4473 case NEON::BI__builtin_neon_vbsl_v:
4474 case NEON::BI__builtin_neon_vbslq_v: {
4475 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
4476 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl");
4477 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl");
4478 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl");
4479
4480 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
4481 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
4482 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
4483 return Builder.CreateBitCast(Ops[0], Ty);
4484 }
4485 case NEON::BI__builtin_neon_vfma_lane_v:
4486 case NEON::BI__builtin_neon_vfmaq_lane_v: { // Only used for FP types
4487 // The ARM builtins (and instructions) have the addend as the first
4488 // operand, but the 'fma' intrinsics have it last. Swap it around here.
4489 Value *Addend = Ops[0];
4490 Value *Multiplicand = Ops[1];
4491 Value *LaneSource = Ops[2];
4492 Ops[0] = Multiplicand;
4493 Ops[1] = LaneSource;
4494 Ops[2] = Addend;
4495
4496 // Now adjust things to handle the lane access.
4497 llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ?
4498 llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) :
4499 VTy;
4500 llvm::Constant *cst = cast<Constant>(Ops[3]);
4501 Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst);
4502 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
4503 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
4504
4505 Ops.pop_back();
4506 Int = Intrinsic::fma;
4507 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla");
4508 }
4509 case NEON::BI__builtin_neon_vfma_laneq_v: {
4510 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
4511 // v1f64 fma should be mapped to Neon scalar f64 fma
4512 if (VTy && VTy->getElementType() == DoubleTy) {
4513 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
4514 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
4515 llvm::Type *VTy = GetNeonType(this,
4516 NeonTypeFlags(NeonTypeFlags::Float64, false, true));
4517 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
4518 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
4519 Value *F = CGM.getIntrinsic(Intrinsic::fma, DoubleTy);
4520 Value *Result = Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
4521 return Builder.CreateBitCast(Result, Ty);
4522 }
4523 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
4524 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4525 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4526
4527 llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(),
4528 VTy->getNumElements() * 2);
4529 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
4530 Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(),
4531 cast<ConstantInt>(Ops[3]));
4532 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
4533
4534 return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]);
4535 }
4536 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
4537 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
4538 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4539 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
4540
4541 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
4542 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
4543 return Builder.CreateCall3(F, Ops[2], Ops[1], Ops[0]);
4544 }
4545 case NEON::BI__builtin_neon_vfmas_lane_f32:
4546 case NEON::BI__builtin_neon_vfmas_laneq_f32:
4547 case NEON::BI__builtin_neon_vfmad_lane_f64:
4548 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
4549 Ops.push_back(EmitScalarExpr(E->getArg(3)));
4550 llvm::Type *Ty = ConvertType(E->getCallReturnType());
4551 Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
4552 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
4553 return Builder.CreateCall3(F, Ops[1], Ops[2], Ops[0]);
4554 }
4555 case NEON::BI__builtin_neon_vfms_v:
4556 case NEON::BI__builtin_neon_vfmsq_v: { // Only used for FP types
4557 // FIXME: probably remove when we no longer support aarch64_simd.h
4558 // (arm_neon.h delegates to vfma).
4559
4560 // The ARM builtins (and instructions) have the addend as the first
4561 // operand, but the 'fma' intrinsics have it last. Swap it around here.
4562 Value *Subtrahend = Ops[0];
4563 Value *Multiplicand = Ops[2];
4564 Ops[0] = Multiplicand;
4565 Ops[2] = Subtrahend;
4566 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
4567 Ops[1] = Builder.CreateFNeg(Ops[1]);
4568 Int = Intrinsic::fma;
4569 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmls");
4570 }
4571 case NEON::BI__builtin_neon_vmull_v:
4572 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
4573 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
4574 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
4575 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
4576 case NEON::BI__builtin_neon_vmax_v:
4577 case NEON::BI__builtin_neon_vmaxq_v:
4578 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
4579 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
4580 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
4581 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax");
4582 case NEON::BI__builtin_neon_vmin_v:
4583 case NEON::BI__builtin_neon_vminq_v:
4584 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
4585 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
4586 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
4587 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin");
4588 case NEON::BI__builtin_neon_vabd_v:
4589 case NEON::BI__builtin_neon_vabdq_v:
4590 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
4591 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
4592 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
4593 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd");
4594 case NEON::BI__builtin_neon_vpadal_v:
4595 case NEON::BI__builtin_neon_vpadalq_v: {
4596 unsigned ArgElts = VTy->getNumElements();
4597 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
4598 unsigned BitWidth = EltTy->getBitWidth();
4599 llvm::Type *ArgTy = llvm::VectorType::get(
4600 llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts);
4601 llvm::Type* Tys[2] = { VTy, ArgTy };
4602 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
4603 SmallVector<llvm::Value*, 1> TmpOps;
4604 TmpOps.push_back(Ops[1]);
4605 Function *F = CGM.getIntrinsic(Int, Tys);
4606 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vpadal");
4607 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
4608 return Builder.CreateAdd(tmp, addend);
4609 }
4610 case NEON::BI__builtin_neon_vpmin_v:
4611 case NEON::BI__builtin_neon_vpminq_v:
4612 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
4613 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
4614 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
4615 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin");
4616 case NEON::BI__builtin_neon_vpmax_v:
4617 case NEON::BI__builtin_neon_vpmaxq_v:
4618 // FIXME: improve sharing scheme to cope with 3 alternative LLVM intrinsics.
4619 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
4620 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
4621 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax");
4622 case NEON::BI__builtin_neon_vminnm_v:
4623 case NEON::BI__builtin_neon_vminnmq_v:
4624 Int = Intrinsic::aarch64_neon_fminnm;
4625 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm");
4626 case NEON::BI__builtin_neon_vmaxnm_v:
4627 case NEON::BI__builtin_neon_vmaxnmq_v:
4628 Int = Intrinsic::aarch64_neon_fmaxnm;
4629 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm");
4630 case NEON::BI__builtin_neon_vrecpss_f32: {
4631 llvm::Type *f32Type = llvm::Type::getFloatTy(getLLVMContext());
4632 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4633 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f32Type),
4634 Ops, "vrecps");
4635 }
4636 case NEON::BI__builtin_neon_vrecpsd_f64: {
4637 llvm::Type *f64Type = llvm::Type::getDoubleTy(getLLVMContext());
4638 Ops.push_back(EmitScalarExpr(E->getArg(1)));
4639 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, f64Type),
4640 Ops, "vrecps");
4641 }
4642 case NEON::BI__builtin_neon_vrshr_n_v:
4643 case NEON::BI__builtin_neon_vrshrq_n_v:
4644 // FIXME: this can be shared with 32-bit ARM, but not AArch64 at the
4645 // moment. After the final merge it should be added to
4646 // EmitCommonNeonBuiltinExpr.
4647 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
4648 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", 1, true);
4649 case NEON::BI__builtin_neon_vqshlu_n_v:
4650 case NEON::BI__builtin_neon_vqshluq_n_v:
4651 // FIXME: AArch64 and ARM use different intrinsics for this, but are
4652 // essentially compatible. It should be in EmitCommonNeonBuiltinExpr after
4653 // the final merge.
4654 Int = Intrinsic::aarch64_neon_sqshlu;
4655 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", 1, false);
4656 case NEON::BI__builtin_neon_vqshrun_n_v:
4657 // FIXME: as above
4658 Int = Intrinsic::aarch64_neon_sqshrun;
4659 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n");
4660 case NEON::BI__builtin_neon_vqrshrun_n_v:
4661 // FIXME: and again.
4662 Int = Intrinsic::aarch64_neon_sqrshrun;
4663 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n");
4664 case NEON::BI__builtin_neon_vqshrn_n_v:
4665 // FIXME: guess
4666 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
4667 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n");
4668 case NEON::BI__builtin_neon_vrshrn_n_v:
4669 // FIXME: there might be a pattern here.
4670 Int = Intrinsic::aarch64_neon_rshrn;
4671 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n");
4672 case NEON::BI__builtin_neon_vqrshrn_n_v:
4673 // FIXME: another one
4674 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
4675 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n");
4676 case NEON::BI__builtin_neon_vrnda_v:
4677 case NEON::BI__builtin_neon_vrndaq_v: {
4678 Int = Intrinsic::round;
4679 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda");
4680 }
4681 case NEON::BI__builtin_neon_vrndi_v:
4682 case NEON::BI__builtin_neon_vrndiq_v: {
4683 Int = Intrinsic::nearbyint;
4684 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi");
4685 }
4686 case NEON::BI__builtin_neon_vrndm_v:
4687 case NEON::BI__builtin_neon_vrndmq_v: {
4688 Int = Intrinsic::floor;
4689 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm");
4690 }
4691 case NEON::BI__builtin_neon_vrndn_v:
4692 case NEON::BI__builtin_neon_vrndnq_v: {
4693 Int = Intrinsic::aarch64_neon_frintn;
4694 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn");
4695 }
4696 case NEON::BI__builtin_neon_vrndp_v:
4697 case NEON::BI__builtin_neon_vrndpq_v: {
4698 Int = Intrinsic::ceil;
4699 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp");
4700 }
4701 case NEON::BI__builtin_neon_vrndx_v:
4702 case NEON::BI__builtin_neon_vrndxq_v: {
4703 Int = Intrinsic::rint;
4704 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx");
4705 }
4706 case NEON::BI__builtin_neon_vrnd_v:
4707 case NEON::BI__builtin_neon_vrndq_v: {
4708 Int = Intrinsic::trunc;
4709 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz");
4710 }
4711 case NEON::BI__builtin_neon_vceqz_v:
4712 case NEON::BI__builtin_neon_vceqzq_v:
4713 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ,
4714 ICmpInst::ICMP_EQ, "vceqz");
4715 case NEON::BI__builtin_neon_vcgez_v:
4716 case NEON::BI__builtin_neon_vcgezq_v:
4717 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE,
4718 ICmpInst::ICMP_SGE, "vcgez");
4719 case NEON::BI__builtin_neon_vclez_v:
4720 case NEON::BI__builtin_neon_vclezq_v:
4721 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE,
4722 ICmpInst::ICMP_SLE, "vclez");
4723 case NEON::BI__builtin_neon_vcgtz_v:
4724 case NEON::BI__builtin_neon_vcgtzq_v:
4725 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT,
4726 ICmpInst::ICMP_SGT, "vcgtz");
4727 case NEON::BI__builtin_neon_vcltz_v:
4728 case NEON::BI__builtin_neon_vcltzq_v:
4729 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT,
4730 ICmpInst::ICMP_SLT, "vcltz");
4731 case NEON::BI__builtin_neon_vcvt_f64_v:
4732 case NEON::BI__builtin_neon_vcvtq_f64_v:
4733 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4734 Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float64, false, quad));
4735 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
4736 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
4737 case NEON::BI__builtin_neon_vcvt_f64_f32: {
4738 assert(Type.getEltType() == NeonTypeFlags::Float64 && quad &&
4739 "unexpected vcvt_f64_f32 builtin");
4740 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float32, false, false);
4741 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
4742
4743 return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
4744 }
4745 case NEON::BI__builtin_neon_vcvt_f32_f64: {
4746 assert(Type.getEltType() == NeonTypeFlags::Float32 &&
4747 "unexpected vcvt_f32_f64 builtin");
4748 NeonTypeFlags SrcFlag = NeonTypeFlags(NeonTypeFlags::Float64, false, true);
4749 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
4750
4751 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
4752 }
4753 case NEON::BI__builtin_neon_vcvt_s32_v:
4754 case NEON::BI__builtin_neon_vcvt_u32_v:
4755 case NEON::BI__builtin_neon_vcvt_s64_v:
4756 case NEON::BI__builtin_neon_vcvt_u64_v:
4757 case NEON::BI__builtin_neon_vcvtq_s32_v:
4758 case NEON::BI__builtin_neon_vcvtq_u32_v:
4759 case NEON::BI__builtin_neon_vcvtq_s64_v:
4760 case NEON::BI__builtin_neon_vcvtq_u64_v: {
4761 bool Double =
4762 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
4763 llvm::Type *InTy =
4764 GetNeonType(this,
4765 NeonTypeFlags(Double ? NeonTypeFlags::Float64
4766 : NeonTypeFlags::Float32, false, quad));
4767 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
4768 if (usgn)
4769 return Builder.CreateFPToUI(Ops[0], Ty);
4770 return Builder.CreateFPToSI(Ops[0], Ty);
4771 }
4772 case NEON::BI__builtin_neon_vcvta_s32_v:
4773 case NEON::BI__builtin_neon_vcvtaq_s32_v:
4774 case NEON::BI__builtin_neon_vcvta_u32_v:
4775 case NEON::BI__builtin_neon_vcvtaq_u32_v:
4776 case NEON::BI__builtin_neon_vcvta_s64_v:
4777 case NEON::BI__builtin_neon_vcvtaq_s64_v:
4778 case NEON::BI__builtin_neon_vcvta_u64_v:
4779 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
4780 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
4781 bool Double =
4782 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
4783 llvm::Type *InTy =
4784 GetNeonType(this,
4785 NeonTypeFlags(Double ? NeonTypeFlags::Float64
4786 : NeonTypeFlags::Float32, false, quad));
4787 llvm::Type *Tys[2] = { Ty, InTy };
4788 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta");
4789 }
4790 case NEON::BI__builtin_neon_vcvtm_s32_v:
4791 case NEON::BI__builtin_neon_vcvtmq_s32_v:
4792 case NEON::BI__builtin_neon_vcvtm_u32_v:
4793 case NEON::BI__builtin_neon_vcvtmq_u32_v:
4794 case NEON::BI__builtin_neon_vcvtm_s64_v:
4795 case NEON::BI__builtin_neon_vcvtmq_s64_v:
4796 case NEON::BI__builtin_neon_vcvtm_u64_v:
4797 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
4798 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
4799 bool Double =
4800 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
4801 llvm::Type *InTy =
4802 GetNeonType(this,
4803 NeonTypeFlags(Double ? NeonTypeFlags::Float64
4804 : NeonTypeFlags::Float32, false, quad));
4805 llvm::Type *Tys[2] = { Ty, InTy };
4806 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm");
4807 }
4808 case NEON::BI__builtin_neon_vcvtn_s32_v:
4809 case NEON::BI__builtin_neon_vcvtnq_s32_v:
4810 case NEON::BI__builtin_neon_vcvtn_u32_v:
4811 case NEON::BI__builtin_neon_vcvtnq_u32_v:
4812 case NEON::BI__builtin_neon_vcvtn_s64_v:
4813 case NEON::BI__builtin_neon_vcvtnq_s64_v:
4814 case NEON::BI__builtin_neon_vcvtn_u64_v:
4815 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
4816 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
4817 bool Double =
4818 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
4819 llvm::Type *InTy =
4820 GetNeonType(this,
4821 NeonTypeFlags(Double ? NeonTypeFlags::Float64
4822 : NeonTypeFlags::Float32, false, quad));
4823 llvm::Type *Tys[2] = { Ty, InTy };
4824 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn");
4825 }
4826 case NEON::BI__builtin_neon_vcvtp_s32_v:
4827 case NEON::BI__builtin_neon_vcvtpq_s32_v:
4828 case NEON::BI__builtin_neon_vcvtp_u32_v:
4829 case NEON::BI__builtin_neon_vcvtpq_u32_v:
4830 case NEON::BI__builtin_neon_vcvtp_s64_v:
4831 case NEON::BI__builtin_neon_vcvtpq_s64_v:
4832 case NEON::BI__builtin_neon_vcvtp_u64_v:
4833 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
4834 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
4835 bool Double =
4836 (cast<llvm::IntegerType>(VTy->getElementType())->getBitWidth() == 64);
4837 llvm::Type *InTy =
4838 GetNeonType(this,
4839 NeonTypeFlags(Double ? NeonTypeFlags::Float64
4840 : NeonTypeFlags::Float32, false, quad));
4841 llvm::Type *Tys[2] = { Ty, InTy };
4842 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp");
4843 }
4844 case NEON::BI__builtin_neon_vmulx_v:
4845 case NEON::BI__builtin_neon_vmulxq_v: {
4846 Int = Intrinsic::aarch64_neon_fmulx;
4847 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx");
4848 }
4849 case NEON::BI__builtin_neon_vmul_lane_v:
4850 case NEON::BI__builtin_neon_vmul_laneq_v: {
4851 // v1f64 vmul_lane should be mapped to Neon scalar mul lane
4852 bool Quad = false;
4853 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
4854 Quad = true;
4855 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
4856 llvm::Type *VTy = GetNeonType(this,
4857 NeonTypeFlags(NeonTypeFlags::Float64, false, Quad));
4858 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
4859 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
4860 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
4861 return Builder.CreateBitCast(Result, Ty);
4862 }
4863 case NEON::BI__builtin_neon_vnegd_s64:
4864 return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd");
4865 case NEON::BI__builtin_neon_vpmaxnm_v:
4866 case NEON::BI__builtin_neon_vpmaxnmq_v: {
4867 Int = Intrinsic::aarch64_neon_fmaxnmp;
4868 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm");
4869 }
4870 case NEON::BI__builtin_neon_vpminnm_v:
4871 case NEON::BI__builtin_neon_vpminnmq_v: {
4872 Int = Intrinsic::aarch64_neon_fminnmp;
4873 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm");
4874 }
4875 case NEON::BI__builtin_neon_vsqrt_v:
4876 case NEON::BI__builtin_neon_vsqrtq_v: {
4877 Int = Intrinsic::sqrt;
4878 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
4879 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt");
4880 }
4881 case NEON::BI__builtin_neon_vrbit_v:
4882 case NEON::BI__builtin_neon_vrbitq_v: {
4883 Int = Intrinsic::aarch64_neon_rbit;
4884 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit");
4885 }
4886 case NEON::BI__builtin_neon_vaddv_u8:
4887 // FIXME: These are handled by the AArch64 scalar code.
4888 usgn = true;
4889 // FALLTHROUGH
4890 case NEON::BI__builtin_neon_vaddv_s8: {
4891 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
4892 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
4893 VTy =
4894 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
4895 llvm::Type *Tys[2] = { Ty, VTy };
4896 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4897 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
4898 return Builder.CreateTrunc(Ops[0],
4899 llvm::IntegerType::get(getLLVMContext(), 8));
4900 }
4901 case NEON::BI__builtin_neon_vaddv_u16:
4902 usgn = true;
4903 // FALLTHROUGH
4904 case NEON::BI__builtin_neon_vaddv_s16: {
4905 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
4906 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
4907 VTy =
4908 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
4909 llvm::Type *Tys[2] = { Ty, VTy };
4910 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4911 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
4912 return Builder.CreateTrunc(Ops[0],
4913 llvm::IntegerType::get(getLLVMContext(), 16));
4914 }
4915 case NEON::BI__builtin_neon_vaddvq_u8:
4916 usgn = true;
4917 // FALLTHROUGH
4918 case NEON::BI__builtin_neon_vaddvq_s8: {
4919 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
4920 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
4921 VTy =
4922 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
4923 llvm::Type *Tys[2] = { Ty, VTy };
4924 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4925 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
4926 return Builder.CreateTrunc(Ops[0],
4927 llvm::IntegerType::get(getLLVMContext(), 8));
4928 }
4929 case NEON::BI__builtin_neon_vaddvq_u16:
4930 usgn = true;
4931 // FALLTHROUGH
4932 case NEON::BI__builtin_neon_vaddvq_s16: {
4933 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
4934 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
4935 VTy =
4936 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
4937 llvm::Type *Tys[2] = { Ty, VTy };
4938 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4939 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv");
4940 return Builder.CreateTrunc(Ops[0],
4941 llvm::IntegerType::get(getLLVMContext(), 16));
4942 }
4943 case NEON::BI__builtin_neon_vmaxv_u8: {
4944 Int = Intrinsic::aarch64_neon_umaxv;
4945 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
4946 VTy =
4947 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
4948 llvm::Type *Tys[2] = { Ty, VTy };
4949 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4950 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
4951 return Builder.CreateTrunc(Ops[0],
4952 llvm::IntegerType::get(getLLVMContext(), 8));
4953 }
4954 case NEON::BI__builtin_neon_vmaxv_u16: {
4955 Int = Intrinsic::aarch64_neon_umaxv;
4956 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
4957 VTy =
4958 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
4959 llvm::Type *Tys[2] = { Ty, VTy };
4960 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4961 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
4962 return Builder.CreateTrunc(Ops[0],
4963 llvm::IntegerType::get(getLLVMContext(), 16));
4964 }
4965 case NEON::BI__builtin_neon_vmaxvq_u8: {
4966 Int = Intrinsic::aarch64_neon_umaxv;
4967 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
4968 VTy =
4969 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
4970 llvm::Type *Tys[2] = { Ty, VTy };
4971 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4972 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
4973 return Builder.CreateTrunc(Ops[0],
4974 llvm::IntegerType::get(getLLVMContext(), 8));
4975 }
4976 case NEON::BI__builtin_neon_vmaxvq_u16: {
4977 Int = Intrinsic::aarch64_neon_umaxv;
4978 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
4979 VTy =
4980 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
4981 llvm::Type *Tys[2] = { Ty, VTy };
4982 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4983 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
4984 return Builder.CreateTrunc(Ops[0],
4985 llvm::IntegerType::get(getLLVMContext(), 16));
4986 }
4987 case NEON::BI__builtin_neon_vmaxv_s8: {
4988 Int = Intrinsic::aarch64_neon_smaxv;
4989 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
4990 VTy =
4991 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
4992 llvm::Type *Tys[2] = { Ty, VTy };
4993 Ops.push_back(EmitScalarExpr(E->getArg(0)));
4994 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
4995 return Builder.CreateTrunc(Ops[0],
4996 llvm::IntegerType::get(getLLVMContext(), 8));
4997 }
4998 case NEON::BI__builtin_neon_vmaxv_s16: {
4999 Int = Intrinsic::aarch64_neon_smaxv;
5000 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5001 VTy =
5002 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
5003 llvm::Type *Tys[2] = { Ty, VTy };
5004 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5005 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
5006 return Builder.CreateTrunc(Ops[0],
5007 llvm::IntegerType::get(getLLVMContext(), 16));
5008 }
5009 case NEON::BI__builtin_neon_vmaxvq_s8: {
5010 Int = Intrinsic::aarch64_neon_smaxv;
5011 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5012 VTy =
5013 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
5014 llvm::Type *Tys[2] = { Ty, VTy };
5015 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5016 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
5017 return Builder.CreateTrunc(Ops[0],
5018 llvm::IntegerType::get(getLLVMContext(), 8));
5019 }
5020 case NEON::BI__builtin_neon_vmaxvq_s16: {
5021 Int = Intrinsic::aarch64_neon_smaxv;
5022 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5023 VTy =
5024 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
5025 llvm::Type *Tys[2] = { Ty, VTy };
5026 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5027 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv");
5028 return Builder.CreateTrunc(Ops[0],
5029 llvm::IntegerType::get(getLLVMContext(), 16));
5030 }
5031 case NEON::BI__builtin_neon_vminv_u8: {
5032 Int = Intrinsic::aarch64_neon_uminv;
5033 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5034 VTy =
5035 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
5036 llvm::Type *Tys[2] = { Ty, VTy };
5037 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5038 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5039 return Builder.CreateTrunc(Ops[0],
5040 llvm::IntegerType::get(getLLVMContext(), 8));
5041 }
5042 case NEON::BI__builtin_neon_vminv_u16: {
5043 Int = Intrinsic::aarch64_neon_uminv;
5044 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5045 VTy =
5046 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
5047 llvm::Type *Tys[2] = { Ty, VTy };
5048 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5049 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5050 return Builder.CreateTrunc(Ops[0],
5051 llvm::IntegerType::get(getLLVMContext(), 16));
5052 }
5053 case NEON::BI__builtin_neon_vminvq_u8: {
5054 Int = Intrinsic::aarch64_neon_uminv;
5055 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5056 VTy =
5057 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
5058 llvm::Type *Tys[2] = { Ty, VTy };
5059 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5060 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5061 return Builder.CreateTrunc(Ops[0],
5062 llvm::IntegerType::get(getLLVMContext(), 8));
5063 }
5064 case NEON::BI__builtin_neon_vminvq_u16: {
5065 Int = Intrinsic::aarch64_neon_uminv;
5066 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5067 VTy =
5068 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
5069 llvm::Type *Tys[2] = { Ty, VTy };
5070 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5071 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5072 return Builder.CreateTrunc(Ops[0],
5073 llvm::IntegerType::get(getLLVMContext(), 16));
5074 }
5075 case NEON::BI__builtin_neon_vminv_s8: {
5076 Int = Intrinsic::aarch64_neon_sminv;
5077 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5078 VTy =
5079 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
5080 llvm::Type *Tys[2] = { Ty, VTy };
5081 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5082 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5083 return Builder.CreateTrunc(Ops[0],
5084 llvm::IntegerType::get(getLLVMContext(), 8));
5085 }
5086 case NEON::BI__builtin_neon_vminv_s16: {
5087 Int = Intrinsic::aarch64_neon_sminv;
5088 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5089 VTy =
5090 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
5091 llvm::Type *Tys[2] = { Ty, VTy };
5092 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5093 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5094 return Builder.CreateTrunc(Ops[0],
5095 llvm::IntegerType::get(getLLVMContext(), 16));
5096 }
5097 case NEON::BI__builtin_neon_vminvq_s8: {
5098 Int = Intrinsic::aarch64_neon_sminv;
5099 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5100 VTy =
5101 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
5102 llvm::Type *Tys[2] = { Ty, VTy };
5103 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5104 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5105 return Builder.CreateTrunc(Ops[0],
5106 llvm::IntegerType::get(getLLVMContext(), 8));
5107 }
5108 case NEON::BI__builtin_neon_vminvq_s16: {
5109 Int = Intrinsic::aarch64_neon_sminv;
5110 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5111 VTy =
5112 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
5113 llvm::Type *Tys[2] = { Ty, VTy };
5114 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5115 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv");
5116 return Builder.CreateTrunc(Ops[0],
5117 llvm::IntegerType::get(getLLVMContext(), 16));
5118 }
5119 case NEON::BI__builtin_neon_vmul_n_f64: {
5120 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
5121 Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy);
5122 return Builder.CreateFMul(Ops[0], RHS);
5123 }
5124 case NEON::BI__builtin_neon_vaddlv_u8: {
5125 Int = Intrinsic::aarch64_neon_uaddlv;
5126 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5127 VTy =
5128 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
5129 llvm::Type *Tys[2] = { Ty, VTy };
5130 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5131 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5132 return Builder.CreateTrunc(Ops[0],
5133 llvm::IntegerType::get(getLLVMContext(), 16));
5134 }
5135 case NEON::BI__builtin_neon_vaddlv_u16: {
5136 Int = Intrinsic::aarch64_neon_uaddlv;
5137 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5138 VTy =
5139 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
5140 llvm::Type *Tys[2] = { Ty, VTy };
5141 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5142 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5143 }
5144 case NEON::BI__builtin_neon_vaddlvq_u8: {
5145 Int = Intrinsic::aarch64_neon_uaddlv;
5146 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5147 VTy =
5148 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
5149 llvm::Type *Tys[2] = { Ty, VTy };
5150 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5151 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5152 return Builder.CreateTrunc(Ops[0],
5153 llvm::IntegerType::get(getLLVMContext(), 16));
5154 }
5155 case NEON::BI__builtin_neon_vaddlvq_u16: {
5156 Int = Intrinsic::aarch64_neon_uaddlv;
5157 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5158 VTy =
5159 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
5160 llvm::Type *Tys[2] = { Ty, VTy };
5161 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5162 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5163 }
5164 case NEON::BI__builtin_neon_vaddlv_s8: {
5165 Int = Intrinsic::aarch64_neon_saddlv;
5166 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5167 VTy =
5168 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 8);
5169 llvm::Type *Tys[2] = { Ty, VTy };
5170 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5171 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5172 return Builder.CreateTrunc(Ops[0],
5173 llvm::IntegerType::get(getLLVMContext(), 16));
5174 }
5175 case NEON::BI__builtin_neon_vaddlv_s16: {
5176 Int = Intrinsic::aarch64_neon_saddlv;
5177 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5178 VTy =
5179 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 4);
5180 llvm::Type *Tys[2] = { Ty, VTy };
5181 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5182 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5183 }
5184 case NEON::BI__builtin_neon_vaddlvq_s8: {
5185 Int = Intrinsic::aarch64_neon_saddlv;
5186 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5187 VTy =
5188 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 8), 16);
5189 llvm::Type *Tys[2] = { Ty, VTy };
5190 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5191 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5192 return Builder.CreateTrunc(Ops[0],
5193 llvm::IntegerType::get(getLLVMContext(), 16));
5194 }
5195 case NEON::BI__builtin_neon_vaddlvq_s16: {
5196 Int = Intrinsic::aarch64_neon_saddlv;
5197 Ty = llvm::IntegerType::get(getLLVMContext(), 32);
5198 VTy =
5199 llvm::VectorType::get(llvm::IntegerType::get(getLLVMContext(), 16), 8);
5200 llvm::Type *Tys[2] = { Ty, VTy };
5201 Ops.push_back(EmitScalarExpr(E->getArg(0)));
5202 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv");
5203 }
5204 case NEON::BI__builtin_neon_vsri_n_v:
5205 case NEON::BI__builtin_neon_vsriq_n_v: {
5206 Int = Intrinsic::aarch64_neon_vsri;
5207 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
5208 return EmitNeonCall(Intrin, Ops, "vsri_n");
5209 }
5210 case NEON::BI__builtin_neon_vsli_n_v:
5211 case NEON::BI__builtin_neon_vsliq_n_v: {
5212 Int = Intrinsic::aarch64_neon_vsli;
5213 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
5214 return EmitNeonCall(Intrin, Ops, "vsli_n");
5215 }
5216 case NEON::BI__builtin_neon_vsra_n_v:
5217 case NEON::BI__builtin_neon_vsraq_n_v:
5218 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5219 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n");
5220 return Builder.CreateAdd(Ops[0], Ops[1]);
5221 case NEON::BI__builtin_neon_vrsra_n_v:
5222 case NEON::BI__builtin_neon_vrsraq_n_v: {
5223 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
5224 SmallVector<llvm::Value*,2> TmpOps;
5225 TmpOps.push_back(Ops[1]);
5226 TmpOps.push_back(Ops[2]);
5227 Function* F = CGM.getIntrinsic(Int, Ty);
5228 llvm::Value *tmp = EmitNeonCall(F, TmpOps, "vrshr_n", 1, true);
5229 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
5230 return Builder.CreateAdd(Ops[0], tmp);
5231 }
5232 // FIXME: Sharing loads & stores with 32-bit is complicated by the absence
5233 // of an Align parameter here.
5234 case NEON::BI__builtin_neon_vld1_x2_v:
5235 case NEON::BI__builtin_neon_vld1q_x2_v:
5236 case NEON::BI__builtin_neon_vld1_x3_v:
5237 case NEON::BI__builtin_neon_vld1q_x3_v:
5238 case NEON::BI__builtin_neon_vld1_x4_v:
5239 case NEON::BI__builtin_neon_vld1q_x4_v: {
5240 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
5241 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5242 llvm::Type *Tys[2] = { VTy, PTy };
5243 unsigned Int;
5244 switch (BuiltinID) {
5245 case NEON::BI__builtin_neon_vld1_x2_v:
5246 case NEON::BI__builtin_neon_vld1q_x2_v:
5247 Int = Intrinsic::aarch64_neon_ld1x2;
5248 break;
5249 case NEON::BI__builtin_neon_vld1_x3_v:
5250 case NEON::BI__builtin_neon_vld1q_x3_v:
5251 Int = Intrinsic::aarch64_neon_ld1x3;
5252 break;
5253 case NEON::BI__builtin_neon_vld1_x4_v:
5254 case NEON::BI__builtin_neon_vld1q_x4_v:
5255 Int = Intrinsic::aarch64_neon_ld1x4;
5256 break;
5257 }
5258 Function *F = CGM.getIntrinsic(Int, Tys);
5259 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN");
5260 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5261 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5262 return Builder.CreateStore(Ops[1], Ops[0]);
5263 }
5264 case NEON::BI__builtin_neon_vst1_x2_v:
5265 case NEON::BI__builtin_neon_vst1q_x2_v:
5266 case NEON::BI__builtin_neon_vst1_x3_v:
5267 case NEON::BI__builtin_neon_vst1q_x3_v:
5268 case NEON::BI__builtin_neon_vst1_x4_v:
5269 case NEON::BI__builtin_neon_vst1q_x4_v: {
5270 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
5271 llvm::Type *Tys[2] = { VTy, PTy };
5272 unsigned Int;
5273 switch (BuiltinID) {
5274 case NEON::BI__builtin_neon_vst1_x2_v:
5275 case NEON::BI__builtin_neon_vst1q_x2_v:
5276 Int = Intrinsic::aarch64_neon_st1x2;
5277 break;
5278 case NEON::BI__builtin_neon_vst1_x3_v:
5279 case NEON::BI__builtin_neon_vst1q_x3_v:
5280 Int = Intrinsic::aarch64_neon_st1x3;
5281 break;
5282 case NEON::BI__builtin_neon_vst1_x4_v:
5283 case NEON::BI__builtin_neon_vst1q_x4_v:
5284 Int = Intrinsic::aarch64_neon_st1x4;
5285 break;
5286 }
5287 SmallVector<Value *, 4> IntOps(Ops.begin()+1, Ops.end());
5288 IntOps.push_back(Ops[0]);
5289 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), IntOps, "");
5290 }
5291 case NEON::BI__builtin_neon_vld1_v:
5292 case NEON::BI__builtin_neon_vld1q_v:
5293 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
5294 return Builder.CreateLoad(Ops[0]);
5295 case NEON::BI__builtin_neon_vst1_v:
5296 case NEON::BI__builtin_neon_vst1q_v:
5297 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
5298 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
5299 return Builder.CreateStore(Ops[1], Ops[0]);
5300 case NEON::BI__builtin_neon_vld1_lane_v:
5301 case NEON::BI__builtin_neon_vld1q_lane_v:
5302 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5303 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
5304 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5305 Ops[0] = Builder.CreateLoad(Ops[0]);
5306 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
5307 case NEON::BI__builtin_neon_vld1_dup_v:
5308 case NEON::BI__builtin_neon_vld1q_dup_v: {
5309 Value *V = UndefValue::get(Ty);
5310 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
5311 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5312 Ops[0] = Builder.CreateLoad(Ops[0]);
5313 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
5314 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
5315 return EmitNeonSplat(Ops[0], CI);
5316 }
5317 case NEON::BI__builtin_neon_vst1_lane_v:
5318 case NEON::BI__builtin_neon_vst1q_lane_v:
5319 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5320 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
5321 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5322 return Builder.CreateStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty));
5323 case NEON::BI__builtin_neon_vld2_v:
5324 case NEON::BI__builtin_neon_vld2q_v: {
5325 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
5326 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5327 llvm::Type *Tys[2] = { VTy, PTy };
5328 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
5329 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
5330 Ops[0] = Builder.CreateBitCast(Ops[0],
5331 llvm::PointerType::getUnqual(Ops[1]->getType()));
5332 return Builder.CreateStore(Ops[1], Ops[0]);
5333 }
5334 case NEON::BI__builtin_neon_vld3_v:
5335 case NEON::BI__builtin_neon_vld3q_v: {
5336 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
5337 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5338 llvm::Type *Tys[2] = { VTy, PTy };
5339 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
5340 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
5341 Ops[0] = Builder.CreateBitCast(Ops[0],
5342 llvm::PointerType::getUnqual(Ops[1]->getType()));
5343 return Builder.CreateStore(Ops[1], Ops[0]);
5344 }
5345 case NEON::BI__builtin_neon_vld4_v:
5346 case NEON::BI__builtin_neon_vld4q_v: {
5347 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
5348 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5349 llvm::Type *Tys[2] = { VTy, PTy };
5350 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
5351 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
5352 Ops[0] = Builder.CreateBitCast(Ops[0],
5353 llvm::PointerType::getUnqual(Ops[1]->getType()));
5354 return Builder.CreateStore(Ops[1], Ops[0]);
5355 }
5356 case NEON::BI__builtin_neon_vld2_dup_v:
5357 case NEON::BI__builtin_neon_vld2q_dup_v: {
5358 llvm::Type *PTy =
5359 llvm::PointerType::getUnqual(VTy->getElementType());
5360 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5361 llvm::Type *Tys[2] = { VTy, PTy };
5362 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
5363 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
5364 Ops[0] = Builder.CreateBitCast(Ops[0],
5365 llvm::PointerType::getUnqual(Ops[1]->getType()));
5366 return Builder.CreateStore(Ops[1], Ops[0]);
5367 }
5368 case NEON::BI__builtin_neon_vld3_dup_v:
5369 case NEON::BI__builtin_neon_vld3q_dup_v: {
5370 llvm::Type *PTy =
5371 llvm::PointerType::getUnqual(VTy->getElementType());
5372 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5373 llvm::Type *Tys[2] = { VTy, PTy };
5374 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
5375 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
5376 Ops[0] = Builder.CreateBitCast(Ops[0],
5377 llvm::PointerType::getUnqual(Ops[1]->getType()));
5378 return Builder.CreateStore(Ops[1], Ops[0]);
5379 }
5380 case NEON::BI__builtin_neon_vld4_dup_v:
5381 case NEON::BI__builtin_neon_vld4q_dup_v: {
5382 llvm::Type *PTy =
5383 llvm::PointerType::getUnqual(VTy->getElementType());
5384 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5385 llvm::Type *Tys[2] = { VTy, PTy };
5386 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
5387 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
5388 Ops[0] = Builder.CreateBitCast(Ops[0],
5389 llvm::PointerType::getUnqual(Ops[1]->getType()));
5390 return Builder.CreateStore(Ops[1], Ops[0]);
5391 }
5392 case NEON::BI__builtin_neon_vld2_lane_v:
5393 case NEON::BI__builtin_neon_vld2q_lane_v: {
5394 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
5395 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
5396 Ops.push_back(Ops[1]);
5397 Ops.erase(Ops.begin()+1);
5398 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5399 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5400 Ops[3] = Builder.CreateZExt(Ops[3],
5401 llvm::IntegerType::get(getLLVMContext(), 64));
5402 Ops[1] = Builder.CreateCall(F,
5403 ArrayRef<Value*>(Ops).slice(1), "vld2_lane");
5404 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5405 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5406 return Builder.CreateStore(Ops[1], Ops[0]);
5407 }
5408 case NEON::BI__builtin_neon_vld3_lane_v:
5409 case NEON::BI__builtin_neon_vld3q_lane_v: {
5410 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
5411 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
5412 Ops.push_back(Ops[1]);
5413 Ops.erase(Ops.begin()+1);
5414 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5415 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5416 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
5417 Ops[4] = Builder.CreateZExt(Ops[4],
5418 llvm::IntegerType::get(getLLVMContext(), 64));
5419 Ops[1] = Builder.CreateCall(F,
5420 ArrayRef<Value*>(Ops).slice(1), "vld3_lane");
5421 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5422 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5423 return Builder.CreateStore(Ops[1], Ops[0]);
5424 }
5425 case NEON::BI__builtin_neon_vld4_lane_v:
5426 case NEON::BI__builtin_neon_vld4q_lane_v: {
5427 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
5428 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
5429 Ops.push_back(Ops[1]);
5430 Ops.erase(Ops.begin()+1);
5431 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5432 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5433 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
5434 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
5435 Ops[5] = Builder.CreateZExt(Ops[5],
5436 llvm::IntegerType::get(getLLVMContext(), 64));
5437 Ops[1] = Builder.CreateCall(F,
5438 ArrayRef<Value*>(Ops).slice(1), "vld4_lane");
5439 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5440 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5441 return Builder.CreateStore(Ops[1], Ops[0]);
5442 }
5443 case NEON::BI__builtin_neon_vst2_v:
5444 case NEON::BI__builtin_neon_vst2q_v: {
5445 Ops.push_back(Ops[0]);
5446 Ops.erase(Ops.begin());
5447 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
5448 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
5449 Ops, "");
5450 }
5451 case NEON::BI__builtin_neon_vst2_lane_v:
5452 case NEON::BI__builtin_neon_vst2q_lane_v: {
5453 Ops.push_back(Ops[0]);
5454 Ops.erase(Ops.begin());
5455 Ops[2] = Builder.CreateZExt(Ops[2],
5456 llvm::IntegerType::get(getLLVMContext(), 64));
5457 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
5458 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
5459 Ops, "");
5460 }
5461 case NEON::BI__builtin_neon_vst3_v:
5462 case NEON::BI__builtin_neon_vst3q_v: {
5463 Ops.push_back(Ops[0]);
5464 Ops.erase(Ops.begin());
5465 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
5466 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
5467 Ops, "");
5468 }
5469 case NEON::BI__builtin_neon_vst3_lane_v:
5470 case NEON::BI__builtin_neon_vst3q_lane_v: {
5471 Ops.push_back(Ops[0]);
5472 Ops.erase(Ops.begin());
5473 Ops[3] = Builder.CreateZExt(Ops[3],
5474 llvm::IntegerType::get(getLLVMContext(), 64));
5475 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
5476 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
5477 Ops, "");
5478 }
5479 case NEON::BI__builtin_neon_vst4_v:
5480 case NEON::BI__builtin_neon_vst4q_v: {
5481 Ops.push_back(Ops[0]);
5482 Ops.erase(Ops.begin());
5483 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
5484 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
5485 Ops, "");
5486 }
5487 case NEON::BI__builtin_neon_vst4_lane_v:
5488 case NEON::BI__builtin_neon_vst4q_lane_v: {
5489 Ops.push_back(Ops[0]);
5490 Ops.erase(Ops.begin());
5491 Ops[4] = Builder.CreateZExt(Ops[4],
5492 llvm::IntegerType::get(getLLVMContext(), 64));
5493 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
5494 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
5495 Ops, "");
5496 }
5497 case NEON::BI__builtin_neon_vtrn_v:
5498 case NEON::BI__builtin_neon_vtrnq_v: {
5499 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5500 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5501 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5502 Value *SV = nullptr;
5503
5504 for (unsigned vi = 0; vi != 2; ++vi) {
5505 SmallVector<Constant*, 16> Indices;
5506 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
5507 Indices.push_back(ConstantInt::get(Int32Ty, i+vi));
5508 Indices.push_back(ConstantInt::get(Int32Ty, i+e+vi));
5509 }
5510 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
5511 SV = llvm::ConstantVector::get(Indices);
5512 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn");
5513 SV = Builder.CreateStore(SV, Addr);
5514 }
5515 return SV;
5516 }
5517 case NEON::BI__builtin_neon_vuzp_v:
5518 case NEON::BI__builtin_neon_vuzpq_v: {
5519 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5520 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5521 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5522 Value *SV = nullptr;
5523
5524 for (unsigned vi = 0; vi != 2; ++vi) {
5525 SmallVector<Constant*, 16> Indices;
5526 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
5527 Indices.push_back(ConstantInt::get(Int32Ty, 2*i+vi));
5528
5529 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
5530 SV = llvm::ConstantVector::get(Indices);
5531 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp");
5532 SV = Builder.CreateStore(SV, Addr);
5533 }
5534 return SV;
5535 }
5536 case NEON::BI__builtin_neon_vzip_v:
5537 case NEON::BI__builtin_neon_vzipq_v: {
5538 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5539 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5540 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5541 Value *SV = nullptr;
5542
5543 for (unsigned vi = 0; vi != 2; ++vi) {
5544 SmallVector<Constant*, 16> Indices;
5545 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
5546 Indices.push_back(ConstantInt::get(Int32Ty, (i + vi*e) >> 1));
5547 Indices.push_back(ConstantInt::get(Int32Ty, ((i + vi*e) >> 1)+e));
5548 }
5549 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ops[0], vi);
5550 SV = llvm::ConstantVector::get(Indices);
5551 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip");
5552 SV = Builder.CreateStore(SV, Addr);
5553 }
5554 return SV;
5555 }
5556 case NEON::BI__builtin_neon_vqtbl1q_v: {
5557 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
5558 Ops, "vtbl1");
5559 }
5560 case NEON::BI__builtin_neon_vqtbl2q_v: {
5561 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
5562 Ops, "vtbl2");
5563 }
5564 case NEON::BI__builtin_neon_vqtbl3q_v: {
5565 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
5566 Ops, "vtbl3");
5567 }
5568 case NEON::BI__builtin_neon_vqtbl4q_v: {
5569 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
5570 Ops, "vtbl4");
5571 }
5572 case NEON::BI__builtin_neon_vqtbx1q_v: {
5573 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
5574 Ops, "vtbx1");
5575 }
5576 case NEON::BI__builtin_neon_vqtbx2q_v: {
5577 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
5578 Ops, "vtbx2");
5579 }
5580 case NEON::BI__builtin_neon_vqtbx3q_v: {
5581 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
5582 Ops, "vtbx3");
5583 }
5584 case NEON::BI__builtin_neon_vqtbx4q_v: {
5585 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
5586 Ops, "vtbx4");
5587 }
5588 case NEON::BI__builtin_neon_vsqadd_v:
5589 case NEON::BI__builtin_neon_vsqaddq_v: {
5590 Int = Intrinsic::aarch64_neon_usqadd;
5591 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd");
5592 }
5593 case NEON::BI__builtin_neon_vuqadd_v:
5594 case NEON::BI__builtin_neon_vuqaddq_v: {
5595 Int = Intrinsic::aarch64_neon_suqadd;
5596 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd");
5597 }
5598 }
5599 }
5600
5601 llvm::Value *CodeGenFunction::
BuildVector(ArrayRef<llvm::Value * > Ops)5602 BuildVector(ArrayRef<llvm::Value*> Ops) {
5603 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
5604 "Not a power-of-two sized vector!");
5605 bool AllConstants = true;
5606 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
5607 AllConstants &= isa<Constant>(Ops[i]);
5608
5609 // If this is a constant vector, create a ConstantVector.
5610 if (AllConstants) {
5611 SmallVector<llvm::Constant*, 16> CstOps;
5612 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
5613 CstOps.push_back(cast<Constant>(Ops[i]));
5614 return llvm::ConstantVector::get(CstOps);
5615 }
5616
5617 // Otherwise, insertelement the values to build the vector.
5618 Value *Result =
5619 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
5620
5621 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
5622 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i));
5623
5624 return Result;
5625 }
5626
EmitX86BuiltinExpr(unsigned BuiltinID,const CallExpr * E)5627 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
5628 const CallExpr *E) {
5629 SmallVector<Value*, 4> Ops;
5630
5631 // Find out if any arguments are required to be integer constant expressions.
5632 unsigned ICEArguments = 0;
5633 ASTContext::GetBuiltinTypeError Error;
5634 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
5635 assert(Error == ASTContext::GE_None && "Should not codegen an error");
5636
5637 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
5638 // If this is a normal argument, just emit it as a scalar.
5639 if ((ICEArguments & (1 << i)) == 0) {
5640 Ops.push_back(EmitScalarExpr(E->getArg(i)));
5641 continue;
5642 }
5643
5644 // If this is required to be a constant, constant fold it so that we know
5645 // that the generated intrinsic gets a ConstantInt.
5646 llvm::APSInt Result;
5647 bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result, getContext());
5648 assert(IsConst && "Constant arg isn't actually constant?"); (void)IsConst;
5649 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
5650 }
5651
5652 switch (BuiltinID) {
5653 default: return nullptr;
5654 case X86::BI_mm_prefetch: {
5655 Value *Address = EmitScalarExpr(E->getArg(0));
5656 Value *RW = ConstantInt::get(Int32Ty, 0);
5657 Value *Locality = EmitScalarExpr(E->getArg(1));
5658 Value *Data = ConstantInt::get(Int32Ty, 1);
5659 Value *F = CGM.getIntrinsic(Intrinsic::prefetch);
5660 return Builder.CreateCall4(F, Address, RW, Locality, Data);
5661 }
5662 case X86::BI__builtin_ia32_vec_init_v8qi:
5663 case X86::BI__builtin_ia32_vec_init_v4hi:
5664 case X86::BI__builtin_ia32_vec_init_v2si:
5665 return Builder.CreateBitCast(BuildVector(Ops),
5666 llvm::Type::getX86_MMXTy(getLLVMContext()));
5667 case X86::BI__builtin_ia32_vec_ext_v2si:
5668 return Builder.CreateExtractElement(Ops[0],
5669 llvm::ConstantInt::get(Ops[1]->getType(), 0));
5670 case X86::BI__builtin_ia32_ldmxcsr: {
5671 Value *Tmp = CreateMemTemp(E->getArg(0)->getType());
5672 Builder.CreateStore(Ops[0], Tmp);
5673 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
5674 Builder.CreateBitCast(Tmp, Int8PtrTy));
5675 }
5676 case X86::BI__builtin_ia32_stmxcsr: {
5677 Value *Tmp = CreateMemTemp(E->getType());
5678 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
5679 Builder.CreateBitCast(Tmp, Int8PtrTy));
5680 return Builder.CreateLoad(Tmp, "stmxcsr");
5681 }
5682 case X86::BI__builtin_ia32_storehps:
5683 case X86::BI__builtin_ia32_storelps: {
5684 llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty);
5685 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
5686
5687 // cast val v2i64
5688 Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast");
5689
5690 // extract (0, 1)
5691 unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1;
5692 llvm::Value *Idx = llvm::ConstantInt::get(SizeTy, Index);
5693 Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract");
5694
5695 // cast pointer to i64 & store
5696 Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy);
5697 return Builder.CreateStore(Ops[1], Ops[0]);
5698 }
5699 case X86::BI__builtin_ia32_palignr: {
5700 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
5701
5702 // If palignr is shifting the pair of input vectors less than 9 bytes,
5703 // emit a shuffle instruction.
5704 if (shiftVal <= 8) {
5705 SmallVector<llvm::Constant*, 8> Indices;
5706 for (unsigned i = 0; i != 8; ++i)
5707 Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
5708
5709 Value* SV = llvm::ConstantVector::get(Indices);
5710 return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
5711 }
5712
5713 // If palignr is shifting the pair of input vectors more than 8 but less
5714 // than 16 bytes, emit a logical right shift of the destination.
5715 if (shiftVal < 16) {
5716 // MMX has these as 1 x i64 vectors for some odd optimization reasons.
5717 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1);
5718
5719 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
5720 Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8);
5721
5722 // create i32 constant
5723 llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_mmx_psrl_q);
5724 return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
5725 }
5726
5727 // If palignr is shifting the pair of vectors more than 16 bytes, emit zero.
5728 return llvm::Constant::getNullValue(ConvertType(E->getType()));
5729 }
5730 case X86::BI__builtin_ia32_palignr128: {
5731 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
5732
5733 // If palignr is shifting the pair of input vectors less than 17 bytes,
5734 // emit a shuffle instruction.
5735 if (shiftVal <= 16) {
5736 SmallVector<llvm::Constant*, 16> Indices;
5737 for (unsigned i = 0; i != 16; ++i)
5738 Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
5739
5740 Value* SV = llvm::ConstantVector::get(Indices);
5741 return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
5742 }
5743
5744 // If palignr is shifting the pair of input vectors more than 16 but less
5745 // than 32 bytes, emit a logical right shift of the destination.
5746 if (shiftVal < 32) {
5747 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
5748
5749 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
5750 Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8);
5751
5752 // create i32 constant
5753 llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_sse2_psrl_dq);
5754 return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
5755 }
5756
5757 // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
5758 return llvm::Constant::getNullValue(ConvertType(E->getType()));
5759 }
5760 case X86::BI__builtin_ia32_palignr256: {
5761 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
5762
5763 // If palignr is shifting the pair of input vectors less than 17 bytes,
5764 // emit a shuffle instruction.
5765 if (shiftVal <= 16) {
5766 SmallVector<llvm::Constant*, 32> Indices;
5767 // 256-bit palignr operates on 128-bit lanes so we need to handle that
5768 for (unsigned l = 0; l != 2; ++l) {
5769 unsigned LaneStart = l * 16;
5770 unsigned LaneEnd = (l+1) * 16;
5771 for (unsigned i = 0; i != 16; ++i) {
5772 unsigned Idx = shiftVal + i + LaneStart;
5773 if (Idx >= LaneEnd) Idx += 16; // end of lane, switch operand
5774 Indices.push_back(llvm::ConstantInt::get(Int32Ty, Idx));
5775 }
5776 }
5777
5778 Value* SV = llvm::ConstantVector::get(Indices);
5779 return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
5780 }
5781
5782 // If palignr is shifting the pair of input vectors more than 16 but less
5783 // than 32 bytes, emit a logical right shift of the destination.
5784 if (shiftVal < 32) {
5785 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 4);
5786
5787 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
5788 Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8);
5789
5790 // create i32 constant
5791 llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_avx2_psrl_dq);
5792 return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
5793 }
5794
5795 // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
5796 return llvm::Constant::getNullValue(ConvertType(E->getType()));
5797 }
5798 case X86::BI__builtin_ia32_movntps:
5799 case X86::BI__builtin_ia32_movntps256:
5800 case X86::BI__builtin_ia32_movntpd:
5801 case X86::BI__builtin_ia32_movntpd256:
5802 case X86::BI__builtin_ia32_movntdq:
5803 case X86::BI__builtin_ia32_movntdq256:
5804 case X86::BI__builtin_ia32_movnti:
5805 case X86::BI__builtin_ia32_movnti64: {
5806 llvm::MDNode *Node = llvm::MDNode::get(getLLVMContext(),
5807 Builder.getInt32(1));
5808
5809 // Convert the type of the pointer to a pointer to the stored type.
5810 Value *BC = Builder.CreateBitCast(Ops[0],
5811 llvm::PointerType::getUnqual(Ops[1]->getType()),
5812 "cast");
5813 StoreInst *SI = Builder.CreateStore(Ops[1], BC);
5814 SI->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
5815
5816 // If the operand is an integer, we can't assume alignment. Otherwise,
5817 // assume natural alignment.
5818 QualType ArgTy = E->getArg(1)->getType();
5819 unsigned Align;
5820 if (ArgTy->isIntegerType())
5821 Align = 1;
5822 else
5823 Align = getContext().getTypeSizeInChars(ArgTy).getQuantity();
5824 SI->setAlignment(Align);
5825 return SI;
5826 }
5827 // 3DNow!
5828 case X86::BI__builtin_ia32_pswapdsf:
5829 case X86::BI__builtin_ia32_pswapdsi: {
5830 const char *name = nullptr;
5831 Intrinsic::ID ID = Intrinsic::not_intrinsic;
5832 switch(BuiltinID) {
5833 default: llvm_unreachable("Unsupported intrinsic!");
5834 case X86::BI__builtin_ia32_pswapdsf:
5835 case X86::BI__builtin_ia32_pswapdsi:
5836 name = "pswapd";
5837 ID = Intrinsic::x86_3dnowa_pswapd;
5838 break;
5839 }
5840 llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext());
5841 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast");
5842 llvm::Function *F = CGM.getIntrinsic(ID);
5843 return Builder.CreateCall(F, Ops, name);
5844 }
5845 case X86::BI__builtin_ia32_rdrand16_step:
5846 case X86::BI__builtin_ia32_rdrand32_step:
5847 case X86::BI__builtin_ia32_rdrand64_step:
5848 case X86::BI__builtin_ia32_rdseed16_step:
5849 case X86::BI__builtin_ia32_rdseed32_step:
5850 case X86::BI__builtin_ia32_rdseed64_step: {
5851 Intrinsic::ID ID;
5852 switch (BuiltinID) {
5853 default: llvm_unreachable("Unsupported intrinsic!");
5854 case X86::BI__builtin_ia32_rdrand16_step:
5855 ID = Intrinsic::x86_rdrand_16;
5856 break;
5857 case X86::BI__builtin_ia32_rdrand32_step:
5858 ID = Intrinsic::x86_rdrand_32;
5859 break;
5860 case X86::BI__builtin_ia32_rdrand64_step:
5861 ID = Intrinsic::x86_rdrand_64;
5862 break;
5863 case X86::BI__builtin_ia32_rdseed16_step:
5864 ID = Intrinsic::x86_rdseed_16;
5865 break;
5866 case X86::BI__builtin_ia32_rdseed32_step:
5867 ID = Intrinsic::x86_rdseed_32;
5868 break;
5869 case X86::BI__builtin_ia32_rdseed64_step:
5870 ID = Intrinsic::x86_rdseed_64;
5871 break;
5872 }
5873
5874 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID));
5875 Builder.CreateStore(Builder.CreateExtractValue(Call, 0), Ops[0]);
5876 return Builder.CreateExtractValue(Call, 1);
5877 }
5878 // AVX2 broadcast
5879 case X86::BI__builtin_ia32_vbroadcastsi256: {
5880 Value *VecTmp = CreateMemTemp(E->getArg(0)->getType());
5881 Builder.CreateStore(Ops[0], VecTmp);
5882 Value *F = CGM.getIntrinsic(Intrinsic::x86_avx2_vbroadcasti128);
5883 return Builder.CreateCall(F, Builder.CreateBitCast(VecTmp, Int8PtrTy));
5884 }
5885 }
5886 }
5887
5888
EmitPPCBuiltinExpr(unsigned BuiltinID,const CallExpr * E)5889 Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
5890 const CallExpr *E) {
5891 SmallVector<Value*, 4> Ops;
5892
5893 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
5894 Ops.push_back(EmitScalarExpr(E->getArg(i)));
5895
5896 Intrinsic::ID ID = Intrinsic::not_intrinsic;
5897
5898 switch (BuiltinID) {
5899 default: return nullptr;
5900
5901 // vec_ld, vec_lvsl, vec_lvsr
5902 case PPC::BI__builtin_altivec_lvx:
5903 case PPC::BI__builtin_altivec_lvxl:
5904 case PPC::BI__builtin_altivec_lvebx:
5905 case PPC::BI__builtin_altivec_lvehx:
5906 case PPC::BI__builtin_altivec_lvewx:
5907 case PPC::BI__builtin_altivec_lvsl:
5908 case PPC::BI__builtin_altivec_lvsr:
5909 {
5910 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
5911
5912 Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]);
5913 Ops.pop_back();
5914
5915 switch (BuiltinID) {
5916 default: llvm_unreachable("Unsupported ld/lvsl/lvsr intrinsic!");
5917 case PPC::BI__builtin_altivec_lvx:
5918 ID = Intrinsic::ppc_altivec_lvx;
5919 break;
5920 case PPC::BI__builtin_altivec_lvxl:
5921 ID = Intrinsic::ppc_altivec_lvxl;
5922 break;
5923 case PPC::BI__builtin_altivec_lvebx:
5924 ID = Intrinsic::ppc_altivec_lvebx;
5925 break;
5926 case PPC::BI__builtin_altivec_lvehx:
5927 ID = Intrinsic::ppc_altivec_lvehx;
5928 break;
5929 case PPC::BI__builtin_altivec_lvewx:
5930 ID = Intrinsic::ppc_altivec_lvewx;
5931 break;
5932 case PPC::BI__builtin_altivec_lvsl:
5933 ID = Intrinsic::ppc_altivec_lvsl;
5934 break;
5935 case PPC::BI__builtin_altivec_lvsr:
5936 ID = Intrinsic::ppc_altivec_lvsr;
5937 break;
5938 }
5939 llvm::Function *F = CGM.getIntrinsic(ID);
5940 return Builder.CreateCall(F, Ops, "");
5941 }
5942
5943 // vec_st
5944 case PPC::BI__builtin_altivec_stvx:
5945 case PPC::BI__builtin_altivec_stvxl:
5946 case PPC::BI__builtin_altivec_stvebx:
5947 case PPC::BI__builtin_altivec_stvehx:
5948 case PPC::BI__builtin_altivec_stvewx:
5949 {
5950 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy);
5951 Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]);
5952 Ops.pop_back();
5953
5954 switch (BuiltinID) {
5955 default: llvm_unreachable("Unsupported st intrinsic!");
5956 case PPC::BI__builtin_altivec_stvx:
5957 ID = Intrinsic::ppc_altivec_stvx;
5958 break;
5959 case PPC::BI__builtin_altivec_stvxl:
5960 ID = Intrinsic::ppc_altivec_stvxl;
5961 break;
5962 case PPC::BI__builtin_altivec_stvebx:
5963 ID = Intrinsic::ppc_altivec_stvebx;
5964 break;
5965 case PPC::BI__builtin_altivec_stvehx:
5966 ID = Intrinsic::ppc_altivec_stvehx;
5967 break;
5968 case PPC::BI__builtin_altivec_stvewx:
5969 ID = Intrinsic::ppc_altivec_stvewx;
5970 break;
5971 }
5972 llvm::Function *F = CGM.getIntrinsic(ID);
5973 return Builder.CreateCall(F, Ops, "");
5974 }
5975 }
5976 }
5977
EmitR600BuiltinExpr(unsigned BuiltinID,const CallExpr * E)5978 Value *CodeGenFunction::EmitR600BuiltinExpr(unsigned BuiltinID,
5979 const CallExpr *E) {
5980 switch (BuiltinID) {
5981 case R600::BI__builtin_amdgpu_div_scale:
5982 case R600::BI__builtin_amdgpu_div_scalef: {
5983 // Translate from the intrinsics's struct return to the builtin's out
5984 // argument.
5985
5986 std::pair<llvm::Value *, unsigned> FlagOutPtr
5987 = EmitPointerWithAlignment(E->getArg(3));
5988
5989 llvm::Value *X = EmitScalarExpr(E->getArg(0));
5990 llvm::Value *Y = EmitScalarExpr(E->getArg(1));
5991 llvm::Value *Z = EmitScalarExpr(E->getArg(2));
5992
5993 llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::AMDGPU_div_scale,
5994 X->getType());
5995
5996 llvm::Value *Tmp = Builder.CreateCall3(Callee, X, Y, Z);
5997
5998 llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0);
5999 llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
6000
6001 llvm::Type *RealFlagType
6002 = FlagOutPtr.first->getType()->getPointerElementType();
6003
6004 llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType);
6005 llvm::StoreInst *FlagStore = Builder.CreateStore(FlagExt, FlagOutPtr.first);
6006 FlagStore->setAlignment(FlagOutPtr.second);
6007 return Result;
6008 } default:
6009 return nullptr;
6010 }
6011 }
6012