Lines Matching refs:Ops
2163 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, in EmitNeonCall() argument
2170 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); in EmitNeonCall()
2172 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); in EmitNeonCall()
2174 return Builder.CreateCall(F, Ops, name); in EmitNeonCall()
2864 SmallVectorImpl<Value *> &Ops, in EmitCommonNeonSISDBuiltinExpr() argument
2887 std::swap(Ops[0], Ops[1]); in EmitCommonNeonSISDBuiltinExpr()
2903 if (Ops[j]->getType()->getPrimitiveSizeInBits() == in EmitCommonNeonSISDBuiltinExpr()
2907 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); in EmitCommonNeonSISDBuiltinExpr()
2910 Ops[j] = in EmitCommonNeonSISDBuiltinExpr()
2911 CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType()); in EmitCommonNeonSISDBuiltinExpr()
2912 Ops[j] = in EmitCommonNeonSISDBuiltinExpr()
2913 CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0); in EmitCommonNeonSISDBuiltinExpr()
2916 Value *Result = CGF.EmitNeonCall(F, Ops, s); in EmitCommonNeonSISDBuiltinExpr()
2928 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1) { in EmitCommonNeonBuiltinExpr() argument
2958 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
2959 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
2965 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
2966 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
2967 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); in EmitCommonNeonBuiltinExpr()
2972 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); in EmitCommonNeonBuiltinExpr()
2975 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); in EmitCommonNeonBuiltinExpr()
2981 std::swap(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
2991 return EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
2997 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); in EmitCommonNeonBuiltinExpr()
3001 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
3003 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
3004 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
3012 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
3024 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
3034 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); in EmitCommonNeonBuiltinExpr()
3035 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
3036 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
3071 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
3075 int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); in EmitCommonNeonBuiltinExpr()
3080 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
3081 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
3083 return Builder.CreateShuffleVector(Ops[0], Ops[1], SV, "vext"); in EmitCommonNeonBuiltinExpr()
3088 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
3089 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
3090 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
3093 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); in EmitCommonNeonBuiltinExpr()
3098 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
3099 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1"); in EmitCommonNeonBuiltinExpr()
3110 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); in EmitCommonNeonBuiltinExpr()
3111 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
3112 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
3113 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
3122 Ops[0] = Builder.CreateInsertElement(V, Ld, CI); in EmitCommonNeonBuiltinExpr()
3123 return EmitNeonSplat(Ops[0], CI); in EmitCommonNeonBuiltinExpr()
3133 for (unsigned I = 2; I < Ops.size() - 1; ++I) in EmitCommonNeonBuiltinExpr()
3134 Ops[I] = Builder.CreateBitCast(Ops[I], Ty); in EmitCommonNeonBuiltinExpr()
3135 Ops.push_back(getAlignmentValue32(PtrOp1)); in EmitCommonNeonBuiltinExpr()
3136 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); in EmitCommonNeonBuiltinExpr()
3137 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
3138 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
3139 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
3143 Ops[0] = Builder.CreateBitCast(Ops[0], DTy); in EmitCommonNeonBuiltinExpr()
3145 return Builder.CreateZExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
3146 return Builder.CreateSExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
3150 Ops[0] = Builder.CreateBitCast(Ops[0], QTy); in EmitCommonNeonBuiltinExpr()
3151 return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); in EmitCommonNeonBuiltinExpr()
3161 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitCommonNeonBuiltinExpr()
3171 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
3181 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); in EmitCommonNeonBuiltinExpr()
3185 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); in EmitCommonNeonBuiltinExpr()
3186 Ops[1] = in EmitCommonNeonBuiltinExpr()
3188 Ops.resize(2); in EmitCommonNeonBuiltinExpr()
3189 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
3193 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", in EmitCommonNeonBuiltinExpr()
3197 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", in EmitCommonNeonBuiltinExpr()
3204 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
3208 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", in EmitCommonNeonBuiltinExpr()
3212 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); in EmitCommonNeonBuiltinExpr()
3213 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], in EmitCommonNeonBuiltinExpr()
3217 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
3219 Ops[0] = Builder.CreateZExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
3221 Ops[0] = Builder.CreateSExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
3222 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); in EmitCommonNeonBuiltinExpr()
3223 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); in EmitCommonNeonBuiltinExpr()
3227 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
3228 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); in EmitCommonNeonBuiltinExpr()
3230 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
3232 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
3233 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); in EmitCommonNeonBuiltinExpr()
3237 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); in EmitCommonNeonBuiltinExpr()
3253 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
3254 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
3261 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
3262 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
3263 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); in EmitCommonNeonBuiltinExpr()
3268 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); in EmitCommonNeonBuiltinExpr()
3271 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); in EmitCommonNeonBuiltinExpr()
3275 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
3276 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
3277 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
3286 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
3288 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); in EmitCommonNeonBuiltinExpr()
3295 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
3296 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
3297 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
3298 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitCommonNeonBuiltinExpr()
3300 return Builder.CreateSExt(Ops[0], Ty, "vtst"); in EmitCommonNeonBuiltinExpr()
3304 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
3305 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
3306 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
3314 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
3316 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); in EmitCommonNeonBuiltinExpr()
3323 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
3324 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
3325 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
3334 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
3336 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); in EmitCommonNeonBuiltinExpr()
3348 Value *Result = EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
3376 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in packTBLDVectorList() argument
3386 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType()); in packTBLDVectorList()
3393 int PairPos = 0, End = Ops.size() - 1; in packTBLDVectorList()
3395 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
3396 Ops[PairPos+1], SV, Name)); in packTBLDVectorList()
3404 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
3467 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; in EmitSpecialRegisterBuiltin() local
3468 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitSpecialRegisterBuiltin()
3596 Value *Ops[2]; in EmitARMBuiltinExpr() local
3598 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitARMBuiltinExpr()
3602 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitARMBuiltinExpr()
3801 SmallVector<Value*, 4> Ops; in EmitARMBuiltinExpr() local
3832 Ops.push_back(PtrOp0.getPointer()); in EmitARMBuiltinExpr()
3856 Ops.push_back(PtrOp1.getPointer()); in EmitARMBuiltinExpr()
3862 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitARMBuiltinExpr()
3869 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); in EmitARMBuiltinExpr()
3886 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); in EmitARMBuiltinExpr()
3898 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitARMBuiltinExpr()
3901 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, in EmitARMBuiltinExpr()
3904 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, in EmitARMBuiltinExpr()
3907 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, in EmitARMBuiltinExpr()
3910 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, in EmitARMBuiltinExpr()
3919 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], in EmitARMBuiltinExpr()
3920 Ops[3], Ops[4], Ops[5]}); in EmitARMBuiltinExpr()
3946 return Builder.CreateCall(F, Ops, "vcvtr"); in EmitARMBuiltinExpr()
3967 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1); in EmitARMBuiltinExpr()
3977 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
3978 uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitARMBuiltinExpr()
3980 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
3986 Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); in EmitARMBuiltinExpr()
3990 return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane"); in EmitARMBuiltinExpr()
3994 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
3997 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); in EmitARMBuiltinExpr()
4019 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup"); in EmitARMBuiltinExpr()
4020 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitARMBuiltinExpr()
4021 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
4022 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitARMBuiltinExpr()
4041 Args.push_back(Ops[1]); in EmitARMBuiltinExpr()
4048 Ops[1] = Builder.CreateCall(F, Args, "vld_dup"); in EmitARMBuiltinExpr()
4051 Value *Val = Builder.CreateExtractValue(Ops[1], i); in EmitARMBuiltinExpr()
4055 Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i); in EmitARMBuiltinExpr()
4057 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitARMBuiltinExpr()
4058 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
4059 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitARMBuiltinExpr()
4064 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", in EmitARMBuiltinExpr()
4068 Ops, "vqrshrun_n", 1, true); in EmitARMBuiltinExpr()
4071 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", in EmitARMBuiltinExpr()
4075 Ops, "vqshrun_n", 1, true); in EmitARMBuiltinExpr()
4079 Ops, "vrecpe"); in EmitARMBuiltinExpr()
4082 Ops, "vrshrn_n", 1, true); in EmitARMBuiltinExpr()
4085 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
4086 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
4087 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); in EmitARMBuiltinExpr()
4089 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); in EmitARMBuiltinExpr()
4090 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); in EmitARMBuiltinExpr()
4096 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); in EmitARMBuiltinExpr()
4098 Ops, "vsli_n"); in EmitARMBuiltinExpr()
4101 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
4102 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitARMBuiltinExpr()
4103 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitARMBuiltinExpr()
4108 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
4109 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); in EmitARMBuiltinExpr()
4110 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
4111 Ops[2] = getAlignmentValue32(PtrOp0); in EmitARMBuiltinExpr()
4112 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()}; in EmitARMBuiltinExpr()
4114 Tys), Ops); in EmitARMBuiltinExpr()
4118 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
4119 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitARMBuiltinExpr()
4120 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitARMBuiltinExpr()
4121 auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty)); in EmitARMBuiltinExpr()
4126 Ops, "vtbl1"); in EmitARMBuiltinExpr()
4129 Ops, "vtbl2"); in EmitARMBuiltinExpr()
4132 Ops, "vtbl3"); in EmitARMBuiltinExpr()
4135 Ops, "vtbl4"); in EmitARMBuiltinExpr()
4138 Ops, "vtbx1"); in EmitARMBuiltinExpr()
4141 Ops, "vtbx2"); in EmitARMBuiltinExpr()
4144 Ops, "vtbx3"); in EmitARMBuiltinExpr()
4147 Ops, "vtbx4"); in EmitARMBuiltinExpr()
4153 SmallVectorImpl<Value *> &Ops) { in EmitAArch64TblBuiltinExpr() argument
4208 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr, in EmitAArch64TblBuiltinExpr()
4209 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1, in EmitAArch64TblBuiltinExpr()
4213 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr, in EmitAArch64TblBuiltinExpr()
4214 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1, in EmitAArch64TblBuiltinExpr()
4218 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr, in EmitAArch64TblBuiltinExpr()
4219 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2, in EmitAArch64TblBuiltinExpr()
4223 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr, in EmitAArch64TblBuiltinExpr()
4224 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2, in EmitAArch64TblBuiltinExpr()
4229 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2], in EmitAArch64TblBuiltinExpr()
4233 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); in EmitAArch64TblBuiltinExpr()
4236 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
4241 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0], in EmitAArch64TblBuiltinExpr()
4242 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1, in EmitAArch64TblBuiltinExpr()
4247 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4], in EmitAArch64TblBuiltinExpr()
4251 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], in EmitAArch64TblBuiltinExpr()
4255 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
4260 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0], in EmitAArch64TblBuiltinExpr()
4261 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2, in EmitAArch64TblBuiltinExpr()
4295 return CGF.EmitNeonCall(F, Ops, s); in EmitAArch64TblBuiltinExpr()
4378 Value *Ops[2]; in EmitAArch64BuiltinExpr() local
4380 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitAArch64BuiltinExpr()
4384 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitAArch64BuiltinExpr()
4555 llvm::SmallVector<Value*, 4> Ops; in EmitAArch64BuiltinExpr() local
4558 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitAArch64BuiltinExpr()
4566 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); in EmitAArch64BuiltinExpr()
4575 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); in EmitAArch64BuiltinExpr()
4576 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); in EmitAArch64BuiltinExpr()
4601 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); in EmitAArch64BuiltinExpr()
4610 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
4611 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; in EmitAArch64BuiltinExpr()
4614 Ops[0] = Builder.CreateBitCast(Ops[0], FTy); in EmitAArch64BuiltinExpr()
4616 return Builder.CreateFPToUI(Ops[0], InTy); in EmitAArch64BuiltinExpr()
4617 return Builder.CreateFPToSI(Ops[0], InTy); in EmitAArch64BuiltinExpr()
4625 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
4626 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; in EmitAArch64BuiltinExpr()
4629 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); in EmitAArch64BuiltinExpr()
4631 return Builder.CreateUIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
4632 return Builder.CreateSIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
4675 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
4677 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
4682 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
4684 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
4689 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
4691 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
4696 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
4698 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
4703 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
4705 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
4709 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
4710 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
4711 Ops[0] = in EmitAArch64BuiltinExpr()
4712 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty)); in EmitAArch64BuiltinExpr()
4713 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd"); in EmitAArch64BuiltinExpr()
4729 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
4730 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
4731 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
4732 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
4733 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
4749 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
4750 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); in EmitAArch64BuiltinExpr()
4751 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); in EmitAArch64BuiltinExpr()
4752 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
4753 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
4779 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
4780 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
4781 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
4782 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
4783 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); in EmitAArch64BuiltinExpr()
4787 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
4788 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
4789 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
4790 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
4791 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitAArch64BuiltinExpr()
4793 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd"); in EmitAArch64BuiltinExpr()
4805 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
4806 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
4809 Ops[1] = Builder.CreateBitCast(Ops[1], in EmitAArch64BuiltinExpr()
4811 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
4812 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
4815 Ops[1] = Builder.CreateBitCast(Ops[1], in EmitAArch64BuiltinExpr()
4817 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
4818 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
4822 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8)); in EmitAArch64BuiltinExpr()
4823 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4827 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16)); in EmitAArch64BuiltinExpr()
4828 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4832 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4)); in EmitAArch64BuiltinExpr()
4833 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4837 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8)); in EmitAArch64BuiltinExpr()
4838 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4842 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2)); in EmitAArch64BuiltinExpr()
4843 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4846 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
4848 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4852 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4)); in EmitAArch64BuiltinExpr()
4853 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4857 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1)); in EmitAArch64BuiltinExpr()
4858 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4861 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
4863 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4867 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); in EmitAArch64BuiltinExpr()
4868 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4871 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
4873 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4876 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
4878 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4882 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
4884 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4888 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
4890 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
4894 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); in EmitAArch64BuiltinExpr()
4897 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); in EmitAArch64BuiltinExpr()
4901 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
4904 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
4907 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
4912 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
4915 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
4916 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
4918 Ops, "vqshlu_n"); in EmitAArch64BuiltinExpr()
4925 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
4926 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
4927 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); in EmitAArch64BuiltinExpr()
4934 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
4935 int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); in EmitAArch64BuiltinExpr()
4936 Ops[1] = ConstantInt::get(Int64Ty, -SV); in EmitAArch64BuiltinExpr()
4937 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); in EmitAArch64BuiltinExpr()
4944 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
4945 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); in EmitAArch64BuiltinExpr()
4946 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), in EmitAArch64BuiltinExpr()
4947 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); in EmitAArch64BuiltinExpr()
4948 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); in EmitAArch64BuiltinExpr()
4954 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); in EmitAArch64BuiltinExpr()
4959 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
4969 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
4974 Ops[1] = Builder.CreateAShr( in EmitAArch64BuiltinExpr()
4975 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
4978 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
4986 return Ops[0]; in EmitAArch64BuiltinExpr()
4987 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
4989 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
4995 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
4998 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
4999 ProductOps.push_back(vectorWrapScalar16(Ops[2])); in EmitAArch64BuiltinExpr()
5001 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
5004 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
5005 Ops.pop_back(); in EmitAArch64BuiltinExpr()
5011 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
5016 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
5018 Ops[1] = in EmitAArch64BuiltinExpr()
5025 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
5031 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
5034 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
5035 ProductOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
5036 Ops[1] = in EmitAArch64BuiltinExpr()
5039 Ops.pop_back(); in EmitAArch64BuiltinExpr()
5045 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
5062 Builtin->NameHint, Builtin->TypeModifier, E, Ops, in EmitAArch64BuiltinExpr()
5065 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops)) in EmitAArch64BuiltinExpr()
5074 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
5075 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
5076 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
5078 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); in EmitAArch64BuiltinExpr()
5079 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
5080 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
5081 return Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5087 Value *Addend = Ops[0]; in EmitAArch64BuiltinExpr()
5088 Value *Multiplicand = Ops[1]; in EmitAArch64BuiltinExpr()
5089 Value *LaneSource = Ops[2]; in EmitAArch64BuiltinExpr()
5090 Ops[0] = Multiplicand; in EmitAArch64BuiltinExpr()
5091 Ops[1] = LaneSource; in EmitAArch64BuiltinExpr()
5092 Ops[2] = Addend; in EmitAArch64BuiltinExpr()
5098 llvm::Constant *cst = cast<Constant>(Ops[3]); in EmitAArch64BuiltinExpr()
5100 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); in EmitAArch64BuiltinExpr()
5101 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); in EmitAArch64BuiltinExpr()
5103 Ops.pop_back(); in EmitAArch64BuiltinExpr()
5105 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); in EmitAArch64BuiltinExpr()
5111 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
5112 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
5115 Ops[2] = Builder.CreateBitCast(Ops[2], VTy); in EmitAArch64BuiltinExpr()
5116 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
5118 Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
5122 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5123 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
5127 Ops[2] = Builder.CreateBitCast(Ops[2], STy); in EmitAArch64BuiltinExpr()
5129 cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
5130 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); in EmitAArch64BuiltinExpr()
5132 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
5136 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5137 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
5139 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
5140 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
5141 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
5147 Ops.push_back(EmitScalarExpr(E->getArg(3))); in EmitAArch64BuiltinExpr()
5150 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
5151 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
5160 Value *Subtrahend = Ops[0]; in EmitAArch64BuiltinExpr()
5161 Value *Multiplicand = Ops[2]; in EmitAArch64BuiltinExpr()
5162 Ops[0] = Multiplicand; in EmitAArch64BuiltinExpr()
5163 Ops[2] = Subtrahend; in EmitAArch64BuiltinExpr()
5164 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
5165 Ops[1] = Builder.CreateFNeg(Ops[1]); in EmitAArch64BuiltinExpr()
5167 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmls"); in EmitAArch64BuiltinExpr()
5173 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitAArch64BuiltinExpr()
5179 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); in EmitAArch64BuiltinExpr()
5185 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); in EmitAArch64BuiltinExpr()
5191 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); in EmitAArch64BuiltinExpr()
5202 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
5205 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); in EmitAArch64BuiltinExpr()
5213 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); in EmitAArch64BuiltinExpr()
5219 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); in EmitAArch64BuiltinExpr()
5223 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); in EmitAArch64BuiltinExpr()
5227 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); in EmitAArch64BuiltinExpr()
5229 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
5231 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
5234 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
5236 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
5240 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); in EmitAArch64BuiltinExpr()
5243 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); in EmitAArch64BuiltinExpr()
5246 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); in EmitAArch64BuiltinExpr()
5249 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); in EmitAArch64BuiltinExpr()
5252 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); in EmitAArch64BuiltinExpr()
5256 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); in EmitAArch64BuiltinExpr()
5261 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi"); in EmitAArch64BuiltinExpr()
5266 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); in EmitAArch64BuiltinExpr()
5271 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
5276 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); in EmitAArch64BuiltinExpr()
5281 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); in EmitAArch64BuiltinExpr()
5286 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); in EmitAArch64BuiltinExpr()
5290 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, in EmitAArch64BuiltinExpr()
5294 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, in EmitAArch64BuiltinExpr()
5298 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, in EmitAArch64BuiltinExpr()
5302 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, in EmitAArch64BuiltinExpr()
5306 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, in EmitAArch64BuiltinExpr()
5310 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5312 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitAArch64BuiltinExpr()
5313 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
5318 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
5320 return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
5326 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
5328 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
5338 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); in EmitAArch64BuiltinExpr()
5340 return Builder.CreateFPToUI(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5341 return Builder.CreateFPToSI(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5353 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); in EmitAArch64BuiltinExpr()
5365 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); in EmitAArch64BuiltinExpr()
5377 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); in EmitAArch64BuiltinExpr()
5389 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); in EmitAArch64BuiltinExpr()
5394 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); in EmitAArch64BuiltinExpr()
5402 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
5405 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
5406 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); in EmitAArch64BuiltinExpr()
5407 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
5415 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); in EmitAArch64BuiltinExpr()
5420 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); in EmitAArch64BuiltinExpr()
5425 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5426 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); in EmitAArch64BuiltinExpr()
5431 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); in EmitAArch64BuiltinExpr()
5442 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5443 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
5444 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5454 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5455 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
5456 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5466 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5467 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
5468 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5478 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5479 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
5480 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5487 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5488 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5489 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5496 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5497 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5498 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5505 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5506 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5507 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5514 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5515 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5516 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5523 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5524 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5525 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5532 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5533 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5534 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5541 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5542 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5543 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5550 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5551 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5552 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5559 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5560 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
5561 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5568 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5569 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
5570 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5577 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5578 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
5579 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5586 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5587 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
5588 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5595 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5596 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
5597 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5604 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5605 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
5606 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5613 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5614 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
5615 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5622 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5623 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
5624 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5627 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
5629 return Builder.CreateFMul(Ops[0], RHS); in EmitAArch64BuiltinExpr()
5636 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5637 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
5638 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5645 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5646 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
5653 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5654 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
5655 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5662 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5663 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
5670 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5671 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
5672 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5679 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5680 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
5687 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5688 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
5689 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5696 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5697 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
5703 return EmitNeonCall(Intrin, Ops, "vsri_n"); in EmitAArch64BuiltinExpr()
5709 return EmitNeonCall(Intrin, Ops, "vsli_n"); in EmitAArch64BuiltinExpr()
5713 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5714 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitAArch64BuiltinExpr()
5715 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
5720 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
5721 TmpOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
5724 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitAArch64BuiltinExpr()
5725 return Builder.CreateAdd(Ops[0], tmp); in EmitAArch64BuiltinExpr()
5736 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
5754 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); in EmitAArch64BuiltinExpr()
5755 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
5756 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5757 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
5782 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
5783 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); in EmitAArch64BuiltinExpr()
5787 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); in EmitAArch64BuiltinExpr()
5788 return Builder.CreateDefaultAlignedLoad(Ops[0]); in EmitAArch64BuiltinExpr()
5791 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); in EmitAArch64BuiltinExpr()
5792 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
5793 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
5796 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
5798 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5799 Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); in EmitAArch64BuiltinExpr()
5800 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); in EmitAArch64BuiltinExpr()
5805 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5806 Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); in EmitAArch64BuiltinExpr()
5808 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); in EmitAArch64BuiltinExpr()
5809 return EmitNeonSplat(Ops[0], CI); in EmitAArch64BuiltinExpr()
5813 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
5814 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitAArch64BuiltinExpr()
5815 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
5816 return Builder.CreateDefaultAlignedStore(Ops[1], in EmitAArch64BuiltinExpr()
5817 Builder.CreateBitCast(Ops[0], Ty)); in EmitAArch64BuiltinExpr()
5821 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
5824 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
5825 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
5826 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
5827 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
5832 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
5835 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
5836 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
5837 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
5838 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
5843 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
5846 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
5847 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
5848 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
5849 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
5855 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
5858 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
5859 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
5860 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
5861 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
5867 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
5870 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
5871 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
5872 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
5873 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
5879 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
5882 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
5883 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
5884 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
5885 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
5889 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
5891 Ops.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
5892 Ops.erase(Ops.begin()+1); in EmitAArch64BuiltinExpr()
5893 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
5894 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
5895 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
5896 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane"); in EmitAArch64BuiltinExpr()
5897 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
5898 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5899 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
5903 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
5905 Ops.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
5906 Ops.erase(Ops.begin()+1); in EmitAArch64BuiltinExpr()
5907 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
5908 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
5909 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
5910 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
5911 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane"); in EmitAArch64BuiltinExpr()
5912 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
5913 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5914 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
5918 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
5920 Ops.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
5921 Ops.erase(Ops.begin()+1); in EmitAArch64BuiltinExpr()
5922 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
5923 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
5924 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
5925 Ops[4] = Builder.CreateBitCast(Ops[4], Ty); in EmitAArch64BuiltinExpr()
5926 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty); in EmitAArch64BuiltinExpr()
5927 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane"); in EmitAArch64BuiltinExpr()
5928 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
5929 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5930 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
5934 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
5935 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
5936 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; in EmitAArch64BuiltinExpr()
5938 Ops, ""); in EmitAArch64BuiltinExpr()
5942 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
5943 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
5944 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitAArch64BuiltinExpr()
5945 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
5947 Ops, ""); in EmitAArch64BuiltinExpr()
5951 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
5952 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
5953 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
5955 Ops, ""); in EmitAArch64BuiltinExpr()
5959 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
5960 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
5961 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
5962 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
5964 Ops, ""); in EmitAArch64BuiltinExpr()
5968 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
5969 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
5970 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
5972 Ops, ""); in EmitAArch64BuiltinExpr()
5976 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
5977 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
5978 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
5979 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; in EmitAArch64BuiltinExpr()
5981 Ops, ""); in EmitAArch64BuiltinExpr()
5985 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
5986 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
5987 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
5996 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
5998 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vtrn"); in EmitAArch64BuiltinExpr()
6005 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
6006 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
6007 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
6015 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
6017 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vuzp"); in EmitAArch64BuiltinExpr()
6024 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
6025 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
6026 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
6035 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
6037 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], SV, "vzip"); in EmitAArch64BuiltinExpr()
6044 Ops, "vtbl1"); in EmitAArch64BuiltinExpr()
6048 Ops, "vtbl2"); in EmitAArch64BuiltinExpr()
6052 Ops, "vtbl3"); in EmitAArch64BuiltinExpr()
6056 Ops, "vtbl4"); in EmitAArch64BuiltinExpr()
6060 Ops, "vtbx1"); in EmitAArch64BuiltinExpr()
6064 Ops, "vtbx2"); in EmitAArch64BuiltinExpr()
6068 Ops, "vtbx3"); in EmitAArch64BuiltinExpr()
6072 Ops, "vtbx4"); in EmitAArch64BuiltinExpr()
6077 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); in EmitAArch64BuiltinExpr()
6082 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); in EmitAArch64BuiltinExpr()
6088 BuildVector(ArrayRef<llvm::Value*> Ops) { in BuildVector() argument
6089 assert((Ops.size() & (Ops.size() - 1)) == 0 && in BuildVector()
6092 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) in BuildVector()
6093 AllConstants &= isa<Constant>(Ops[i]); in BuildVector()
6098 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
6099 CstOps.push_back(cast<Constant>(Ops[i])); in BuildVector()
6105 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size())); in BuildVector()
6107 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
6108 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i)); in BuildVector()
6140 SmallVector<Value*, 4> Ops; in EmitX86BuiltinExpr() local
6151 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitX86BuiltinExpr()
6160 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); in EmitX86BuiltinExpr()
6244 Value *Address = Ops[0]; in EmitX86BuiltinExpr()
6246 Value *Locality = Ops[1]; in EmitX86BuiltinExpr()
6258 return Builder.CreateBitCast(BuildVector(Ops), in EmitX86BuiltinExpr()
6261 return Builder.CreateExtractElement(Ops[0], in EmitX86BuiltinExpr()
6262 llvm::ConstantInt::get(Ops[1]->getType(), 0)); in EmitX86BuiltinExpr()
6265 Builder.CreateStore(Ops[0], Tmp); in EmitX86BuiltinExpr()
6309 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty); in EmitX86BuiltinExpr()
6310 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty); in EmitX86BuiltinExpr()
6311 Ops[1] = Mhi; in EmitX86BuiltinExpr()
6312 Ops.push_back(Mlo); in EmitX86BuiltinExpr()
6313 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
6321 Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast"); in EmitX86BuiltinExpr()
6326 Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract"); in EmitX86BuiltinExpr()
6329 Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); in EmitX86BuiltinExpr()
6330 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitX86BuiltinExpr()
6334 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
6337 cast<llvm::VectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
6351 Ops[1] = Ops[0]; in EmitX86BuiltinExpr()
6352 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
6368 return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr"); in EmitX86BuiltinExpr()
6372 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3; in EmitX86BuiltinExpr()
6389 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); in EmitX86BuiltinExpr()
6393 SV = Builder.CreateShuffleVector(Zero, Ops[0], SV, "pslldq"); in EmitX86BuiltinExpr()
6399 unsigned shiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() >> 3; in EmitX86BuiltinExpr()
6416 Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast"); in EmitX86BuiltinExpr()
6420 SV = Builder.CreateShuffleVector(Ops[0], Zero, SV, "psrldq"); in EmitX86BuiltinExpr()
6436 Value *BC = Builder.CreateBitCast(Ops[0], in EmitX86BuiltinExpr()
6437 llvm::PointerType::getUnqual(Ops[1]->getType()), in EmitX86BuiltinExpr()
6439 StoreInst *SI = Builder.CreateDefaultAlignedStore(Ops[1], BC); in EmitX86BuiltinExpr()
6457 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); in EmitX86BuiltinExpr()
6459 return Builder.CreateCall(F, Ops, "pswapd"); in EmitX86BuiltinExpr()
6492 Ops[0]); in EmitX86BuiltinExpr()
6640 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm)); in EmitX86BuiltinExpr()
6642 return Builder.CreateCall(F, Ops, name); in EmitX86BuiltinExpr()
6649 SmallVector<Value*, 4> Ops; in EmitPPCBuiltinExpr() local
6652 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitPPCBuiltinExpr()
6675 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); in EmitPPCBuiltinExpr()
6677 Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); in EmitPPCBuiltinExpr()
6678 Ops.pop_back(); in EmitPPCBuiltinExpr()
6711 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
6723 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); in EmitPPCBuiltinExpr()
6724 Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); in EmitPPCBuiltinExpr()
6725 Ops.pop_back(); in EmitPPCBuiltinExpr()
6752 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()