Lines Matching full:ops
2572 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, in EmitNeonCall() argument
2579 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); in EmitNeonCall()
2581 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); in EmitNeonCall()
2583 return Builder.CreateCall(F, Ops, name); in EmitNeonCall()
3273 SmallVectorImpl<Value *> &Ops, in EmitCommonNeonSISDBuiltinExpr() argument
3296 std::swap(Ops[0], Ops[1]); in EmitCommonNeonSISDBuiltinExpr()
3312 if (Ops[j]->getType()->getPrimitiveSizeInBits() == in EmitCommonNeonSISDBuiltinExpr()
3316 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); in EmitCommonNeonSISDBuiltinExpr()
3319 Ops[j] = in EmitCommonNeonSISDBuiltinExpr()
3320 CGF.Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType()); in EmitCommonNeonSISDBuiltinExpr()
3321 Ops[j] = in EmitCommonNeonSISDBuiltinExpr()
3322 CGF.Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0); in EmitCommonNeonSISDBuiltinExpr()
3325 Value *Result = CGF.EmitNeonCall(F, Ops, s); in EmitCommonNeonSISDBuiltinExpr()
3337 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1) { in EmitCommonNeonBuiltinExpr() argument
3367 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
3368 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
3374 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
3375 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
3376 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); in EmitCommonNeonBuiltinExpr()
3381 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); in EmitCommonNeonBuiltinExpr()
3384 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); in EmitCommonNeonBuiltinExpr()
3390 std::swap(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
3400 return EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
3406 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); in EmitCommonNeonBuiltinExpr()
3410 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
3412 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
3413 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
3421 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
3433 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
3443 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); in EmitCommonNeonBuiltinExpr()
3444 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
3445 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
3480 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
3484 int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); in EmitCommonNeonBuiltinExpr()
3489 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
3490 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
3491 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext"); in EmitCommonNeonBuiltinExpr()
3496 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
3497 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
3498 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
3501 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); in EmitCommonNeonBuiltinExpr()
3506 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
3507 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1"); in EmitCommonNeonBuiltinExpr()
3518 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); in EmitCommonNeonBuiltinExpr()
3519 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
3520 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
3521 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
3530 Ops[0] = Builder.CreateInsertElement(V, Ld, CI); in EmitCommonNeonBuiltinExpr()
3531 return EmitNeonSplat(Ops[0], CI); in EmitCommonNeonBuiltinExpr()
3541 for (unsigned I = 2; I < Ops.size() - 1; ++I) in EmitCommonNeonBuiltinExpr()
3542 Ops[I] = Builder.CreateBitCast(Ops[I], Ty); in EmitCommonNeonBuiltinExpr()
3543 Ops.push_back(getAlignmentValue32(PtrOp1)); in EmitCommonNeonBuiltinExpr()
3544 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint); in EmitCommonNeonBuiltinExpr()
3545 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
3546 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
3547 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
3551 Ops[0] = Builder.CreateBitCast(Ops[0], DTy); in EmitCommonNeonBuiltinExpr()
3553 return Builder.CreateZExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
3554 return Builder.CreateSExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
3558 Ops[0] = Builder.CreateBitCast(Ops[0], QTy); in EmitCommonNeonBuiltinExpr()
3559 return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); in EmitCommonNeonBuiltinExpr()
3569 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitCommonNeonBuiltinExpr()
3579 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
3589 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); in EmitCommonNeonBuiltinExpr()
3593 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); in EmitCommonNeonBuiltinExpr()
3594 Ops[1] = in EmitCommonNeonBuiltinExpr()
3596 Ops.resize(2); in EmitCommonNeonBuiltinExpr()
3597 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
3601 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", in EmitCommonNeonBuiltinExpr()
3605 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", in EmitCommonNeonBuiltinExpr()
3612 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
3616 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", in EmitCommonNeonBuiltinExpr()
3620 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); in EmitCommonNeonBuiltinExpr()
3621 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], in EmitCommonNeonBuiltinExpr()
3625 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
3627 Ops[0] = Builder.CreateZExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
3629 Ops[0] = Builder.CreateSExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
3630 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); in EmitCommonNeonBuiltinExpr()
3631 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); in EmitCommonNeonBuiltinExpr()
3635 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
3636 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); in EmitCommonNeonBuiltinExpr()
3638 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
3640 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
3641 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); in EmitCommonNeonBuiltinExpr()
3645 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); in EmitCommonNeonBuiltinExpr()
3661 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
3662 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
3669 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
3670 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
3671 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); in EmitCommonNeonBuiltinExpr()
3676 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); in EmitCommonNeonBuiltinExpr()
3679 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); in EmitCommonNeonBuiltinExpr()
3683 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
3684 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
3685 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
3694 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
3695 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); in EmitCommonNeonBuiltinExpr()
3702 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
3703 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
3704 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
3705 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitCommonNeonBuiltinExpr()
3707 return Builder.CreateSExt(Ops[0], Ty, "vtst"); in EmitCommonNeonBuiltinExpr()
3711 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
3712 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
3713 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
3721 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
3722 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); in EmitCommonNeonBuiltinExpr()
3729 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
3730 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
3731 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
3740 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
3741 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); in EmitCommonNeonBuiltinExpr()
3753 Value *Result = EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
3781 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in packTBLDVectorList() argument
3791 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType()); in packTBLDVectorList()
3797 int PairPos = 0, End = Ops.size() - 1; in packTBLDVectorList()
3799 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
3800 Ops[PairPos+1], Indices, in packTBLDVectorList()
3809 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
3876 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; in EmitSpecialRegisterBuiltin() local
3877 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitSpecialRegisterBuiltin()
4005 Value *Ops[2]; in EmitARMBuiltinExpr() local
4007 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitARMBuiltinExpr()
4011 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitARMBuiltinExpr()
4278 SmallVector<Value*, 4> Ops; in EmitARMBuiltinExpr() local
4309 Ops.push_back(PtrOp0.getPointer()); in EmitARMBuiltinExpr()
4333 Ops.push_back(PtrOp1.getPointer()); in EmitARMBuiltinExpr()
4339 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitARMBuiltinExpr()
4346 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); in EmitARMBuiltinExpr()
4363 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); in EmitARMBuiltinExpr()
4375 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitARMBuiltinExpr()
4378 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, in EmitARMBuiltinExpr()
4381 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, in EmitARMBuiltinExpr()
4384 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, in EmitARMBuiltinExpr()
4387 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, in EmitARMBuiltinExpr()
4396 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], in EmitARMBuiltinExpr()
4397 Ops[3], Ops[4], Ops[5]}); in EmitARMBuiltinExpr()
4423 return Builder.CreateCall(F, Ops, "vcvtr"); in EmitARMBuiltinExpr()
4444 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1); in EmitARMBuiltinExpr()
4454 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
4455 uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitARMBuiltinExpr()
4457 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
4463 Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); in EmitARMBuiltinExpr()
4467 return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane"); in EmitARMBuiltinExpr()
4471 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
4474 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); in EmitARMBuiltinExpr()
4496 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, "vld_dup"); in EmitARMBuiltinExpr()
4497 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitARMBuiltinExpr()
4498 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
4499 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitARMBuiltinExpr()
4518 Args.push_back(Ops[1]); in EmitARMBuiltinExpr()
4525 Ops[1] = Builder.CreateCall(F, Args, "vld_dup"); in EmitARMBuiltinExpr()
4528 Value *Val = Builder.CreateExtractValue(Ops[1], i); in EmitARMBuiltinExpr()
4532 Ops[1] = Builder.CreateInsertValue(Ops[1], Elt, i); in EmitARMBuiltinExpr()
4534 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitARMBuiltinExpr()
4535 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
4536 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitARMBuiltinExpr()
4541 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", in EmitARMBuiltinExpr()
4545 Ops, "vqrshrun_n", 1, true); in EmitARMBuiltinExpr()
4548 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", in EmitARMBuiltinExpr()
4552 Ops, "vqshrun_n", 1, true); in EmitARMBuiltinExpr()
4556 Ops, "vrecpe"); in EmitARMBuiltinExpr()
4559 Ops, "vrshrn_n", 1, true); in EmitARMBuiltinExpr()
4562 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
4563 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
4564 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); in EmitARMBuiltinExpr()
4566 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); in EmitARMBuiltinExpr()
4567 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); in EmitARMBuiltinExpr()
4573 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); in EmitARMBuiltinExpr()
4575 Ops, "vsli_n"); in EmitARMBuiltinExpr()
4578 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
4579 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitARMBuiltinExpr()
4580 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitARMBuiltinExpr()
4585 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
4586 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); in EmitARMBuiltinExpr()
4587 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
4588 Ops[2] = getAlignmentValue32(PtrOp0); in EmitARMBuiltinExpr()
4589 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()}; in EmitARMBuiltinExpr()
4591 Tys), Ops); in EmitARMBuiltinExpr()
4595 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
4596 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitARMBuiltinExpr()
4597 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitARMBuiltinExpr()
4598 auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty)); in EmitARMBuiltinExpr()
4603 Ops, "vtbl1"); in EmitARMBuiltinExpr()
4606 Ops, "vtbl2"); in EmitARMBuiltinExpr()
4609 Ops, "vtbl3"); in EmitARMBuiltinExpr()
4612 Ops, "vtbl4"); in EmitARMBuiltinExpr()
4615 Ops, "vtbx1"); in EmitARMBuiltinExpr()
4618 Ops, "vtbx2"); in EmitARMBuiltinExpr()
4621 Ops, "vtbx3"); in EmitARMBuiltinExpr()
4624 Ops, "vtbx4"); in EmitARMBuiltinExpr()
4630 SmallVectorImpl<Value *> &Ops) { in EmitAArch64TblBuiltinExpr() argument
4685 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 1), nullptr, in EmitAArch64TblBuiltinExpr()
4686 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1, in EmitAArch64TblBuiltinExpr()
4690 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 2), nullptr, in EmitAArch64TblBuiltinExpr()
4691 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1, in EmitAArch64TblBuiltinExpr()
4695 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 3), nullptr, in EmitAArch64TblBuiltinExpr()
4696 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2, in EmitAArch64TblBuiltinExpr()
4700 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(0, 4), nullptr, in EmitAArch64TblBuiltinExpr()
4701 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2, in EmitAArch64TblBuiltinExpr()
4706 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 1), nullptr, Ops[2], in EmitAArch64TblBuiltinExpr()
4710 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); in EmitAArch64TblBuiltinExpr()
4713 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
4718 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 2), Ops[0], in EmitAArch64TblBuiltinExpr()
4719 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1, in EmitAArch64TblBuiltinExpr()
4724 packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 3), nullptr, Ops[4], in EmitAArch64TblBuiltinExpr()
4728 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], in EmitAArch64TblBuiltinExpr()
4732 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
4737 return packTBLDVectorList(CGF, makeArrayRef(Ops).slice(1, 4), Ops[0], in EmitAArch64TblBuiltinExpr()
4738 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2, in EmitAArch64TblBuiltinExpr()
4772 return CGF.EmitNeonCall(F, Ops, s); in EmitAArch64TblBuiltinExpr()
4855 Value *Ops[2]; in EmitAArch64BuiltinExpr() local
4857 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitAArch64BuiltinExpr()
4861 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitAArch64BuiltinExpr()
5027 llvm::SmallVector<Value*, 4> Ops; in EmitAArch64BuiltinExpr() local
5030 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitAArch64BuiltinExpr()
5038 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); in EmitAArch64BuiltinExpr()
5047 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); in EmitAArch64BuiltinExpr()
5048 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); in EmitAArch64BuiltinExpr()
5073 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); in EmitAArch64BuiltinExpr()
5082 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5083 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; in EmitAArch64BuiltinExpr()
5086 Ops[0] = Builder.CreateBitCast(Ops[0], FTy); in EmitAArch64BuiltinExpr()
5088 return Builder.CreateFPToUI(Ops[0], InTy); in EmitAArch64BuiltinExpr()
5089 return Builder.CreateFPToSI(Ops[0], InTy); in EmitAArch64BuiltinExpr()
5097 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5098 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; in EmitAArch64BuiltinExpr()
5101 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); in EmitAArch64BuiltinExpr()
5103 return Builder.CreateUIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
5104 return Builder.CreateSIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
5147 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5149 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
5154 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5156 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
5161 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5163 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
5168 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5170 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
5175 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5177 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
5181 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5182 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
5183 Ops[0] = in EmitAArch64BuiltinExpr()
5184 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty)); in EmitAArch64BuiltinExpr()
5185 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd"); in EmitAArch64BuiltinExpr()
5201 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
5202 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
5203 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
5204 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
5205 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
5221 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
5222 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); in EmitAArch64BuiltinExpr()
5223 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); in EmitAArch64BuiltinExpr()
5224 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
5225 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
5251 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
5252 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
5253 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
5254 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
5255 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); in EmitAArch64BuiltinExpr()
5259 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
5260 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
5261 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
5262 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
5263 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitAArch64BuiltinExpr()
5265 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd"); in EmitAArch64BuiltinExpr()
5277 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
5278 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
5281 Ops[1] = Builder.CreateBitCast(Ops[1], in EmitAArch64BuiltinExpr()
5283 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
5284 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
5287 Ops[1] = Builder.CreateBitCast(Ops[1], in EmitAArch64BuiltinExpr()
5289 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
5290 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
5294 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8)); in EmitAArch64BuiltinExpr()
5295 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5299 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16)); in EmitAArch64BuiltinExpr()
5300 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5304 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4)); in EmitAArch64BuiltinExpr()
5305 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5309 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8)); in EmitAArch64BuiltinExpr()
5310 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5314 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2)); in EmitAArch64BuiltinExpr()
5315 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5318 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
5320 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5324 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4)); in EmitAArch64BuiltinExpr()
5325 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5329 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1)); in EmitAArch64BuiltinExpr()
5330 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5333 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
5335 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5339 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2)); in EmitAArch64BuiltinExpr()
5340 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5343 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
5345 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5348 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
5350 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5354 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
5356 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5360 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
5362 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
5366 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); in EmitAArch64BuiltinExpr()
5369 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); in EmitAArch64BuiltinExpr()
5373 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
5376 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
5379 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
5384 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
5387 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
5388 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
5390 Ops, "vqshlu_n"); in EmitAArch64BuiltinExpr()
5397 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
5398 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
5399 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); in EmitAArch64BuiltinExpr()
5406 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
5407 int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); in EmitAArch64BuiltinExpr()
5408 Ops[1] = ConstantInt::get(Int64Ty, -SV); in EmitAArch64BuiltinExpr()
5409 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); in EmitAArch64BuiltinExpr()
5416 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
5417 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); in EmitAArch64BuiltinExpr()
5418 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), in EmitAArch64BuiltinExpr()
5419 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); in EmitAArch64BuiltinExpr()
5420 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); in EmitAArch64BuiltinExpr()
5426 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); in EmitAArch64BuiltinExpr()
5431 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
5441 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
5446 Ops[1] = Builder.CreateAShr( in EmitAArch64BuiltinExpr()
5447 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
5450 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
5456 // As Op + 0 = Op, return Ops[0] directly. in EmitAArch64BuiltinExpr()
5458 return Ops[0]; in EmitAArch64BuiltinExpr()
5459 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
5461 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
5467 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
5470 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
5471 ProductOps.push_back(vectorWrapScalar16(Ops[2])); in EmitAArch64BuiltinExpr()
5473 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
5476 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
5477 Ops.pop_back(); in EmitAArch64BuiltinExpr()
5483 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
5488 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
5490 Ops[1] = in EmitAArch64BuiltinExpr()
5497 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
5503 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
5506 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
5507 ProductOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
5508 Ops[1] = in EmitAArch64BuiltinExpr()
5511 Ops.pop_back(); in EmitAArch64BuiltinExpr()
5517 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
5534 Builtin->NameHint, Builtin->TypeModifier, E, Ops, in EmitAArch64BuiltinExpr()
5537 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops)) in EmitAArch64BuiltinExpr()
5546 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
5547 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
5548 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
5550 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); in EmitAArch64BuiltinExpr()
5551 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
5552 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
5553 return Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5559 Value *Addend = Ops[0]; in EmitAArch64BuiltinExpr()
5560 Value *Multiplicand = Ops[1]; in EmitAArch64BuiltinExpr()
5561 Value *LaneSource = Ops[2]; in EmitAArch64BuiltinExpr()
5562 Ops[0] = Multiplicand; in EmitAArch64BuiltinExpr()
5563 Ops[1] = LaneSource; in EmitAArch64BuiltinExpr()
5564 Ops[2] = Addend; in EmitAArch64BuiltinExpr()
5570 llvm::Constant *cst = cast<Constant>(Ops[3]); in EmitAArch64BuiltinExpr()
5572 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); in EmitAArch64BuiltinExpr()
5573 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); in EmitAArch64BuiltinExpr()
5575 Ops.pop_back(); in EmitAArch64BuiltinExpr()
5577 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); in EmitAArch64BuiltinExpr()
5583 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
5584 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
5587 Ops[2] = Builder.CreateBitCast(Ops[2], VTy); in EmitAArch64BuiltinExpr()
5588 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
5590 Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
5594 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5595 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
5599 Ops[2] = Builder.CreateBitCast(Ops[2], STy); in EmitAArch64BuiltinExpr()
5601 cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
5602 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); in EmitAArch64BuiltinExpr()
5604 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
5608 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5609 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
5611 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
5612 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
5613 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
5619 Ops.push_back(EmitScalarExpr(E->getArg(3))); in EmitAArch64BuiltinExpr()
5622 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
5623 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
5629 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitAArch64BuiltinExpr()
5635 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); in EmitAArch64BuiltinExpr()
5641 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); in EmitAArch64BuiltinExpr()
5647 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); in EmitAArch64BuiltinExpr()
5658 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
5661 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); in EmitAArch64BuiltinExpr()
5669 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); in EmitAArch64BuiltinExpr()
5675 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); in EmitAArch64BuiltinExpr()
5679 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); in EmitAArch64BuiltinExpr()
5683 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); in EmitAArch64BuiltinExpr()
5685 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
5687 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
5690 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
5692 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
5696 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); in EmitAArch64BuiltinExpr()
5699 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); in EmitAArch64BuiltinExpr()
5702 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); in EmitAArch64BuiltinExpr()
5705 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); in EmitAArch64BuiltinExpr()
5708 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); in EmitAArch64BuiltinExpr()
5712 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); in EmitAArch64BuiltinExpr()
5717 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndi"); in EmitAArch64BuiltinExpr()
5722 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); in EmitAArch64BuiltinExpr()
5727 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
5732 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); in EmitAArch64BuiltinExpr()
5737 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); in EmitAArch64BuiltinExpr()
5742 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); in EmitAArch64BuiltinExpr()
5746 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, in EmitAArch64BuiltinExpr()
5750 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, in EmitAArch64BuiltinExpr()
5754 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, in EmitAArch64BuiltinExpr()
5758 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, in EmitAArch64BuiltinExpr()
5762 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, in EmitAArch64BuiltinExpr()
5766 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5768 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitAArch64BuiltinExpr()
5769 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
5774 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
5776 return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
5782 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
5784 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
5794 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); in EmitAArch64BuiltinExpr()
5796 return Builder.CreateFPToUI(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5797 return Builder.CreateFPToSI(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5809 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); in EmitAArch64BuiltinExpr()
5821 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); in EmitAArch64BuiltinExpr()
5833 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); in EmitAArch64BuiltinExpr()
5845 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); in EmitAArch64BuiltinExpr()
5850 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); in EmitAArch64BuiltinExpr()
5858 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
5861 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
5862 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); in EmitAArch64BuiltinExpr()
5863 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
5871 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); in EmitAArch64BuiltinExpr()
5876 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); in EmitAArch64BuiltinExpr()
5881 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
5882 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); in EmitAArch64BuiltinExpr()
5887 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); in EmitAArch64BuiltinExpr()
5898 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5899 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
5900 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5910 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5911 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
5912 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5922 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5923 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
5924 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5934 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5935 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
5936 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5943 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5944 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5945 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5952 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5953 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5954 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5961 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5962 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5963 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5970 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5971 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5972 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5979 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5980 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5981 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
5988 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5989 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5990 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
5997 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
5998 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
5999 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
6006 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6007 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
6008 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
6015 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6016 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
6017 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
6024 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6025 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
6026 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
6033 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6034 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
6035 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
6042 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6043 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
6044 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
6051 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6052 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
6053 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
6060 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6061 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
6062 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
6069 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6070 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
6071 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
6078 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6079 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
6080 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
6083 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
6085 return Builder.CreateFMul(Ops[0], RHS); in EmitAArch64BuiltinExpr()
6092 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6093 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
6094 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
6101 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6102 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
6109 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6110 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
6111 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
6118 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6119 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
6126 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6127 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
6128 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
6135 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6136 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
6143 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6144 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
6145 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
6152 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
6153 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
6159 return EmitNeonCall(Intrin, Ops, "vsri_n"); in EmitAArch64BuiltinExpr()
6165 return EmitNeonCall(Intrin, Ops, "vsli_n"); in EmitAArch64BuiltinExpr()
6169 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
6170 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitAArch64BuiltinExpr()
6171 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
6176 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
6177 TmpOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
6180 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitAArch64BuiltinExpr()
6181 return Builder.CreateAdd(Ops[0], tmp); in EmitAArch64BuiltinExpr()
6192 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
6210 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); in EmitAArch64BuiltinExpr()
6211 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
6212 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
6213 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
6238 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
6239 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); in EmitAArch64BuiltinExpr()
6243 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); in EmitAArch64BuiltinExpr()
6244 return Builder.CreateDefaultAlignedLoad(Ops[0]); in EmitAArch64BuiltinExpr()
6247 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); in EmitAArch64BuiltinExpr()
6248 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
6249 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
6252 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
6254 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
6255 Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); in EmitAArch64BuiltinExpr()
6256 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); in EmitAArch64BuiltinExpr()
6261 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
6262 Ops[0] = Builder.CreateDefaultAlignedLoad(Ops[0]); in EmitAArch64BuiltinExpr()
6264 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); in EmitAArch64BuiltinExpr()
6265 return EmitNeonSplat(Ops[0], CI); in EmitAArch64BuiltinExpr()
6269 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
6270 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitAArch64BuiltinExpr()
6271 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
6272 return Builder.CreateDefaultAlignedStore(Ops[1], in EmitAArch64BuiltinExpr()
6273 Builder.CreateBitCast(Ops[0], Ty)); in EmitAArch64BuiltinExpr()
6277 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
6280 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
6281 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
6282 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
6283 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
6288 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
6291 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
6292 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
6293 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
6294 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
6299 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
6302 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
6303 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
6304 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
6305 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
6311 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
6314 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
6315 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
6316 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
6317 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
6323 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
6326 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
6327 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
6328 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
6329 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
6335 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
6338 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
6339 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
6340 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
6341 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
6345 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
6347 Ops.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
6348 Ops.erase(Ops.begin()+1); in EmitAArch64BuiltinExpr()
6349 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
6350 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
6351 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
6352 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane"); in EmitAArch64BuiltinExpr()
6353 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
6354 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
6355 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
6359 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
6361 Ops.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
6362 Ops.erase(Ops.begin()+1); in EmitAArch64BuiltinExpr()
6363 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
6364 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
6365 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
6366 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
6367 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane"); in EmitAArch64BuiltinExpr()
6368 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
6369 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
6370 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
6374 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
6376 Ops.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
6377 Ops.erase(Ops.begin()+1); in EmitAArch64BuiltinExpr()
6378 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
6379 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
6380 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
6381 Ops[4] = Builder.CreateBitCast(Ops[4], Ty); in EmitAArch64BuiltinExpr()
6382 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty); in EmitAArch64BuiltinExpr()
6383 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld4_lane"); in EmitAArch64BuiltinExpr()
6384 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
6385 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
6386 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
6390 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
6391 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
6392 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; in EmitAArch64BuiltinExpr()
6394 Ops, ""); in EmitAArch64BuiltinExpr()
6398 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
6399 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
6400 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitAArch64BuiltinExpr()
6401 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
6403 Ops, ""); in EmitAArch64BuiltinExpr()
6407 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
6408 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
6409 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
6411 Ops, ""); in EmitAArch64BuiltinExpr()
6415 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
6416 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
6417 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
6418 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
6420 Ops, ""); in EmitAArch64BuiltinExpr()
6424 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
6425 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
6426 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
6428 Ops, ""); in EmitAArch64BuiltinExpr()
6432 Ops.push_back(Ops[0]); in EmitAArch64BuiltinExpr()
6433 Ops.erase(Ops.begin()); in EmitAArch64BuiltinExpr()
6434 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
6435 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; in EmitAArch64BuiltinExpr()
6437 Ops, ""); in EmitAArch64BuiltinExpr()
6441 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
6442 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
6443 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
6452 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
6453 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); in EmitAArch64BuiltinExpr()
6460 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
6461 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
6462 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
6470 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
6471 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); in EmitAArch64BuiltinExpr()
6478 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
6479 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
6480 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
6489 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
6490 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); in EmitAArch64BuiltinExpr()
6497 Ops, "vtbl1"); in EmitAArch64BuiltinExpr()
6501 Ops, "vtbl2"); in EmitAArch64BuiltinExpr()
6505 Ops, "vtbl3"); in EmitAArch64BuiltinExpr()
6509 Ops, "vtbl4"); in EmitAArch64BuiltinExpr()
6513 Ops, "vtbx1"); in EmitAArch64BuiltinExpr()
6517 Ops, "vtbx2"); in EmitAArch64BuiltinExpr()
6521 Ops, "vtbx3"); in EmitAArch64BuiltinExpr()
6525 Ops, "vtbx4"); in EmitAArch64BuiltinExpr()
6530 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); in EmitAArch64BuiltinExpr()
6535 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); in EmitAArch64BuiltinExpr()
6541 BuildVector(ArrayRef<llvm::Value*> Ops) { in BuildVector() argument
6542 assert((Ops.size() & (Ops.size() - 1)) == 0 && in BuildVector()
6545 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) in BuildVector()
6546 AllConstants &= isa<Constant>(Ops[i]); in BuildVector()
6551 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
6552 CstOps.push_back(cast<Constant>(Ops[i])); in BuildVector()
6558 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size())); in BuildVector()
6560 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
6561 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i)); in BuildVector()
6588 SmallVectorImpl<Value *> &Ops, in EmitX86MaskedStore() argument
6591 Ops[0] = CGF.Builder.CreateBitCast(Ops[0], in EmitX86MaskedStore()
6592 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitX86MaskedStore()
6595 if (const auto *C = dyn_cast<Constant>(Ops[2])) in EmitX86MaskedStore()
6597 return CGF.Builder.CreateAlignedStore(Ops[1], Ops[0], Align); in EmitX86MaskedStore()
6599 Value *MaskVec = getMaskVecValue(CGF, Ops[2], in EmitX86MaskedStore()
6600 Ops[1]->getType()->getVectorNumElements()); in EmitX86MaskedStore()
6602 return CGF.Builder.CreateMaskedStore(Ops[1], Ops[0], Align, MaskVec); in EmitX86MaskedStore()
6606 SmallVectorImpl<Value *> &Ops, unsigned Align) { in EmitX86MaskedLoad() argument
6608 Ops[0] = CGF.Builder.CreateBitCast(Ops[0], in EmitX86MaskedLoad()
6609 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitX86MaskedLoad()
6612 if (const auto *C = dyn_cast<Constant>(Ops[2])) in EmitX86MaskedLoad()
6614 return CGF.Builder.CreateAlignedLoad(Ops[0], Align); in EmitX86MaskedLoad()
6616 Value *MaskVec = getMaskVecValue(CGF, Ops[2], in EmitX86MaskedLoad()
6617 Ops[1]->getType()->getVectorNumElements()); in EmitX86MaskedLoad()
6619 return CGF.Builder.CreateMaskedLoad(Ops[0], Align, MaskVec, Ops[1]); in EmitX86MaskedLoad()
6636 bool Signed, SmallVectorImpl<Value *> &Ops) { in EmitX86MaskedCompare() argument
6637 unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86MaskedCompare()
6657 Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]); in EmitX86MaskedCompare()
6660 const auto *C = dyn_cast<Constant>(Ops.back()); in EmitX86MaskedCompare()
6662 Cmp = CGF.Builder.CreateAnd(Cmp, getMaskVecValue(CGF, Ops.back(), NumElts)); in EmitX86MaskedCompare()
6705 SmallVector<Value*, 4> Ops; in EmitX86BuiltinExpr() local
6716 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitX86BuiltinExpr()
6725 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result)); in EmitX86BuiltinExpr()
6734 auto getCmpIntrinsicCall = [this, &Ops](Intrinsic::ID ID, unsigned Imm) { in EmitX86BuiltinExpr()
6735 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm)); in EmitX86BuiltinExpr()
6737 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
6745 auto getVectorFCmpIR = [this, &Ops](CmpInst::Predicate Pred) { in EmitX86BuiltinExpr()
6746 Value *Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
6747 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
6855 Value *Address = Ops[0]; in EmitX86BuiltinExpr()
6857 Value *Locality = Ops[1]; in EmitX86BuiltinExpr()
6869 return Builder.CreateBitCast(BuildVector(Ops), in EmitX86BuiltinExpr()
6872 return Builder.CreateExtractElement(Ops[0], in EmitX86BuiltinExpr()
6873 llvm::ConstantInt::get(Ops[1]->getType(), 0)); in EmitX86BuiltinExpr()
6876 Builder.CreateStore(Ops[0], Tmp); in EmitX86BuiltinExpr()
6920 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty); in EmitX86BuiltinExpr()
6921 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty); in EmitX86BuiltinExpr()
6922 Ops[1] = Mhi; in EmitX86BuiltinExpr()
6923 Ops.push_back(Mlo); in EmitX86BuiltinExpr()
6924 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
6944 return EmitX86MaskedStore(*this, Ops, 1); in EmitX86BuiltinExpr()
6960 return EmitX86MaskedStore(*this, Ops, Align); in EmitX86BuiltinExpr()
6980 return EmitX86MaskedLoad(*this, Ops, 1); in EmitX86BuiltinExpr()
6996 return EmitX86MaskedLoad(*this, Ops, Align); in EmitX86BuiltinExpr()
7004 Ops[1] = Builder.CreateBitCast(Ops[1], VecTy, "cast"); in EmitX86BuiltinExpr()
7009 Ops[1] = Builder.CreateExtractElement(Ops[1], Idx, "extract"); in EmitX86BuiltinExpr()
7012 Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy); in EmitX86BuiltinExpr()
7013 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitX86BuiltinExpr()
7020 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
7022 unsigned NumElts = Ops[0]->getType()->getVectorNumElements(); in EmitX86BuiltinExpr()
7034 Ops[1] = Ops[0]; in EmitX86BuiltinExpr()
7035 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
7049 Value *Align = Builder.CreateShuffleVector(Ops[1], Ops[0], in EmitX86BuiltinExpr()
7054 if (Ops.size() == 3) in EmitX86BuiltinExpr()
7057 return EmitX86Select(*this, Ops[4], Align, Ops[3]); in EmitX86BuiltinExpr()
7066 Value *BC = Builder.CreateBitCast(Ops[0], in EmitX86BuiltinExpr()
7067 llvm::PointerType::getUnqual(Ops[1]->getType()), in EmitX86BuiltinExpr()
7069 StoreInst *SI = Builder.CreateDefaultAlignedStore(Ops[1], BC); in EmitX86BuiltinExpr()
7082 Value *Scl = Builder.CreateExtractElement(Ops[1], (uint64_t)0, "extract"); in EmitX86BuiltinExpr()
7085 Value *BC = Builder.CreateBitCast(Ops[0], in EmitX86BuiltinExpr()
7114 return EmitX86Select(*this, Ops[0], Ops[1], Ops[2]); in EmitX86BuiltinExpr()
7127 return EmitX86MaskedCompare(*this, 0, false, Ops); in EmitX86BuiltinExpr()
7140 return EmitX86MaskedCompare(*this, 6, true, Ops); in EmitX86BuiltinExpr()
7153 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86BuiltinExpr()
7154 return EmitX86MaskedCompare(*this, CC, true, Ops); in EmitX86BuiltinExpr()
7168 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86BuiltinExpr()
7169 return EmitX86MaskedCompare(*this, CC, false, Ops); in EmitX86BuiltinExpr()
7178 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitX86BuiltinExpr()
7179 return EmitX86Select(*this, Ops[2], in EmitX86BuiltinExpr()
7180 Builder.CreateCall(F, {Ops[0],Builder.getInt1(false)}), in EmitX86BuiltinExpr()
7181 Ops[1]); in EmitX86BuiltinExpr()
7191 Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_SGT, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
7192 return Builder.CreateSelect(Cmp, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
7200 Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_UGT, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
7201 return Builder.CreateSelect(Cmp, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
7209 Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_SLT, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
7210 return Builder.CreateSelect(Cmp, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
7218 Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_ULT, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
7219 return Builder.CreateSelect(Cmp, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
7226 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); in EmitX86BuiltinExpr()
7228 return Builder.CreateCall(F, Ops, "pswapd"); in EmitX86BuiltinExpr()
7261 Ops[0]); in EmitX86BuiltinExpr()
7294 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
7329 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
7371 SmallVector<Value*, 4> Ops; in EmitPPCBuiltinExpr() local
7374 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitPPCBuiltinExpr()
7397 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); in EmitPPCBuiltinExpr()
7399 Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]); in EmitPPCBuiltinExpr()
7400 Ops.pop_back(); in EmitPPCBuiltinExpr()
7433 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
7445 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); in EmitPPCBuiltinExpr()
7446 Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]); in EmitPPCBuiltinExpr()
7447 Ops.pop_back(); in EmitPPCBuiltinExpr()
7474 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()