Lines Matching refs:CGF
30 CodeGenFunction &CGF; member in __anon56f3ab900111::AtomicInfo
43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) in AtomicInfo() argument
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), in AtomicInfo()
47 ASTContext &C = CGF.getContext(); in AtomicInfo()
54 EvaluationKind = CGF.getEvaluationKind(ValueTy); in AtomicInfo()
83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); in AtomicInfo()
87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( in AtomicInfo()
89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in AtomicInfo()
91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(), in AtomicInfo()
121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( in AtomicInfo()
172 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); in getAtomicSizeValue()
173 return CGF.CGM.getSize(size); in getAtomicSizeValue()
204 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); in projectValue()
206 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), in projectValue()
291 Address TempAlloca = CGF.CreateMemTemp( in CreateTempAlloca()
298 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in CreateTempAlloca()
303 static RValue emitAtomicLibcall(CodeGenFunction &CGF, in emitAtomicLibcall() argument
308 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); in emitAtomicLibcall()
309 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); in emitAtomicLibcall()
310 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); in emitAtomicLibcall()
311 return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args); in emitAtomicLibcall()
332 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); in requiresMemSetZero()
334 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), in requiresMemSetZero()
350 CGF.Builder.CreateMemSet( in emitMemSetZeroIfNecessary()
351 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), in emitMemSetZeroIfNecessary()
352 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), in emitMemSetZeroIfNecessary()
357 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, in emitAtomicCmpXchg() argument
364 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); in emitAtomicCmpXchg()
365 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); in emitAtomicCmpXchg()
367 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( in emitAtomicCmpXchg()
374 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); in emitAtomicCmpXchg()
375 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); in emitAtomicCmpXchg()
380 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); in emitAtomicCmpXchg()
385 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); in emitAtomicCmpXchg()
389 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); in emitAtomicCmpXchg()
391 CGF.Builder.SetInsertPoint(StoreExpectedBB); in emitAtomicCmpXchg()
393 CGF.Builder.CreateStore(Old, Val1); in emitAtomicCmpXchg()
395 CGF.Builder.CreateBr(ContinueBB); in emitAtomicCmpXchg()
397 CGF.Builder.SetInsertPoint(ContinueBB); in emitAtomicCmpXchg()
399 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); in emitAtomicCmpXchg()
405 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, in emitAtomicCmpXchgFailureSet() argument
437 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, in emitAtomicCmpXchgFailureSet()
445 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); in emitAtomicCmpXchgFailureSet()
448 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); in emitAtomicCmpXchgFailureSet()
450 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); in emitAtomicCmpXchgFailureSet()
452 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); in emitAtomicCmpXchgFailureSet()
454 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); in emitAtomicCmpXchgFailureSet()
461 CGF.Builder.SetInsertPoint(MonotonicBB); in emitAtomicCmpXchgFailureSet()
462 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, in emitAtomicCmpXchgFailureSet()
464 CGF.Builder.CreateBr(ContBB); in emitAtomicCmpXchgFailureSet()
467 CGF.Builder.SetInsertPoint(AcquireBB); in emitAtomicCmpXchgFailureSet()
468 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, in emitAtomicCmpXchgFailureSet()
470 CGF.Builder.CreateBr(ContBB); in emitAtomicCmpXchgFailureSet()
471 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), in emitAtomicCmpXchgFailureSet()
473 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), in emitAtomicCmpXchgFailureSet()
477 CGF.Builder.SetInsertPoint(SeqCstBB); in emitAtomicCmpXchgFailureSet()
478 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, in emitAtomicCmpXchgFailureSet()
480 CGF.Builder.CreateBr(ContBB); in emitAtomicCmpXchgFailureSet()
481 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), in emitAtomicCmpXchgFailureSet()
485 CGF.Builder.SetInsertPoint(ContBB); in emitAtomicCmpXchgFailureSet()
488 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, in EmitAtomicOp() argument
500 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, in EmitAtomicOp()
504 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, in EmitAtomicOp()
510 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, in EmitAtomicOp()
515 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); in EmitAtomicOp()
516 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); in EmitAtomicOp()
518 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); in EmitAtomicOp()
520 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); in EmitAtomicOp()
521 SI->addCase(CGF.Builder.getInt1(false), StrongBB); in EmitAtomicOp()
523 CGF.Builder.SetInsertPoint(StrongBB); in EmitAtomicOp()
524 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, in EmitAtomicOp()
526 CGF.Builder.CreateBr(ContBB); in EmitAtomicOp()
528 CGF.Builder.SetInsertPoint(WeakBB); in EmitAtomicOp()
529 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, in EmitAtomicOp()
531 CGF.Builder.CreateBr(ContBB); in EmitAtomicOp()
533 CGF.Builder.SetInsertPoint(ContBB); in EmitAtomicOp()
540 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); in EmitAtomicOp()
543 CGF.Builder.CreateStore(Load, Dest); in EmitAtomicOp()
550 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); in EmitAtomicOp()
551 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); in EmitAtomicOp()
611 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); in EmitAtomicOp()
613 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order); in EmitAtomicOp()
620 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); in EmitAtomicOp()
622 Result = CGF.Builder.CreateNot(Result); in EmitAtomicOp()
623 CGF.Builder.CreateStore(Result, Dest); in EmitAtomicOp()
629 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { in EmitValToTemp() argument
630 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); in EmitValToTemp()
631 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), in EmitValToTemp()
637 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, in AddDirectArgument() argument
642 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); in AddDirectArgument()
643 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); in AddDirectArgument()
645 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); in AddDirectArgument()
646 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(), in AddDirectArgument()
648 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align); in AddDirectArgument()
649 Val = CGF.EmitLoadOfScalar(Ptr, false, in AddDirectArgument()
650 CGF.getContext().getPointerType(ValTy), in AddDirectArgument()
656 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), in AddDirectArgument()
657 CGF.getContext().VoidPtrTy); in AddDirectArgument()
1145 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); in emitCastToAtomicIntPointer()
1146 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); in emitCastToAtomicIntPointer()
1151 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); in convertToAtomicIntPointer()
1154 CGF.Builder.CreateMemCpy(Tmp, Addr, in convertToAtomicIntPointer()
1172 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); in convertAtomicTempToRValue()
1176 return CGF.convertTempToRValue(addr, getValueType(), loc); in convertAtomicTempToRValue()
1180 return RValue::get(CGF.Builder.CreateLoad(addr)); in convertAtomicTempToRValue()
1182 return CGF.EmitLoadOfBitfieldLValue( in convertAtomicTempToRValue()
1186 return CGF.EmitLoadOfLValue( in convertAtomicTempToRValue()
1190 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( in convertAtomicTempToRValue()
1207 ? CGF.ConvertTypeForMem(ValueTy) in ConvertIntToValueOrAtomic()
1211 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); in ConvertIntToValueOrAtomic()
1213 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); in ConvertIntToValueOrAtomic()
1215 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); in ConvertIntToValueOrAtomic()
1232 CGF.Builder.CreateStore(IntVal, CastTemp) in ConvertIntToValueOrAtomic()
1242 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); in EmitAtomicLoadLibcall()
1243 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), in EmitAtomicLoadLibcall()
1244 CGF.getContext().VoidPtrTy); in EmitAtomicLoadLibcall()
1245 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), in EmitAtomicLoadLibcall()
1246 CGF.getContext().VoidPtrTy); in EmitAtomicLoadLibcall()
1248 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), in EmitAtomicLoadLibcall()
1249 CGF.getContext().IntTy); in EmitAtomicLoadLibcall()
1250 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); in EmitAtomicLoadLibcall()
1257 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); in EmitAtomicLoadOp()
1264 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); in EmitAtomicLoadOp()
1346 CGF.EmitAggregateCopy(getAtomicAddress(), in emitCopyIntoMemory()
1364 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); in emitCopyIntoMemory()
1366 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); in emitCopyIntoMemory()
1380 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); in materializeRValue()
1381 AtomicInfo Atomics(CGF, TempLV); in materializeRValue()
1392 return CGF.EmitToMemory(Value, ValueTy); in convertRValueToInt()
1395 CGF.getLLVMContext(), in convertRValueToInt()
1398 return CGF.Builder.CreatePtrToInt(Value, InputIntTy); in convertRValueToInt()
1400 return CGF.Builder.CreateBitCast(Value, InputIntTy); in convertRValueToInt()
1409 return CGF.Builder.CreateLoad(Addr); in convertRValueToInt()
1417 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), in EmitAtomicCompareExchangeOp()
1425 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); in EmitAtomicCompareExchangeOp()
1426 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); in EmitAtomicCompareExchangeOp()
1438 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); in EmitAtomicCompareExchangeLibcall()
1439 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), in EmitAtomicCompareExchangeLibcall()
1440 CGF.getContext().VoidPtrTy); in EmitAtomicCompareExchangeLibcall()
1441 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), in EmitAtomicCompareExchangeLibcall()
1442 CGF.getContext().VoidPtrTy); in EmitAtomicCompareExchangeLibcall()
1443 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), in EmitAtomicCompareExchangeLibcall()
1444 CGF.getContext().VoidPtrTy); in EmitAtomicCompareExchangeLibcall()
1446 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), in EmitAtomicCompareExchangeLibcall()
1447 CGF.getContext().IntTy); in EmitAtomicCompareExchangeLibcall()
1449 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), in EmitAtomicCompareExchangeLibcall()
1450 CGF.getContext().IntTy); in EmitAtomicCompareExchangeLibcall()
1451 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", in EmitAtomicCompareExchangeLibcall()
1452 CGF.getContext().BoolTy, Args); in EmitAtomicCompareExchangeLibcall()
1492 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, in EmitAtomicUpdateValue() argument
1500 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); in EmitAtomicUpdateValue()
1532 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); in EmitAtomicUpdateValue()
1537 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); in EmitAtomicUpdateValue()
1540 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, in EmitAtomicUpdateValue()
1553 auto *ContBB = CGF.createBasicBlock("atomic_cont"); in EmitAtomicUpdateLibcall()
1554 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); in EmitAtomicUpdateLibcall()
1555 CGF.EmitBlock(ContBB); in EmitAtomicUpdateLibcall()
1559 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); in EmitAtomicUpdateLibcall()
1560 CGF.Builder.CreateStore(OldVal, DesiredAddr); in EmitAtomicUpdateLibcall()
1565 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); in EmitAtomicUpdateLibcall()
1570 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); in EmitAtomicUpdateLibcall()
1571 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in EmitAtomicUpdateLibcall()
1582 auto *ContBB = CGF.createBasicBlock("atomic_cont"); in EmitAtomicUpdateOp()
1583 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); in EmitAtomicUpdateOp()
1584 auto *CurBB = CGF.Builder.GetInsertBlock(); in EmitAtomicUpdateOp()
1585 CGF.EmitBlock(ContBB); in EmitAtomicUpdateOp()
1586 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), in EmitAtomicUpdateOp()
1593 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); in EmitAtomicUpdateOp()
1597 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); in EmitAtomicUpdateOp()
1598 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); in EmitAtomicUpdateOp()
1601 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); in EmitAtomicUpdateOp()
1602 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); in EmitAtomicUpdateOp()
1603 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in EmitAtomicUpdateOp()
1606 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, in EmitAtomicUpdateValue() argument
1630 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); in EmitAtomicUpdateValue()
1640 auto *ContBB = CGF.createBasicBlock("atomic_cont"); in EmitAtomicUpdateLibcall()
1641 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); in EmitAtomicUpdateLibcall()
1642 CGF.EmitBlock(ContBB); in EmitAtomicUpdateLibcall()
1646 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); in EmitAtomicUpdateLibcall()
1647 CGF.Builder.CreateStore(OldVal, DesiredAddr); in EmitAtomicUpdateLibcall()
1649 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); in EmitAtomicUpdateLibcall()
1654 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); in EmitAtomicUpdateLibcall()
1655 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in EmitAtomicUpdateLibcall()
1665 auto *ContBB = CGF.createBasicBlock("atomic_cont"); in EmitAtomicUpdateOp()
1666 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); in EmitAtomicUpdateOp()
1667 auto *CurBB = CGF.Builder.GetInsertBlock(); in EmitAtomicUpdateOp()
1668 CGF.EmitBlock(ContBB); in EmitAtomicUpdateOp()
1669 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), in EmitAtomicUpdateOp()
1676 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); in EmitAtomicUpdateOp()
1678 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); in EmitAtomicUpdateOp()
1679 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); in EmitAtomicUpdateOp()
1682 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); in EmitAtomicUpdateOp()
1683 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); in EmitAtomicUpdateOp()
1684 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in EmitAtomicUpdateOp()