Lines Matching refs:CGF
69 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
73 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
140 void Enter(CodeGenFunction &CGF) override { in Enter() argument
143 auto PartIdLVal = CGF.EmitLoadOfPointerLValue( in Enter()
144 CGF.GetAddrOfLocalVar(PartIDVar), in Enter()
146 auto *Res = CGF.EmitLoadOfScalar(PartIdLVal, SourceLocation()); in Enter()
147 auto *DoneBB = CGF.createBasicBlock(".untied.done."); in Enter()
148 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB); in Enter()
149 CGF.EmitBlock(DoneBB); in Enter()
150 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); in Enter()
151 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in Enter()
152 UntiedSwitch->addCase(CGF.Builder.getInt32(0), in Enter()
153 CGF.Builder.GetInsertBlock()); in Enter()
154 emitUntiedSwitch(CGF); in Enter()
157 void emitUntiedSwitch(CodeGenFunction &CGF) const { in emitUntiedSwitch()
159 auto PartIdLVal = CGF.EmitLoadOfPointerLValue( in emitUntiedSwitch()
160 CGF.GetAddrOfLocalVar(PartIDVar), in emitUntiedSwitch()
162 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
164 UntiedCodeGen(CGF); in emitUntiedSwitch()
166 CGF.getJumpDestInCurrentScope(".untied.next."); in emitUntiedSwitch()
167 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); in emitUntiedSwitch()
168 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); in emitUntiedSwitch()
169 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), in emitUntiedSwitch()
170 CGF.Builder.GetInsertBlock()); in emitUntiedSwitch()
171 CGF.EmitBranchThroughCleanup(CurPoint); in emitUntiedSwitch()
172 CGF.EmitBlock(CurPoint.getBlock()); in emitUntiedSwitch()
192 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
197 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
198 Action.emitUntiedSwitch(CGF); in emitUntiedSwitch()
271 void emitUntiedSwitch(CodeGenFunction &CGF) override { in emitUntiedSwitch() argument
273 OuterRegionInfo->emitUntiedSwitch(CGF); in emitUntiedSwitch()
327 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS) in CGOpenMPInnerExprInfo() argument
328 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen, in CGOpenMPInnerExprInfo()
331 PrivScope(CGF) { in CGOpenMPInnerExprInfo()
347 PrivScope.addPrivate(VD, [&CGF, &DRE]() -> Address { in CGOpenMPInnerExprInfo()
348 return CGF.EmitLValue(&DRE).getAddress(); in CGOpenMPInnerExprInfo()
362 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override { in EmitBody() argument
386 CodeGenFunction &CGF; member in __anon50ddb8e60111::InlinedOpenMPRegionRAII
395 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen, in InlinedOpenMPRegionRAII() argument
397 : CGF(CGF) { in InlinedOpenMPRegionRAII()
399 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo( in InlinedOpenMPRegionRAII()
400 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel); in InlinedOpenMPRegionRAII()
401 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in InlinedOpenMPRegionRAII()
402 LambdaThisCaptureField = CGF.LambdaThisCaptureField; in InlinedOpenMPRegionRAII()
403 CGF.LambdaThisCaptureField = nullptr; in InlinedOpenMPRegionRAII()
409 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI(); in ~InlinedOpenMPRegionRAII()
410 delete CGF.CapturedStmtInfo; in ~InlinedOpenMPRegionRAII()
411 CGF.CapturedStmtInfo = OldCSI; in ~InlinedOpenMPRegionRAII()
412 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); in ~InlinedOpenMPRegionRAII()
413 CGF.LambdaThisCaptureField = LambdaThisCaptureField; in ~InlinedOpenMPRegionRAII()
677 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
678 if (!CGF.HaveInsertPoint()) in Emit()
680 Action->Exit(CGF); in Emit()
686 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const { in operator ()()
687 CodeGenFunction::RunCleanupsScope Scope(CGF); in operator ()()
689 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction); in operator ()()
690 Callback(CodeGen, CGF, *PrePostAction); in operator ()()
693 Callback(CodeGen, CGF, Action); in operator ()()
697 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
698 return CGF.EmitLoadOfPointerLValue( in getThreadIDVariableLValue()
699 CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
703 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) { in EmitBody() argument
704 if (!CGF.HaveInsertPoint()) in EmitBody()
711 CGF.EHStack.pushTerminate(); in EmitBody()
712 CodeGen(CGF); in EmitBody()
713 CGF.EHStack.popTerminate(); in EmitBody()
717 CodeGenFunction &CGF) { in getThreadIDVariableLValue() argument
718 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()), in getThreadIDVariableLValue()
760 CodeGenFunction CGF(CGM); in emitCombinerOrInitializer() local
763 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args); in emitCombinerOrInitializer()
764 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitCombinerOrInitializer()
765 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm); in emitCombinerOrInitializer()
766 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() -> Address { in emitCombinerOrInitializer()
767 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
770 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm); in emitCombinerOrInitializer()
771 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() -> Address { in emitCombinerOrInitializer()
772 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>()) in emitCombinerOrInitializer()
776 CGF.EmitIgnoredExpr(CombinerInitializer); in emitCombinerOrInitializer()
778 CGF.FinishFunction(); in emitCombinerOrInitializer()
783 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) { in emitUserDefinedReduction() argument
807 if (CGF) { in emitUserDefinedReduction()
808 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn); in emitUserDefinedReduction()
834 static Address createIdentFieldGEP(CodeGenFunction &CGF, Address Addr, in createIdentFieldGEP() argument
838 return CGF.Builder.CreateStructGEP(Addr, Field, Offset, Name); in createIdentFieldGEP()
847 CodeGenFunction CGF(CGM, true); in emitParallelOrTeamsOutlinedFunction() local
857 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitParallelOrTeamsOutlinedFunction()
858 return CGF.GenerateOpenMPCapturedStmtFunction(*CS); in emitParallelOrTeamsOutlinedFunction()
866 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF, in emitTaskOutlinedFunction()
868 auto *ThreadID = getThreadID(CGF, D.getLocStart()); in emitTaskOutlinedFunction()
869 auto *UpLoc = emitUpdateLocation(CGF, D.getLocStart()); in emitTaskOutlinedFunction()
872 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar), in emitTaskOutlinedFunction()
875 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs); in emitTaskOutlinedFunction()
884 CodeGenFunction CGF(CGM, true); in emitTaskOutlinedFunction() local
888 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTaskOutlinedFunction()
889 auto *Res = CGF.GenerateCapturedStmtFunction(*CS); in emitTaskOutlinedFunction()
926 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, in emitUpdateLocation() argument
935 assert(CGF.CurFn && "No function in current CodeGenFunction."); in emitUpdateLocation()
938 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); in emitUpdateLocation()
940 LocValue = Address(I->second.DebugLoc, getIdentAlign(CGF.CGM)); in emitUpdateLocation()
946 Address AI = CGF.CreateTempAlloca(IdentTy, getIdentAlign(CGF.CGM), in emitUpdateLocation()
948 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in emitUpdateLocation()
952 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in emitUpdateLocation()
953 CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt); in emitUpdateLocation()
954 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags), in emitUpdateLocation()
955 CGM.getSize(getIdentSize(CGF.CGM))); in emitUpdateLocation()
959 Address PSource = createIdentFieldGEP(CGF, LocValue, IdentField_PSource); in emitUpdateLocation()
966 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); in emitUpdateLocation()
969 dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) { in emitUpdateLocation()
973 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str()); in emitUpdateLocation()
977 CGF.Builder.CreateStore(OMPDebugLoc, PSource); in emitUpdateLocation()
984 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, in getThreadID() argument
986 assert(CGF.CurFn && "No function in current CodeGenFunction."); in getThreadID()
991 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); in getThreadID()
998 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in getThreadID()
1001 auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); in getThreadID()
1002 ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal(); in getThreadID()
1005 if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) { in getThreadID()
1006 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1017 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); in getThreadID()
1018 CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt); in getThreadID()
1020 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num), in getThreadID()
1021 emitUpdateLocation(CGF, Loc)); in getThreadID()
1022 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); in getThreadID()
1027 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) { in functionFinished() argument
1028 assert(CGF.CurFn && "No function in current CodeGenFunction."); in functionFinished()
1029 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) in functionFinished()
1030 OpenMPLocThreadIDMap.erase(CGF.CurFn); in functionFinished()
1031 if (FunctionUDRMap.count(CGF.CurFn) > 0) { in functionFinished()
1032 for(auto *D : FunctionUDRMap[CGF.CurFn]) { in functionFinished()
1035 FunctionUDRMap.erase(CGF.CurFn); in functionFinished()
1733 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, in getAddrOfThreadPrivate() argument
1742 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in getAddrOfThreadPrivate()
1743 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), in getAddrOfThreadPrivate()
1747 return Address(CGF.EmitRuntimeCall( in getAddrOfThreadPrivate()
1753 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, in emitThreadPrivateVarInit() argument
1757 auto OMPLoc = emitUpdateLocation(CGF, Loc); in emitThreadPrivateVarInit()
1758 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num), in emitThreadPrivateVarInit()
1763 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), in emitThreadPrivateVarInit()
1766 CGF.EmitRuntimeCall( in emitThreadPrivateVarInit()
1772 bool PerformInit, CodeGenFunction *CGF) { in emitThreadPrivateVarDefinition() argument
1865 if (!CGF) { in emitThreadPrivateVarDefinition()
1880 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); in emitThreadPrivateVarDefinition()
1892 static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, in emitOMPIfClause() argument
1895 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); in emitOMPIfClause()
1900 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { in emitOMPIfClause()
1902 ThenGen(CGF); in emitOMPIfClause()
1904 ElseGen(CGF); in emitOMPIfClause()
1910 auto ThenBlock = CGF.createBasicBlock("omp_if.then"); in emitOMPIfClause()
1911 auto ElseBlock = CGF.createBasicBlock("omp_if.else"); in emitOMPIfClause()
1912 auto ContBlock = CGF.createBasicBlock("omp_if.end"); in emitOMPIfClause()
1913 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0); in emitOMPIfClause()
1916 CGF.EmitBlock(ThenBlock); in emitOMPIfClause()
1917 ThenGen(CGF); in emitOMPIfClause()
1918 CGF.EmitBranch(ContBlock); in emitOMPIfClause()
1921 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitOMPIfClause()
1922 CGF.EmitBlock(ElseBlock); in emitOMPIfClause()
1923 ElseGen(CGF); in emitOMPIfClause()
1925 (void)ApplyDebugLocation::CreateEmpty(CGF); in emitOMPIfClause()
1926 CGF.EmitBranch(ContBlock); in emitOMPIfClause()
1928 CGF.EmitBlock(ContBlock, /*IsFinished=*/true); in emitOMPIfClause()
1931 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitParallelCall() argument
1935 if (!CGF.HaveInsertPoint()) in emitParallelCall()
1937 auto *RTLoc = emitUpdateLocation(CGF, Loc); in emitParallelCall()
1938 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF, in emitParallelCall()
1941 auto &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
1944 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitParallelCall()
1945 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())}; in emitParallelCall()
1951 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitParallelCall()
1953 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF, in emitParallelCall() argument
1955 auto &RT = CGF.CGM.getOpenMPRuntime(); in emitParallelCall()
1956 auto ThreadID = RT.getThreadID(CGF, Loc); in emitParallelCall()
1960 CGF.EmitRuntimeCall( in emitParallelCall()
1964 auto ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); in emitParallelCall()
1966 CGF.CreateTempAlloca(CGF.Int32Ty, CharUnits::fromQuantity(4), in emitParallelCall()
1968 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); in emitParallelCall()
1973 CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs); in emitParallelCall()
1976 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID}; in emitParallelCall()
1977 CGF.EmitRuntimeCall( in emitParallelCall()
1982 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen); in emitParallelCall()
1985 ThenRCG(CGF); in emitParallelCall()
1995 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, in emitThreadIDAddress() argument
1998 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitThreadIDAddress()
2000 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(); in emitThreadIDAddress()
2002 auto ThreadID = getThreadID(CGF, Loc); in emitThreadIDAddress()
2004 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); in emitThreadIDAddress()
2005 auto ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp."); in emitThreadIDAddress()
2006 CGF.EmitStoreOfScalar(ThreadID, in emitThreadIDAddress()
2007 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty)); in emitThreadIDAddress()
2053 void Enter(CodeGenFunction &CGF) override { in Enter() argument
2054 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); in Enter()
2056 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); in Enter()
2057 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); in Enter()
2058 ContBlock = CGF.createBasicBlock("omp_if.end"); in Enter()
2060 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); in Enter()
2061 CGF.EmitBlock(ThenBlock); in Enter()
2064 void Done(CodeGenFunction &CGF) { in Done() argument
2066 CGF.EmitBranch(ContBlock); in Done()
2067 CGF.EmitBlock(ContBlock, true); in Done()
2069 void Exit(CodeGenFunction &CGF) override { in Exit() argument
2070 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); in Exit()
2075 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, in emitCriticalRegion() argument
2083 if (!CGF.HaveInsertPoint()) in emitCriticalRegion()
2085 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCriticalRegion()
2090 EnterArgs.push_back(CGF.Builder.CreateIntCast( in emitCriticalRegion()
2091 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false)); in emitCriticalRegion()
2098 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); in emitCriticalRegion()
2101 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, in emitMasterRegion() argument
2104 if (!CGF.HaveInsertPoint()) in emitMasterRegion()
2111 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitMasterRegion()
2116 emitInlinedDirective(CGF, OMPD_master, MasterOpGen); in emitMasterRegion()
2117 Action.Done(CGF); in emitMasterRegion()
2120 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, in emitTaskyieldCall() argument
2122 if (!CGF.HaveInsertPoint()) in emitTaskyieldCall()
2126 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitTaskyieldCall()
2128 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args); in emitTaskyieldCall()
2129 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskyieldCall()
2130 Region->emitUntiedSwitch(CGF); in emitTaskyieldCall()
2133 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, in emitTaskgroupRegion() argument
2136 if (!CGF.HaveInsertPoint()) in emitTaskgroupRegion()
2142 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitTaskgroupRegion()
2147 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen); in emitTaskgroupRegion()
2152 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, in emitAddrOfVarFromArray() argument
2156 CGF.Builder.CreateConstArrayGEP(Array, Index, CGF.getPointerSize()); in emitAddrOfVarFromArray()
2157 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr); in emitAddrOfVarFromArray()
2159 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var)); in emitAddrOfVarFromArray()
2160 Addr = CGF.Builder.CreateElementBitCast( in emitAddrOfVarFromArray()
2161 Addr, CGF.ConvertTypeForMem(Var->getType())); in emitAddrOfVarFromArray()
2183 CodeGenFunction CGF(CGM); in emitCopyprivateCopyFunction() local
2184 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args); in emitCopyprivateCopyFunction()
2187 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
2188 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitCopyprivateCopyFunction()
2189 ArgsType), CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
2190 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitCopyprivateCopyFunction()
2191 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitCopyprivateCopyFunction()
2192 ArgsType), CGF.getPointerAlign()); in emitCopyprivateCopyFunction()
2199 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar); in emitCopyprivateCopyFunction()
2202 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar); in emitCopyprivateCopyFunction()
2206 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]); in emitCopyprivateCopyFunction()
2208 CGF.FinishFunction(); in emitCopyprivateCopyFunction()
2212 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, in emitSingleRegion() argument
2219 if (!CGF.HaveInsertPoint()) in emitSingleRegion()
2238 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it"); in emitSingleRegion()
2239 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt); in emitSingleRegion()
2242 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitSingleRegion()
2247 emitInlinedDirective(CGF, OMPD_single, SingleOpGen); in emitSingleRegion()
2250 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt); in emitSingleRegion()
2252 Action.Done(CGF); in emitSingleRegion()
2262 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list"); in emitSingleRegion()
2264 Address Elem = CGF.Builder.CreateConstArrayGEP( in emitSingleRegion()
2265 CopyprivateList, I, CGF.getPointerSize()); in emitSingleRegion()
2266 CGF.Builder.CreateStore( in emitSingleRegion()
2267 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitSingleRegion()
2268 CGF.EmitLValue(CopyprivateVars[I]).getPointer(), CGF.VoidPtrTy), in emitSingleRegion()
2274 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(), in emitSingleRegion()
2276 auto *BufSize = CGF.getTypeSize(CopyprivateArrayTy); in emitSingleRegion()
2278 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList, in emitSingleRegion()
2279 CGF.VoidPtrTy); in emitSingleRegion()
2280 auto *DidItVal = CGF.Builder.CreateLoad(DidIt); in emitSingleRegion()
2282 emitUpdateLocation(CGF, Loc), // ident_t *<loc> in emitSingleRegion()
2283 getThreadID(CGF, Loc), // i32 <gtid> in emitSingleRegion()
2289 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args); in emitSingleRegion()
2293 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, in emitOrderedRegion() argument
2296 if (!CGF.HaveInsertPoint()) in emitOrderedRegion()
2303 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitOrderedRegion()
2308 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
2311 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); in emitOrderedRegion()
2314 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, in emitBarrierCall() argument
2317 if (!CGF.HaveInsertPoint()) in emitBarrierCall()
2334 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), in emitBarrierCall()
2335 getThreadID(CGF, Loc)}; in emitBarrierCall()
2337 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitBarrierCall()
2339 auto *Result = CGF.EmitRuntimeCall( in emitBarrierCall()
2345 auto *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitBarrierCall()
2346 auto *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitBarrierCall()
2347 auto *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitBarrierCall()
2348 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitBarrierCall()
2349 CGF.EmitBlock(ExitBB); in emitBarrierCall()
2352 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitBarrierCall()
2353 CGF.EmitBranchThroughCleanup(CancelDestination); in emitBarrierCall()
2354 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitBarrierCall()
2359 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args); in emitBarrierCall()
2448 void CGOpenMPRuntime::emitForDispatchInit(CodeGenFunction &CGF, in emitForDispatchInit() argument
2454 if (!CGF.HaveInsertPoint()) in emitForDispatchInit()
2469 Chunk = CGF.Builder.getIntN(IVSize, 1); in emitForDispatchInit()
2471 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitForDispatchInit()
2472 CGF.Builder.getInt32(addMonoNonMonoModifier( in emitForDispatchInit()
2474 CGF.Builder.getIntN(IVSize, 0), // Lower in emitForDispatchInit()
2476 CGF.Builder.getIntN(IVSize, 1), // Stride in emitForDispatchInit()
2479 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args); in emitForDispatchInit()
2483 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, in emitForStaticInitCall() argument
2488 if (!CGF.HaveInsertPoint()) in emitForStaticInitCall()
2508 Chunk = CGF.Builder.getIntN(IVSize, 1); in emitForStaticInitCall()
2517 UpdateLocation, ThreadId, CGF.Builder.getInt32(addMonoNonMonoModifier( in emitForStaticInitCall()
2523 CGF.Builder.getIntN(IVSize, 1), // Incr in emitForStaticInitCall()
2526 CGF.EmitRuntimeCall(ForStaticInitFunction, Args); in emitForStaticInitCall()
2529 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF, in emitForStaticInit() argument
2538 auto *UpdatedLocation = emitUpdateLocation(CGF, Loc); in emitForStaticInit()
2539 auto *ThreadId = getThreadID(CGF, Loc); in emitForStaticInit()
2541 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitForStaticInit()
2547 CodeGenFunction &CGF, SourceLocation Loc, in emitDistributeStaticInit() argument
2552 auto *UpdatedLocation = emitUpdateLocation(CGF, Loc); in emitDistributeStaticInit()
2553 auto *ThreadId = getThreadID(CGF, Loc); in emitDistributeStaticInit()
2555 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, in emitDistributeStaticInit()
2561 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, in emitForStaticFinish() argument
2563 if (!CGF.HaveInsertPoint()) in emitForStaticFinish()
2566 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitForStaticFinish()
2567 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini), in emitForStaticFinish()
2571 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, in emitForOrderedIterationEnd() argument
2575 if (!CGF.HaveInsertPoint()) in emitForOrderedIterationEnd()
2578 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitForOrderedIterationEnd()
2579 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args); in emitForOrderedIterationEnd()
2582 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, in emitForNext() argument
2592 emitUpdateLocation(CGF, Loc), in emitForNext()
2593 getThreadID(CGF, Loc), in emitForNext()
2600 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args); in emitForNext()
2601 return CGF.EmitScalarConversion( in emitForNext()
2602 Call, CGF.getContext().getIntTypeForBitwidth(32, /* Signed */ true), in emitForNext()
2603 CGF.getContext().BoolTy, Loc); in emitForNext()
2606 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, in emitNumThreadsClause() argument
2609 if (!CGF.HaveInsertPoint()) in emitNumThreadsClause()
2613 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitNumThreadsClause()
2614 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)}; in emitNumThreadsClause()
2615 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads), in emitNumThreadsClause()
2619 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, in emitProcBindClause() argument
2622 if (!CGF.HaveInsertPoint()) in emitProcBindClause()
2649 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitProcBindClause()
2651 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args); in emitProcBindClause()
2654 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, in emitFlush() argument
2656 if (!CGF.HaveInsertPoint()) in emitFlush()
2659 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush), in emitFlush()
2660 emitUpdateLocation(CGF, Loc)); in emitFlush()
2769 CodeGenFunction CGF(CGM); in createOffloadingBinaryDescriptorFunction() local
2775 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FI, Args, SourceLocation()); in createOffloadingBinaryDescriptorFunction()
2776 Codegen(CGF); in createOffloadingBinaryDescriptorFunction()
2777 CGF.FinishFunction(); in createOffloadingBinaryDescriptorFunction()
2874 [&](CodeGenFunction &CGF, PrePostActionTy &) { in createOffloadingBinaryDescriptorRegistration() argument
2875 CGF.EmitCallOrInvoke(createRuntimeFunction(OMPRTL__tgt_unregister_lib), in createOffloadingBinaryDescriptorRegistration()
2880 [&](CodeGenFunction &CGF, PrePostActionTy &) { in createOffloadingBinaryDescriptorRegistration() argument
2881 CGF.EmitCallOrInvoke(createRuntimeFunction(OMPRTL__tgt_register_lib), in createOffloadingBinaryDescriptorRegistration()
2883 CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc); in createOffloadingBinaryDescriptorRegistration()
3285 CodeGenFunction CGF(CGM); in emitProxyTaskFunction() local
3286 CGF.disableDebugInfo(); in emitProxyTaskFunction()
3287 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args); in emitProxyTaskFunction()
3294 auto *GtidParam = CGF.EmitLoadOfScalar( in emitProxyTaskFunction()
3295 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc); in emitProxyTaskFunction()
3296 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitProxyTaskFunction()
3297 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitProxyTaskFunction()
3302 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitProxyTaskFunction()
3305 auto PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI); in emitProxyTaskFunction()
3309 auto SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI); in emitProxyTaskFunction()
3310 auto *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
3311 CGF.EmitLoadOfLValue(SharedsLVal, Loc).getScalarVal(), in emitProxyTaskFunction()
3312 CGF.ConvertTypeForMem(SharedsPtrTy)); in emitProxyTaskFunction()
3317 auto PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI); in emitProxyTaskFunction()
3318 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitProxyTaskFunction()
3319 PrivatesLVal.getPointer(), CGF.VoidPtrTy); in emitProxyTaskFunction()
3321 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitProxyTaskFunction()
3325 CGF.Builder in emitProxyTaskFunction()
3327 TDBase.getAddress(), CGF.VoidPtrTy) in emitProxyTaskFunction()
3333 auto LBLVal = CGF.EmitLValueForField(Base, *LBFI); in emitProxyTaskFunction()
3334 auto *LBParam = CGF.EmitLoadOfLValue(LBLVal, Loc).getScalarVal(); in emitProxyTaskFunction()
3336 auto UBLVal = CGF.EmitLValueForField(Base, *UBFI); in emitProxyTaskFunction()
3337 auto *UBParam = CGF.EmitLoadOfLValue(UBLVal, Loc).getScalarVal(); in emitProxyTaskFunction()
3339 auto StLVal = CGF.EmitLValueForField(Base, *StFI); in emitProxyTaskFunction()
3340 auto *StParam = CGF.EmitLoadOfLValue(StLVal, Loc).getScalarVal(); in emitProxyTaskFunction()
3342 auto LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitProxyTaskFunction()
3343 auto *LIParam = CGF.EmitLoadOfLValue(LILVal, Loc).getScalarVal(); in emitProxyTaskFunction()
3351 CGF.EmitCallOrInvoke(TaskFunction, CallArgs); in emitProxyTaskFunction()
3352 CGF.EmitStoreThroughLValue( in emitProxyTaskFunction()
3353 RValue::get(CGF.Builder.getInt32(/*C=*/0)), in emitProxyTaskFunction()
3354 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty)); in emitProxyTaskFunction()
3355 CGF.FinishFunction(); in emitProxyTaskFunction()
3381 CodeGenFunction CGF(CGM); in emitDestructorsFunction() local
3382 CGF.disableDebugInfo(); in emitDestructorsFunction()
3383 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo, in emitDestructorsFunction()
3386 LValue Base = CGF.EmitLoadOfPointerLValue( in emitDestructorsFunction()
3387 CGF.GetAddrOfLocalVar(&TaskTypeArg), in emitDestructorsFunction()
3392 Base = CGF.EmitLValueForField(Base, *FI); in emitDestructorsFunction()
3396 auto FieldLValue = CGF.EmitLValueForField(Base, Field); in emitDestructorsFunction()
3397 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType()); in emitDestructorsFunction()
3400 CGF.FinishFunction(); in emitDestructorsFunction()
3469 CodeGenFunction CGF(CGM); in emitTaskPrivateMappingFunction() local
3470 CGF.disableDebugInfo(); in emitTaskPrivateMappingFunction()
3471 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap, in emitTaskPrivateMappingFunction()
3475 LValue Base = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
3476 CGF.GetAddrOfLocalVar(&TaskPrivatesArg), in emitTaskPrivateMappingFunction()
3481 auto FieldLVal = CGF.EmitLValueForField(Base, Field); in emitTaskPrivateMappingFunction()
3483 auto RefLVal = CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); in emitTaskPrivateMappingFunction()
3484 auto RefLoadLVal = CGF.EmitLoadOfPointerLValue( in emitTaskPrivateMappingFunction()
3486 CGF.EmitStoreOfScalar(FieldLVal.getPointer(), RefLoadLVal); in emitTaskPrivateMappingFunction()
3489 CGF.FinishFunction(); in emitTaskPrivateMappingFunction()
3499 static void emitPrivatesInit(CodeGenFunction &CGF, in emitPrivatesInit() argument
3506 auto &C = CGF.getContext(); in emitPrivatesInit()
3508 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI); in emitPrivatesInit()
3511 SrcBase = CGF.MakeAddrLValue( in emitPrivatesInit()
3512 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitPrivatesInit()
3513 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)), in emitPrivatesInit()
3523 !CGF.isTrivialInitializer(Init)))) { in emitPrivatesInit()
3524 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI); in emitPrivatesInit()
3528 auto SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField); in emitPrivatesInit()
3529 SharedRefLValue = CGF.MakeAddrLValue( in emitPrivatesInit()
3535 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) { in emitPrivatesInit()
3537 CGF.EmitAggregateAssign(PrivateLValue.getAddress(), in emitPrivatesInit()
3542 CGF.EmitOMPAggregateAssign( in emitPrivatesInit()
3544 [&CGF, Elem, Init, &CapturesInfo](Address DestElement, in emitPrivatesInit()
3547 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
3553 CGF, &CapturesInfo); in emitPrivatesInit()
3554 CGF.EmitAnyExprToMem(Init, DestElement, in emitPrivatesInit()
3560 CodeGenFunction::OMPPrivateScope InitScope(CGF); in emitPrivatesInit()
3565 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo); in emitPrivatesInit()
3566 CGF.EmitExprAsInit(Init, VD, PrivateLValue, in emitPrivatesInit()
3570 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false); in emitPrivatesInit()
3577 static bool checkInitIsRequired(CodeGenFunction &CGF, in checkInitIsRequired() argument
3584 !CGF.isTrivialInitializer(Init)); in checkInitIsRequired()
3626 CodeGenFunction CGF(CGM); in emitTaskDupFunction() local
3627 CGF.disableDebugInfo(); in emitTaskDupFunction()
3628 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args); in emitTaskDupFunction()
3630 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
3631 CGF.GetAddrOfLocalVar(&DstArg), in emitTaskDupFunction()
3636 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
3638 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); in emitTaskDupFunction()
3639 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar( in emitTaskDupFunction()
3640 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc); in emitTaskDupFunction()
3641 CGF.EmitStoreOfScalar(Lastpriv, LILVal); in emitTaskDupFunction()
3648 LValue TDBase = CGF.EmitLoadOfPointerLValue( in emitTaskDupFunction()
3649 CGF.GetAddrOfLocalVar(&SrcArg), in emitTaskDupFunction()
3651 LValue Base = CGF.EmitLValueForField( in emitTaskDupFunction()
3654 CGF.EmitLoadOfScalar(CGF.EmitLValueForField( in emitTaskDupFunction()
3658 CGF.getNaturalTypeAlignment(SharedsTy)); in emitTaskDupFunction()
3660 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD, in emitTaskDupFunction()
3662 CGF.FinishFunction(); in emitTaskDupFunction()
3682 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskInit() argument
3736 auto *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy); in emitTaskInit()
3738 auto *KmpTaskTWithPrivatesTySize = CGF.getTypeSize(KmpTaskTWithPrivatesQTy); in emitTaskInit()
3752 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
3788 ? CGF.Builder.CreateSelect(Data.Final.getPointer(), in emitTaskInit()
3789 CGF.Builder.getInt32(FinalFlag), in emitTaskInit()
3790 CGF.Builder.getInt32(/*C=*/0)) in emitTaskInit()
3791 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0); in emitTaskInit()
3792 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags)); in emitTaskInit()
3794 llvm::Value *AllocArgs[] = {emitUpdateLocation(CGF, Loc), in emitTaskInit()
3795 getThreadID(CGF, Loc), TaskFlags, in emitTaskInit()
3797 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
3799 auto *NewTask = CGF.EmitRuntimeCall( in emitTaskInit()
3801 auto *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
3803 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, in emitTaskInit()
3806 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); in emitTaskInit()
3812 Address(CGF.EmitLoadOfScalar( in emitTaskInit()
3813 CGF.EmitLValueForField( in emitTaskInit()
3817 CGF.getNaturalTypeAlignment(SharedsTy)); in emitTaskInit()
3818 CGF.EmitAggregateCopy(KmpTaskSharedsPtr, Shareds, SharedsTy); in emitTaskInit()
3823 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD, in emitTaskInit()
3827 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) { in emitTaskInit()
3843 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI); in emitTaskInit()
3844 LValue DestructorsLV = CGF.EmitLValueForField( in emitTaskInit()
3846 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskInit()
3852 LValue Data2LV = CGF.EmitLValueForField( in emitTaskInit()
3854 LValue PriorityLV = CGF.EmitLValueForField( in emitTaskInit()
3856 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV); in emitTaskInit()
3866 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskCall() argument
3872 if (!CGF.HaveInsertPoint()) in emitTaskCall()
3876 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskCall()
3893 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); in emitTaskCall()
3911 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr"); in emitTaskCall()
3914 auto Addr = CGF.EmitLValue(E); in emitTaskCall()
3919 CGF.EmitOMPArraySectionExpr(ASE, /*LowerBound=*/false); in emitTaskCall()
3921 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.getPointer(), /*Idx0=*/1); in emitTaskCall()
3923 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy); in emitTaskCall()
3924 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy); in emitTaskCall()
3925 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); in emitTaskCall()
3927 Size = CGF.getTypeSize(Ty); in emitTaskCall()
3928 auto Base = CGF.MakeAddrLValue( in emitTaskCall()
3929 CGF.Builder.CreateConstArrayGEP(DependenciesArray, i, DependencySize), in emitTaskCall()
3932 auto BaseAddrLVal = CGF.EmitLValueForField( in emitTaskCall()
3934 CGF.EmitStoreOfScalar( in emitTaskCall()
3935 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy), in emitTaskCall()
3938 auto LenLVal = CGF.EmitLValueForField( in emitTaskCall()
3940 CGF.EmitStoreOfScalar(Size, LenLVal); in emitTaskCall()
3957 auto FlagsLVal = CGF.EmitLValueForField( in emitTaskCall()
3959 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind), in emitTaskCall()
3962 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitTaskCall()
3963 CGF.Builder.CreateStructGEP(DependenciesArray, 0, CharUnits::Zero()), in emitTaskCall()
3964 CGF.VoidPtrTy); in emitTaskCall()
3973 auto *ThreadID = getThreadID(CGF, Loc); in emitTaskCall()
3974 auto *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskCall()
3981 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies); in emitTaskCall()
3983 DepTaskArgs[5] = CGF.Builder.getInt32(0); in emitTaskCall()
3984 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
3988 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) { in emitTaskCall() argument
3991 auto PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI); in emitTaskCall()
3992 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal); in emitTaskCall()
3995 CGF.EmitRuntimeCall( in emitTaskCall()
3998 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), in emitTaskCall()
4003 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskCall()
4004 Region->emitUntiedSwitch(CGF); in emitTaskCall()
4011 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies); in emitTaskCall()
4013 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); in emitTaskCall()
4014 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in emitTaskCall()
4017 NumDependencies, &DepWaitTaskArgs](CodeGenFunction &CGF, in emitTaskCall()
4019 auto &RT = CGF.CGM.getOpenMPRuntime(); in emitTaskCall()
4020 CodeGenFunction::RunCleanupsScope LocalScope(CGF); in emitTaskCall()
4026 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps), in emitTaskCall()
4030 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTaskCall() argument
4031 Action.Enter(CGF); in emitTaskCall()
4033 CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs); in emitTaskCall()
4045 RCG(CGF); in emitTaskCall()
4049 emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); in emitTaskCall()
4052 ThenRCG(CGF); in emitTaskCall()
4056 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, in emitTaskLoopCall() argument
4062 if (!CGF.HaveInsertPoint()) in emitTaskLoopCall()
4065 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); in emitTaskLoopCall()
4071 llvm::Value *ThreadID = getThreadID(CGF, Loc); in emitTaskLoopCall()
4072 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); in emitTaskLoopCall()
4075 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy, in emitTaskLoopCall()
4078 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1); in emitTaskLoopCall()
4080 LValue LBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
4085 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(), in emitTaskLoopCall()
4087 LValue UBLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
4092 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(), in emitTaskLoopCall()
4094 LValue StLVal = CGF.EmitLValueForField( in emitTaskLoopCall()
4099 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(), in emitTaskLoopCall()
4104 UBLVal.getPointer(), CGF.EmitLoadOfScalar(StLVal, SourceLocation()), in emitTaskLoopCall()
4105 llvm::ConstantInt::getSigned(CGF.IntTy, Data.Nogroup ? 1 : 0), in emitTaskLoopCall()
4107 CGF.IntTy, Data.Schedule.getPointer() in emitTaskLoopCall()
4111 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty, in emitTaskLoopCall()
4113 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0), in emitTaskLoopCall()
4115 ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Result.TaskDupFn, in emitTaskLoopCall()
4116 CGF.VoidPtrTy) in emitTaskLoopCall()
4117 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)}; in emitTaskLoopCall()
4118 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs); in emitTaskLoopCall()
4131 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, in EmitOMPAggregateReduction() argument
4133 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *, in EmitOMPAggregateReduction() argument
4139 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar); in EmitOMPAggregateReduction()
4140 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar); in EmitOMPAggregateReduction()
4144 auto NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); in EmitOMPAggregateReduction()
4149 auto LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements); in EmitOMPAggregateReduction()
4151 auto BodyBB = CGF.createBasicBlock("omp.arraycpy.body"); in EmitOMPAggregateReduction()
4152 auto DoneBB = CGF.createBasicBlock("omp.arraycpy.done"); in EmitOMPAggregateReduction()
4154 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty"); in EmitOMPAggregateReduction()
4155 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); in EmitOMPAggregateReduction()
4158 auto EntryBB = CGF.Builder.GetInsertBlock(); in EmitOMPAggregateReduction()
4159 CGF.EmitBlock(BodyBB); in EmitOMPAggregateReduction()
4161 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); in EmitOMPAggregateReduction()
4163 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
4170 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI( in EmitOMPAggregateReduction()
4178 CodeGenFunction::OMPPrivateScope Scope(CGF); in EmitOMPAggregateReduction()
4182 RedOpGen(CGF, XExpr, EExpr, UpExpr); in EmitOMPAggregateReduction()
4186 auto LHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
4188 auto RHSElementNext = CGF.Builder.CreateConstGEP1_32( in EmitOMPAggregateReduction()
4192 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done"); in EmitOMPAggregateReduction()
4193 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); in EmitOMPAggregateReduction()
4194 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
4195 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock()); in EmitOMPAggregateReduction()
4198 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in EmitOMPAggregateReduction()
4204 static void emitReductionCombiner(CodeGenFunction &CGF, in emitReductionCombiner() argument
4212 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); in emitReductionCombiner()
4214 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); in emitReductionCombiner()
4215 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
4218 CGF.EmitIgnoredExpr(ReductionOp); in emitReductionCombiner()
4242 CodeGenFunction CGF(CGM); in emitReductionFunction() local
4243 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args); in emitReductionFunction()
4247 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
4248 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), in emitReductionFunction()
4249 ArgsType), CGF.getPointerAlign()); in emitReductionFunction()
4250 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReductionFunction()
4251 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), in emitReductionFunction()
4252 ArgsType), CGF.getPointerAlign()); in emitReductionFunction()
4257 CodeGenFunction::OMPPrivateScope Scope(CGF); in emitReductionFunction()
4263 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar); in emitReductionFunction()
4267 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar); in emitReductionFunction()
4274 CGF.Builder.CreateConstArrayGEP(LHS, Idx, CGF.getPointerSize()); in emitReductionFunction()
4275 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem); in emitReductionFunction()
4276 auto *VLA = CGF.getContext().getAsVariableArrayType(PrivTy); in emitReductionFunction()
4279 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy))); in emitReductionFunction()
4280 CGF.EmitVariablyModifiedType(PrivTy); in emitReductionFunction()
4293 CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReductionFunction()
4294 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitReductionFunction() argument
4295 emitReductionCombiner(CGF, E); in emitReductionFunction()
4299 emitReductionCombiner(CGF, E); in emitReductionFunction()
4305 CGF.FinishFunction(); in emitReductionFunction()
4309 static void emitSingleReductionCombiner(CodeGenFunction &CGF, in emitSingleReductionCombiner() argument
4319 CGF, PrivateRef->getType(), LHSVar, RHSVar, in emitSingleReductionCombiner()
4320 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { in emitSingleReductionCombiner() argument
4321 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
4325 emitReductionCombiner(CGF, ReductionOp); in emitSingleReductionCombiner()
4328 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, in emitReduction() argument
4334 if (!CGF.HaveInsertPoint()) in emitReduction()
4374 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitReduction()
4379 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
4401 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); in emitReduction()
4406 CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, CGF.getPointerSize()); in emitReduction()
4407 CGF.Builder.CreateStore( in emitReduction()
4408 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitReduction()
4409 CGF.EmitLValue(RHSExprs[I]).getPointer(), CGF.VoidPtrTy), in emitReduction()
4414 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx, in emitReduction()
4415 CGF.getPointerSize()); in emitReduction()
4416 llvm::Value *Size = CGF.Builder.CreateIntCast( in emitReduction()
4417 CGF.getVLASize( in emitReduction()
4418 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) in emitReduction()
4420 CGF.SizeTy, /*isSigned=*/false); in emitReduction()
4421 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), in emitReduction()
4428 CGM, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates, in emitReduction()
4436 auto *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE); in emitReduction()
4437 auto *ThreadId = getThreadID(CGF, Loc); in emitReduction()
4438 auto *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); in emitReduction()
4440 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList.getPointer(), in emitReduction()
4441 CGF.VoidPtrTy); in emitReduction()
4445 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n> in emitReduction()
4451 auto Res = CGF.EmitRuntimeCall( in emitReduction()
4457 auto *DefaultBB = CGF.createBasicBlock(".omp.reduction.default"); in emitReduction()
4458 auto *SwInst = CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2); in emitReduction()
4466 auto *Case1BB = CGF.createBasicBlock(".omp.reduction.case1"); in emitReduction()
4467 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB); in emitReduction()
4468 CGF.EmitBlock(Case1BB); in emitReduction()
4477 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
4482 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), in emitReduction()
4496 RCG(CGF); in emitReduction()
4498 CGF.EmitBranch(DefaultBB); in emitReduction()
4505 auto *Case2BB = CGF.createBasicBlock(".omp.reduction.case2"); in emitReduction()
4506 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB); in emitReduction()
4507 CGF.EmitBlock(Case2BB); in emitReduction()
4510 CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
4544 Loc](CodeGenFunction &CGF, const Expr *XExpr, in emitReduction()
4546 LValue X = CGF.EmitLValue(XExpr); in emitReduction()
4549 E = CGF.EmitAnyExpr(EExpr); in emitReduction()
4550 CGF.EmitOMPAtomicSimpleUpdateExpr( in emitReduction()
4553 [&CGF, UpExpr, VD, IPriv, Loc](RValue XRValue) { in emitReduction()
4554 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitReduction()
4556 VD, [&CGF, VD, XRValue, Loc]() -> Address { in emitReduction()
4557 Address LHSTemp = CGF.CreateMemTemp(VD->getType()); in emitReduction()
4558 CGF.emitOMPSimpleStore( in emitReduction()
4559 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue, in emitReduction()
4564 return CGF.EmitAnyExpr(UpExpr); in emitReduction()
4570 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar, in emitReduction()
4574 AtomicRedGen(CGF, XExpr, EExpr, UpExpr); in emitReduction()
4577 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *, in emitReduction()
4579 auto &RT = CGF.CGM.getOpenMPRuntime(); in emitReduction()
4581 CGF, ".atomic_reduction", in emitReduction()
4582 [=](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitReduction() argument
4583 Action.Enter(CGF); in emitReduction()
4584 emitReductionCombiner(CGF, E); in emitReduction()
4591 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, in emitReduction()
4594 CritRedGen(CGF, nullptr, nullptr, nullptr); in emitReduction()
4613 AtomicRCG(CGF); in emitReduction()
4615 AtomicRCG(CGF); in emitReduction()
4617 CGF.EmitBranch(DefaultBB); in emitReduction()
4618 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true); in emitReduction()
4621 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, in emitTaskwaitCall() argument
4623 if (!CGF.HaveInsertPoint()) in emitTaskwaitCall()
4627 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; in emitTaskwaitCall()
4629 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskwait), Args); in emitTaskwaitCall()
4630 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) in emitTaskwaitCall()
4631 Region->emitUntiedSwitch(CGF); in emitTaskwaitCall()
4634 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, in emitInlinedDirective() argument
4638 if (!CGF.HaveInsertPoint()) in emitInlinedDirective()
4640 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel); in emitInlinedDirective()
4641 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); in emitInlinedDirective()
4670 CodeGenFunction &CGF, SourceLocation Loc, in emitCancellationPointCall() argument
4672 if (!CGF.HaveInsertPoint()) in emitCancellationPointCall()
4677 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancellationPointCall()
4680 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), in emitCancellationPointCall()
4681 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancellationPointCall()
4683 auto *Result = CGF.EmitRuntimeCall( in emitCancellationPointCall()
4689 auto *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancellationPointCall()
4690 auto *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancellationPointCall()
4691 auto *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancellationPointCall()
4692 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancellationPointCall()
4693 CGF.EmitBlock(ExitBB); in emitCancellationPointCall()
4695 emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); in emitCancellationPointCall()
4698 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancellationPointCall()
4699 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancellationPointCall()
4700 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancellationPointCall()
4705 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, in emitCancelCall() argument
4708 if (!CGF.HaveInsertPoint()) in emitCancelCall()
4713 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { in emitCancelCall()
4714 auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](CodeGenFunction &CGF, in emitCancelCall()
4716 auto &RT = CGF.CGM.getOpenMPRuntime(); in emitCancelCall()
4718 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc), in emitCancelCall()
4719 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; in emitCancelCall()
4721 auto *Result = CGF.EmitRuntimeCall( in emitCancelCall()
4727 auto *ExitBB = CGF.createBasicBlock(".cancel.exit"); in emitCancelCall()
4728 auto *ContBB = CGF.createBasicBlock(".cancel.continue"); in emitCancelCall()
4729 auto *Cmp = CGF.Builder.CreateIsNotNull(Result); in emitCancelCall()
4730 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); in emitCancelCall()
4731 CGF.EmitBlock(ExitBB); in emitCancelCall()
4733 RT.emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); in emitCancelCall()
4736 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); in emitCancelCall()
4737 CGF.EmitBranchThroughCleanup(CancelDest); in emitCancelCall()
4738 CGF.EmitBlock(ContBB, /*IsFinished=*/true); in emitCancelCall()
4741 emitOMPIfClause(CGF, IfCond, ThenGen, in emitCancelCall()
4745 ThenRCG(CGF); in emitCancelCall()
4814 CodeGenFunction CGF(CGM, true); in emitTargetOutlinedFunctionHelper() local
4816 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitTargetOutlinedFunctionHelper()
4818 OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS); in emitTargetOutlinedFunctionHelper()
4864 CodeGenFunction &CGF, in emitNumTeamsClauseForTargetDirective() argument
4867 assert(!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the " in emitNumTeamsClauseForTargetDirective()
4891 CGOpenMPInnerExprInfo CGInfo(CGF, CS); in emitNumTeamsClauseForTargetDirective()
4892 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitNumTeamsClauseForTargetDirective()
4893 llvm::Value *NumTeams = CGF.EmitScalarExpr(NTE->getNumTeams()); in emitNumTeamsClauseForTargetDirective()
4894 return CGF.Builder.CreateIntCast(NumTeams, CGF.Int32Ty, in emitNumTeamsClauseForTargetDirective()
4900 return CGF.Builder.getInt32(0); in emitNumTeamsClauseForTargetDirective()
4913 CodeGenFunction &CGF, in emitThreadLimitClauseForTargetDirective() argument
4916 assert(!CGF.getLangOpts().OpenMPIsDevice && "Clauses associated with the " in emitThreadLimitClauseForTargetDirective()
4940 CGOpenMPInnerExprInfo CGInfo(CGF, CS); in emitThreadLimitClauseForTargetDirective()
4941 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); in emitThreadLimitClauseForTargetDirective()
4942 llvm::Value *ThreadLimit = CGF.EmitScalarExpr(TLE->getThreadLimit()); in emitThreadLimitClauseForTargetDirective()
4943 return CGF.Builder.CreateIntCast(ThreadLimit, CGF.Int32Ty, in emitThreadLimitClauseForTargetDirective()
4949 return CGF.Builder.getInt32(0); in emitThreadLimitClauseForTargetDirective()
4999 CodeGenFunction &CGF; member in __anon50ddb8e62b11::MappableExprsHandler
5022 return CGF.getTypeSize(BaseTy); in getExprTypeSize()
5026 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType()); in getExprTypeSize()
5030 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType()); in getExprTypeSize()
5038 auto *LengthVal = CGF.EmitScalarExpr(OAE->getLength()); in getExprTypeSize()
5040 CGF.Builder.CreateIntCast(LengthVal, CGF.SizeTy, /*isSigned=*/false); in getExprTypeSize()
5041 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize); in getExprTypeSize()
5043 return CGF.getTypeSize(ExprTy); in getExprTypeSize()
5119 if (!Length->EvaluateAsInt(ConstLength, CGF.getContext())) in isFinalArraySectionExpression()
5265 BP = CGF.EmitScalarExpr(ME->getBase()); in generateInfoForComponentList()
5269 BP = CGF.EmitLValue(cast<DeclRefExpr>(I->getAssociatedExpression())) in generateInfoForComponentList()
5277 auto PtrAddr = CGF.MakeNaturalAlignAddrLValue( in generateInfoForComponentList()
5279 BP = CGF.EmitLoadOfPointerLValue(PtrAddr.getAddress(), in generateInfoForComponentList()
5328 auto *LB = CGF.EmitLValue(I->getAssociatedExpression()).getPointer(); in generateInfoForComponentList()
5375 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF) in MappableExprsHandler() argument
5376 : Directive(Dir), CGF(CGF) { in MappableExprsHandler()
5493 CurSizes.push_back(CGF.getTypeSize(PtrTy->getPointeeType())); in generateDefaultMapInfo()
5504 CurSizes.push_back(CGF.getTypeSize(RI.getType())); in generateDefaultMapInfo()
5509 CurSizes.push_back(llvm::Constant::getNullValue(CGF.SizeTy)); in generateDefaultMapInfo()
5519 CurSizes.push_back(CGF.getTypeSize(ElementType)); in generateDefaultMapInfo()
5551 emitOffloadingArrays(CodeGenFunction &CGF, llvm::Value *&BasePointersArray, in emitOffloadingArrays() argument
5558 auto &CGM = CGF.CGM; in emitOffloadingArrays()
5559 auto &Ctx = CGF.getContext(); in emitOffloadingArrays()
5579 CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer(); in emitOffloadingArrays()
5581 CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer(); in emitOffloadingArrays()
5591 CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer(); in emitOffloadingArrays()
5612 llvm::ConstantDataArray::get(CGF.Builder.getContext(), MapTypes); in emitOffloadingArrays()
5623 BPVal = CGF.Builder.CreateBitCast(BPVal, CGM.VoidPtrTy); in emitOffloadingArrays()
5627 BPVal = CGF.Builder.CreateIntToPtr(BPVal, CGM.VoidPtrTy); in emitOffloadingArrays()
5629 llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
5633 CGF.Builder.CreateStore(BPVal, BPAddr); in emitOffloadingArrays()
5637 PVal = CGF.Builder.CreateBitCast(PVal, CGM.VoidPtrTy); in emitOffloadingArrays()
5641 PVal = CGF.Builder.CreateIntToPtr(PVal, CGM.VoidPtrTy); in emitOffloadingArrays()
5643 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
5647 CGF.Builder.CreateStore(PVal, PAddr); in emitOffloadingArrays()
5650 llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArrays()
5655 CGF.Builder.CreateStore( in emitOffloadingArrays()
5656 CGF.Builder.CreateIntCast(Sizes[i], CGM.SizeTy, /*isSigned=*/true), in emitOffloadingArrays()
5665 CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg, in emitOffloadingArraysArgument() argument
5670 auto &CGM = CGF.CGM; in emitOffloadingArraysArgument()
5672 BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
5675 PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
5679 SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
5682 MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( in emitOffloadingArraysArgument()
5695 void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF, in emitTargetCall() argument
5701 if (!CGF.HaveInsertPoint()) in emitTargetCall()
5706 auto &Ctx = CGF.getContext(); in emitTargetCall()
5721 MappableExprsHandler MEHandler(D, CGF); in emitTargetCall()
5742 CurSizes.push_back(CGF.getTypeSize(RI->getType())); in emitTargetCall()
5775 auto OffloadError = CGF.MakeAddrLValue( in emitTargetCall()
5776 CGF.CreateMemTemp(OffloadErrorQType, ".run_host_version"), in emitTargetCall()
5778 CGF.EmitStoreOfScalar(llvm::Constant::getNullValue(CGM.Int32Ty), in emitTargetCall()
5784 &D](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
5785 auto &RT = CGF.CGM.getOpenMPRuntime(); in emitTargetCall()
5791 emitOffloadingArrays(CGF, BasePointersArray, PointersArray, SizesArray, in emitTargetCall()
5794 emitOffloadingArraysArgument(CGF, BasePointersArray, PointersArray, in emitTargetCall()
5814 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetCall()
5815 CGF.Int32Ty, /*isSigned=*/true); in emitTargetCall()
5817 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF); in emitTargetCall()
5820 llvm::Value *PointerNum = CGF.Builder.getInt32(BasePointers.size()); in emitTargetCall()
5825 auto *NumTeams = emitNumTeamsClauseForTargetDirective(RT, CGF, D); in emitTargetCall()
5826 auto *ThreadLimit = emitThreadLimitClauseForTargetDirective(RT, CGF, D); in emitTargetCall()
5841 Return = CGF.EmitRuntimeCall( in emitTargetCall()
5847 Return = CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target), in emitTargetCall()
5851 CGF.EmitStoreOfScalar(Return, OffloadError); in emitTargetCall()
5855 auto &&ElseGen = [OffloadError](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetCall() argument
5856 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.Int32Ty, /*V=*/-1u), in emitTargetCall()
5866 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen); in emitTargetCall()
5869 ThenRCG(CGF); in emitTargetCall()
5873 ElseRCG(CGF); in emitTargetCall()
5877 auto OffloadFailedBlock = CGF.createBasicBlock("omp_offload.failed"); in emitTargetCall()
5878 auto OffloadContBlock = CGF.createBasicBlock("omp_offload.cont"); in emitTargetCall()
5879 auto OffloadErrorVal = CGF.EmitLoadOfScalar(OffloadError, SourceLocation()); in emitTargetCall()
5880 auto Failed = CGF.Builder.CreateIsNotNull(OffloadErrorVal); in emitTargetCall()
5881 CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock); in emitTargetCall()
5883 CGF.EmitBlock(OffloadFailedBlock); in emitTargetCall()
5884 CGF.Builder.CreateCall(OutlinedFn, KernelArgs); in emitTargetCall()
5885 CGF.EmitBranch(OffloadContBlock); in emitTargetCall()
5887 CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true); in emitTargetCall()
6006 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF, in emitTeamsCall() argument
6011 if (!CGF.HaveInsertPoint()) in emitTeamsCall()
6014 auto *RTLoc = emitUpdateLocation(CGF, Loc); in emitTeamsCall()
6015 CodeGenFunction::RunCleanupsScope Scope(CGF); in emitTeamsCall()
6020 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars in emitTeamsCall()
6021 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())}; in emitTeamsCall()
6027 CGF.EmitRuntimeCall(RTLFn, RealArgs); in emitTeamsCall()
6030 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF, in emitNumTeamsClause() argument
6034 if (!CGF.HaveInsertPoint()) in emitNumTeamsClause()
6037 auto *RTLoc = emitUpdateLocation(CGF, Loc); in emitNumTeamsClause()
6041 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams), in emitNumTeamsClause()
6042 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
6043 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
6047 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit), in emitNumTeamsClause()
6048 CGF.CGM.Int32Ty, /* isSigned = */ true) in emitNumTeamsClause()
6049 : CGF.Builder.getInt32(0); in emitNumTeamsClause()
6052 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal, in emitNumTeamsClause()
6054 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_teams), in emitNumTeamsClause()
6058 void CGOpenMPRuntime::emitTargetDataCalls(CodeGenFunction &CGF, in emitTargetDataCalls() argument
6064 if (!CGF.HaveInsertPoint()) in emitTargetDataCalls()
6078 &NumOfPtrs](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataCalls() argument
6086 MappableExprsHandler MCHandler(D, CGF); in emitTargetDataCalls()
6091 emitOffloadingArrays(CGF, BasePointersArray, PointersArray, SizesArray, in emitTargetDataCalls()
6099 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, in emitTargetDataCalls()
6107 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
6108 CGF.Int32Ty, /*isSigned=*/true); in emitTargetDataCalls()
6110 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
6113 auto *PointerNum = CGF.Builder.getInt32(NumOfPtrs); in emitTargetDataCalls()
6118 auto &RT = CGF.CGM.getOpenMPRuntime(); in emitTargetDataCalls()
6119 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target_data_begin), in emitTargetDataCalls()
6126 &NumOfPtrs](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataCalls() argument
6134 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, in emitTargetDataCalls()
6142 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataCalls()
6143 CGF.Int32Ty, /*isSigned=*/true); in emitTargetDataCalls()
6145 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF); in emitTargetDataCalls()
6148 auto *PointerNum = CGF.Builder.getInt32(NumOfPtrs); in emitTargetDataCalls()
6153 auto &RT = CGF.CGM.getOpenMPRuntime(); in emitTargetDataCalls()
6154 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__tgt_target_data_end), in emitTargetDataCalls()
6160 auto &&ElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {}; in emitTargetDataCalls() argument
6163 emitOMPIfClause(CGF, IfCond, BeginThenGen, ElseGen); in emitTargetDataCalls()
6166 BeginThenRCG(CGF); in emitTargetDataCalls()
6169 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, CodeGen); in emitTargetDataCalls()
6172 emitOMPIfClause(CGF, IfCond, EndThenGen, ElseGen); in emitTargetDataCalls()
6175 EndThenRCG(CGF); in emitTargetDataCalls()
6180 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, in emitTargetDataStandAloneCall() argument
6182 if (!CGF.HaveInsertPoint()) in emitTargetDataStandAloneCall()
6191 auto &&ThenGen = [this, &D, Device](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetDataStandAloneCall() argument
6199 MappableExprsHandler MEHandler(D, CGF); in emitTargetDataStandAloneCall()
6208 emitOffloadingArrays(CGF, BasePointersArrayArg, PointersArrayArg, in emitTargetDataStandAloneCall()
6212 CGF, BasePointersArrayArg, PointersArrayArg, SizesArrayArg, in emitTargetDataStandAloneCall()
6219 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), in emitTargetDataStandAloneCall()
6220 CGF.Int32Ty, /*isSigned=*/true); in emitTargetDataStandAloneCall()
6222 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF); in emitTargetDataStandAloneCall()
6225 auto *PointerNum = CGF.Builder.getInt32(BasePointers.size()); in emitTargetDataStandAloneCall()
6231 auto &RT = CGF.CGM.getOpenMPRuntime(); in emitTargetDataStandAloneCall()
6249 CGF.EmitRuntimeCall(RT.createRuntimeFunction(RTLFn), OffloadingArgs); in emitTargetDataStandAloneCall()
6254 auto &&ElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {}; in emitTargetDataStandAloneCall() argument
6257 emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen); in emitTargetDataStandAloneCall()
6260 ThenGenRCG(CGF); in emitTargetDataStandAloneCall()
6505 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { in Emit() argument
6506 if (!CGF.HaveInsertPoint()) in Emit()
6508 CGF.EmitRuntimeCall(RTLFn, Args); in Emit()
6513 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, in emitDoacrossInit() argument
6515 if (!CGF.HaveInsertPoint()) in emitDoacrossInit()
6537 Address DimsAddr = CGF.CreateMemTemp(KmpDimTy, "dims"); in emitDoacrossInit()
6538 CGF.EmitNullInitialization(DimsAddr, KmpDimTy); in emitDoacrossInit()
6541 LValue DimsLVal = CGF.MakeAddrLValue(DimsAddr, KmpDimTy); in emitDoacrossInit()
6544 CGF.EmitLValueForField(DimsLVal, *std::next(RD->field_begin(), UpperFD)); in emitDoacrossInit()
6545 llvm::Value *NumIterVal = CGF.EmitScalarConversion( in emitDoacrossInit()
6546 CGF.EmitScalarExpr(D.getNumIterations()), D.getNumIterations()->getType(), in emitDoacrossInit()
6548 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal); in emitDoacrossInit()
6551 CGF.EmitLValueForField(DimsLVal, *std::next(RD->field_begin(), StrideFD)); in emitDoacrossInit()
6552 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1), in emitDoacrossInit()
6557 llvm::Value *Args[] = {emitUpdateLocation(CGF, D.getLocStart()), in emitDoacrossInit()
6558 getThreadID(CGF, D.getLocStart()), in emitDoacrossInit()
6560 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in emitDoacrossInit()
6564 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossInit()
6566 emitUpdateLocation(CGF, D.getLocEnd()), getThreadID(CGF, D.getLocEnd())}; in emitDoacrossInit()
6568 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn, in emitDoacrossInit()
6572 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, in emitDoacrossOrdered() argument
6578 llvm::Value *CntVal = CGF.EmitScalarConversion(CGF.EmitScalarExpr(CounterVal), in emitDoacrossOrdered()
6581 Address CntAddr = CGF.CreateMemTemp(Int64Ty, ".cnt.addr"); in emitDoacrossOrdered()
6582 CGF.EmitStoreOfScalar(CntVal, CntAddr, /*Volatile=*/false, Int64Ty); in emitDoacrossOrdered()
6583 llvm::Value *Args[] = {emitUpdateLocation(CGF, C->getLocStart()), in emitDoacrossOrdered()
6584 getThreadID(CGF, C->getLocStart()), in emitDoacrossOrdered()
6593 CGF.EmitRuntimeCall(RTLFn, Args); in emitDoacrossOrdered()