Lines Matching refs:CGF
42 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitPreInitStmt() argument
49 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
52 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in emitPreInitStmt()
53 CGF.EmitAutoVarCleanups(Emission); in emitPreInitStmt()
62 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
63 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
64 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
65 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
66 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
71 CodeGenFunction &CGF, const OMPExecutableDirective &S, in OMPLexicalScope() argument
74 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPLexicalScope()
75 InlinedShareds(CGF) { in OMPLexicalScope()
77 emitPreInitStmt(CGF, S); in OMPLexicalScope()
89 CGF.getContext(), const_cast<VarDecl *>(VD), in OMPLexicalScope()
90 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo && in OMPLexicalScope()
93 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { in OMPLexicalScope()
94 return CGF.EmitLValue(&DRE).getAddress(CGF); in OMPLexicalScope()
113 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPParallelScope() argument
114 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, in OMPParallelScope()
128 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPTeamsScope() argument
129 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None, in OMPTeamsScope()
136 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) { in emitPreInitStmt() argument
143 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType())); in emitPreInitStmt()
151 CGF, OrigVD, in emitPreInitStmt()
153 CGF.ConvertTypeForMem(CGF.getContext().getPointerType( in emitPreInitStmt()
155 CGF.getContext().getDeclAlign(OrigVD))); in emitPreInitStmt()
159 (void)PreCondVars.apply(CGF); in emitPreInitStmt()
173 CGF.EmitStmt(Init); in emitPreInitStmt()
174 CGF.EmitStmt(CXXFor->getRangeStmt()); in emitPreInitStmt()
175 CGF.EmitStmt(CXXFor->getEndStmt()); in emitPreInitStmt()
181 CGF.EmitVarDecl(cast<VarDecl>(*I)); in emitPreInitStmt()
183 PreCondVars.restore(CGF); in emitPreInitStmt()
187 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S) in OMPLoopScope() argument
188 : CodeGenFunction::RunCleanupsScope(CGF) { in OMPLoopScope()
189 emitPreInitStmt(CGF, S); in OMPLoopScope()
196 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) { in isCapturedVar() argument
197 return CGF.LambdaCaptureFields.lookup(VD) || in isCapturedVar()
198 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) || in isCapturedVar()
199 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) && in isCapturedVar()
200 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD)); in isCapturedVar()
204 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S) in OMPSimdLexicalScope() argument
205 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()), in OMPSimdLexicalScope()
206 InlinedShareds(CGF) { in OMPSimdLexicalScope()
213 CGF.EmitVarDecl(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
216 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); in OMPSimdLexicalScope()
217 CGF.EmitAutoVarCleanups(Emission); in OMPSimdLexicalScope()
225 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
231 CGF.EmitVarDecl(*OED); in OMPSimdLexicalScope()
236 CGF.EmitOMPPrivateClause(S, InlinedShareds); in OMPSimdLexicalScope()
239 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl())); in OMPSimdLexicalScope()
248 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), in OMPSimdLexicalScope()
249 isCapturedVar(CGF, VD) || in OMPSimdLexicalScope()
250 (CGF.CapturedStmtInfo && in OMPSimdLexicalScope()
254 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address { in OMPSimdLexicalScope()
255 return CGF.EmitLValue(&DRE).getAddress(CGF); in OMPSimdLexicalScope()
267 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
353 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, in castValueFromUintptr() argument
356 ASTContext &Ctx = CGF.getContext(); in castValueFromUintptr()
358 llvm::Value *CastedPtr = CGF.EmitScalarConversion( in castValueFromUintptr()
359 AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(), in castValueFromUintptr()
362 CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType)) in castValueFromUintptr()
363 .getAddress(CGF); in castValueFromUintptr()
408 CodeGenFunction &CGF, FunctionArgList &Args, in emitOutlinedFunctionPrologue() argument
420 CodeGenModule &CGM = CGF.CGM; in emitOutlinedFunctionPrologue()
505 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs, in emitOutlinedFunctionPrologue()
515 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt], in emitOutlinedFunctionPrologue()
518 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]); in emitOutlinedFunctionPrologue()
531 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(), in emitOutlinedFunctionPrologue()
535 ArgLVal = CGF.MakeAddrLValue( in emitOutlinedFunctionPrologue()
536 castValueFromUintptr(CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
540 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
546 Address ArgAddr = ArgLVal.getAddress(CGF); in emitOutlinedFunctionPrologue()
548 ArgAddr = CGF.EmitLoadOfReference(ArgLVal); in emitOutlinedFunctionPrologue()
551 ArgAddr = CGF.EmitLoadOfPointer( in emitOutlinedFunctionPrologue()
566 CGF, I->getLocation(), FD->getType(), in emitOutlinedFunctionPrologue()
568 : ArgLVal.getAddress(CGF)}}); in emitOutlinedFunctionPrologue()
572 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation()); in emitOutlinedFunctionPrologue()
573 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}}); in emitOutlinedFunctionPrologue()
1216 [&Emission](CodeGenFunction &CGF) { in EmitOMPReductionClauseInit() argument
1217 CGF.EmitAutoVarInit(Emission); in EmitOMPReductionClauseInit()
1433 CodeGenFunction &CGF, const OMPExecutableDirective &D, in emitPostUpdateForReductionClause() argument
1435 if (!CGF.HaveInsertPoint()) in emitPostUpdateForReductionClause()
1441 if (llvm::Value *Cond = CondGen(CGF)) { in emitPostUpdateForReductionClause()
1444 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu"); in emitPostUpdateForReductionClause()
1445 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done"); in emitPostUpdateForReductionClause()
1446 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB); in emitPostUpdateForReductionClause()
1447 CGF.EmitBlock(ThenBB); in emitPostUpdateForReductionClause()
1450 CGF.EmitIgnoredExpr(PostUpdate); in emitPostUpdateForReductionClause()
1454 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); in emitPostUpdateForReductionClause()
1468 checkForLastprivateConditionalUpdate(CodeGenFunction &CGF, in checkForLastprivateConditionalUpdate() argument
1470 if (CGF.getLangOpts().OpenMP < 50) in checkForLastprivateConditionalUpdate()
1481 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1492 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1503 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref); in checkForLastprivateConditionalUpdate()
1520 CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional( in checkForLastprivateConditionalUpdate()
1521 CGF, S, PrivateDecls); in checkForLastprivateConditionalUpdate()
1525 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitCommonOMPParallelDirective() argument
1530 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction( in emitCommonOMPParallelDirective()
1533 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); in emitCommonOMPParallelDirective()
1535 CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(), in emitCommonOMPParallelDirective()
1537 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause( in emitCommonOMPParallelDirective()
1538 CGF, NumThreads, NumThreadsClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1541 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF); in emitCommonOMPParallelDirective()
1542 CGF.CGM.getOpenMPRuntime().emitProcBindClause( in emitCommonOMPParallelDirective()
1543 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc()); in emitCommonOMPParallelDirective()
1554 OMPParallelScope Scope(CGF, S); in emitCommonOMPParallelDirective()
1560 CodeGenBoundParameters(CGF, S, CapturedVars); in emitCommonOMPParallelDirective()
1561 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPParallelDirective()
1562 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn, in emitCommonOMPParallelDirective()
1582 CodeGenFunction &CGF, const VarDecl *VD) { in getAddressOfLocalVariable() argument
1583 CodeGenModule &CGM = CGF.CGM; in getAddressOfLocalVariable()
1594 Size = CGF.getTypeSize(CVD->getType()); in getAddressOfLocalVariable()
1596 Size = CGF.Builder.CreateNUWAdd( in getAddressOfLocalVariable()
1598 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
1599 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); in getAddressOfLocalVariable()
1608 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator()); in getAddressOfLocalVariable()
1612 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy); in getAddressOfLocalVariable()
1614 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator, in getAddressOfLocalVariable()
1618 CGF.Builder, Size, Allocator, in getAddressOfLocalVariable()
1621 OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator); in getAddressOfLocalVariable()
1623 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI); in getAddressOfLocalVariable()
1624 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( in getAddressOfLocalVariable()
1626 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())), in getAddressOfLocalVariable()
1632 CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, in getAddrOfThreadPrivate() argument
1634 CodeGenModule &CGM = CGF.CGM; in getAddrOfThreadPrivate()
1643 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy); in getAddrOfThreadPrivate()
1649 OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName); in getAddrOfThreadPrivate()
1727 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelDirective() argument
1728 Action.Enter(CGF); in EmitOMPParallelDirective()
1729 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelDirective()
1730 bool Copyins = CGF.EmitOMPCopyinClause(S); in EmitOMPParallelDirective()
1731 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1736 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitOMPParallelDirective()
1737 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitOMPParallelDirective()
1740 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelDirective()
1741 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelDirective()
1743 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt()); in EmitOMPParallelDirective()
1744 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelDirective()
1758 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, in emitBody() argument
1764 CGF.getContext().getSourceManager(), CS->getLBracLoc(), in emitBody()
1768 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange()); in emitBody()
1770 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level); in emitBody()
1780 CGF.EmitStmt(CXXFor->getLoopVarStmt()); in emitBody()
1786 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1); in emitBody()
1790 CGF.EmitStmt(S); in emitBody()
1996 static void emitAlignedClause(CodeGenFunction &CGF, in emitAlignedClause() argument
1998 if (!CGF.HaveInsertPoint()) in emitAlignedClause()
2004 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr)); in emitAlignedClause()
2014 CGF.getContext() in emitAlignedClause()
2015 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign( in emitAlignedClause()
2022 llvm::Value *PtrValue = CGF.EmitScalarExpr(E); in emitAlignedClause()
2023 CGF.emitAlignmentAssumption( in emitAlignedClause()
2025 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment)); in emitAlignedClause()
2081 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitPreCond() argument
2084 if (!CGF.HaveInsertPoint()) in emitPreCond()
2087 CodeGenFunction::OMPPrivateScope PreCondScope(CGF); in emitPreCond()
2088 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope); in emitPreCond()
2092 CGF.EmitIgnoredExpr(I); in emitPreCond()
2105 CGF.CreateMemTemp(VD->getType().getNonReferenceType()); in emitPreCond()
2106 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr); in emitPreCond()
2108 (void)PreCondVars.apply(CGF); in emitPreCond()
2112 CGF.EmitIgnoredExpr(E); in emitPreCond()
2115 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount); in emitPreCond()
2116 PreCondVars.restore(CGF); in emitPreCond()
2154 static void emitSimdlenSafelenClause(CodeGenFunction &CGF, in emitSimdlenSafelenClause() argument
2157 if (!CGF.HaveInsertPoint()) in emitSimdlenSafelenClause()
2160 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
2163 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
2168 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>()); in emitSimdlenSafelenClause()
2170 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(), in emitSimdlenSafelenClause()
2173 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue()); in emitSimdlenSafelenClause()
2177 CGF.LoopStack.setParallel(/*Enable=*/false); in emitSimdlenSafelenClause()
2247 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, in emitOMPLoopBodyWithStopPoint() argument
2250 CGF.EmitOMPLoopBody(S, LoopExit); in emitOMPLoopBodyWithStopPoint()
2251 CGF.EmitStopPoint(&S); in emitOMPLoopBodyWithStopPoint()
2255 static LValue EmitOMPHelperVar(CodeGenFunction &CGF, in EmitOMPHelperVar() argument
2258 CGF.EmitVarDecl(*VDecl); in EmitOMPHelperVar()
2259 return CGF.EmitLValue(Helper); in EmitOMPHelperVar()
2262 static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitCommonSimdLoop() argument
2265 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF, in emitCommonSimdLoop()
2267 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S); in emitCommonSimdLoop()
2268 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitCommonSimdLoop()
2269 SimdInitGen(CGF); in emitCommonSimdLoop()
2271 BodyCodeGen(CGF); in emitCommonSimdLoop()
2273 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonSimdLoop() argument
2274 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitCommonSimdLoop()
2275 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false); in emitCommonSimdLoop()
2277 BodyCodeGen(CGF); in emitCommonSimdLoop()
2282 if (CGF.getLangOpts().OpenMP >= 50 && in emitCommonSimdLoop()
2291 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen); in emitCommonSimdLoop()
2294 ThenRCG(CGF); in emitCommonSimdLoop()
2298 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, in emitOMPSimdRegion() argument
2300 Action.Enter(CGF); in emitOMPSimdRegion()
2303 OMPLoopScope PreInitScope(CGF, S); in emitOMPSimdRegion()
2312 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable())); in emitOMPSimdRegion()
2313 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable())); in emitOMPSimdRegion()
2321 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in emitOMPSimdRegion()
2325 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then"); in emitOMPSimdRegion()
2326 ContBlock = CGF.createBasicBlock("simd.if.end"); in emitOMPSimdRegion()
2327 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in emitOMPSimdRegion()
2328 CGF.getProfileCount(&S)); in emitOMPSimdRegion()
2329 CGF.EmitBlock(ThenBlock); in emitOMPSimdRegion()
2330 CGF.incrementProfileCounter(&S); in emitOMPSimdRegion()
2336 CGF.EmitVarDecl(*IVDecl); in emitOMPSimdRegion()
2337 CGF.EmitIgnoredExpr(S.getInit()); in emitOMPSimdRegion()
2343 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in emitOMPSimdRegion()
2345 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in emitOMPSimdRegion()
2348 emitAlignedClause(CGF, S); in emitOMPSimdRegion()
2349 (void)CGF.EmitOMPLinearClauseInit(S); in emitOMPSimdRegion()
2351 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in emitOMPSimdRegion()
2352 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in emitOMPSimdRegion()
2353 CGF.EmitOMPLinearClause(S, LoopScope); in emitOMPSimdRegion()
2354 CGF.EmitOMPPrivateClause(S, LoopScope); in emitOMPSimdRegion()
2355 CGF.EmitOMPReductionClauseInit(S, LoopScope); in emitOMPSimdRegion()
2357 CGF, S, CGF.EmitLValue(S.getIterationVariable())); in emitOMPSimdRegion()
2358 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in emitOMPSimdRegion()
2361 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitOMPSimdRegion()
2364 CGF, S, in emitOMPSimdRegion()
2365 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitOMPSimdRegion() argument
2366 CGF.EmitOMPSimdInit(S); in emitOMPSimdRegion()
2368 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in emitOMPSimdRegion() argument
2369 CGF.EmitOMPInnerLoop( in emitOMPSimdRegion()
2371 [&S](CodeGenFunction &CGF) { in emitOMPSimdRegion() argument
2372 emitOMPLoopBodyWithStopPoint(CGF, S, in emitOMPSimdRegion()
2377 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
2380 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true); in emitOMPSimdRegion()
2381 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd); in emitOMPSimdRegion()
2382 emitPostUpdateForReductionClause(CGF, S, in emitOMPSimdRegion()
2385 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; }); in emitOMPSimdRegion()
2388 CGF.EmitBranch(ContBlock); in emitOMPSimdRegion()
2389 CGF.EmitBlock(ContBlock, true); in emitOMPSimdRegion()
2396 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSimdDirective() argument
2397 emitOMPSimdRegion(CGF, S, Action); in EmitOMPSimdDirective()
2471 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPOuterLoop() argument
2475 CGF.LoopStack.setParallel(!IsMonotonic); in EmitOMPOuterLoop()
2478 CGF.LoopStack.setParallel(/*Enable=*/true); in EmitOMPOuterLoop()
2480 CGF.EmitOMPSimdInit(S, IsMonotonic); in EmitOMPOuterLoop()
2484 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPOuterLoop() argument
2491 CGF.EmitOMPInnerLoop( in EmitOMPOuterLoop()
2493 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2494 CodeGenLoop(CGF, S, LoopExit); in EmitOMPOuterLoop()
2496 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2497 CodeGenOrdered(CGF, Loc, IVSize, IVSigned); in EmitOMPOuterLoop()
2515 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) { in EmitOMPOuterLoop() argument
2517 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPOuterLoop()
2610 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc, in EmitOMPForOuterLoop()
2614 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize, in EmitOMPForOuterLoop()
2694 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, in emitDistributeParallelForInnerBounds() argument
2698 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitDistributeParallelForInnerBounds()
2700 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitDistributeParallelForInnerBounds()
2708 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable()); in emitDistributeParallelForInnerBounds()
2709 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable()); in emitDistributeParallelForInnerBounds()
2710 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
2712 PrevLBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
2716 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar( in emitDistributeParallelForInnerBounds()
2718 PrevUBVal = CGF.EmitScalarConversion( in emitDistributeParallelForInnerBounds()
2723 CGF.EmitStoreOfScalar(PrevLBVal, LB); in emitDistributeParallelForInnerBounds()
2724 CGF.EmitStoreOfScalar(PrevUBVal, UB); in emitDistributeParallelForInnerBounds()
2737 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, in emitDistributeParallelForDispatchBounds() argument
2748 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
2750 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc()); in emitDistributeParallelForDispatchBounds()
2755 CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDistributeParallelForDistributeInnerBoundParams() argument
2759 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
2761 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)), in emitDistributeParallelForDistributeInnerBoundParams()
2762 CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
2765 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable())); in emitDistributeParallelForDistributeInnerBoundParams()
2768 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)), in emitDistributeParallelForDistributeInnerBoundParams()
2769 CGF.SizeTy, /*isSigned=*/false); in emitDistributeParallelForDistributeInnerBoundParams()
2774 emitInnerParallelForWhenCombined(CodeGenFunction &CGF, in emitInnerParallelForWhenCombined() argument
2777 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF, in emitInnerParallelForWhenCombined()
2779 Action.Enter(CGF); in emitInnerParallelForWhenCombined()
2790 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitInnerParallelForWhenCombined()
2792 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(), in emitInnerParallelForWhenCombined()
2798 CGF, S, in emitInnerParallelForWhenCombined()
2806 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForDirective() argument
2807 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForDirective()
2816 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeParallelForSimdDirective() argument
2817 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPDistributeParallelForSimdDirective()
2826 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeSimdDirective() argument
2827 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeSimdDirective()
2836 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDeviceFunction() argument
2837 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDeviceFunction()
2849 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetSimdDirective() argument
2850 emitOMPSimdRegion(CGF, S, Action); in EmitOMPTargetSimdDirective()
3003 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPWorksharingLoop() argument
3005 CGF.EmitOMPSimdInit(S, IsMonotonic); in EmitOMPWorksharingLoop()
3008 CGF.LoopStack.setParallel(/*Enable=*/true); in EmitOMPWorksharingLoop()
3013 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPWorksharingLoop() argument
3020 IVSize, IVSigned, Ordered, IL.getAddress(CGF), in EmitOMPWorksharingLoop()
3021 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF), in EmitOMPWorksharingLoop()
3023 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitOMPWorksharingLoop()
3024 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, in EmitOMPWorksharingLoop()
3028 CGF.EmitIgnoredExpr(S.getEnsureUpperBound()); in EmitOMPWorksharingLoop()
3030 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPWorksharingLoop()
3044 CGF.EmitOMPInnerLoop( in EmitOMPWorksharingLoop()
3049 [&S, LoopExit](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3050 emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit); in EmitOMPWorksharingLoop()
3056 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3057 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitOMPWorksharingLoop()
3071 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3072 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3073 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3082 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3083 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3084 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3092 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPWorksharingLoop() argument
3093 return CGF.Builder.CreateIsNotNull( in EmitOMPWorksharingLoop()
3094 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPWorksharingLoop()
3110 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitForLoopBounds() argument
3113 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable())); in emitForLoopBounds()
3115 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable())); in emitForLoopBounds()
3124 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, in emitDispatchForLoopBounds() argument
3128 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType()); in emitDispatchForLoopBounds()
3129 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0); in emitDispatchForLoopBounds()
3130 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration()); in emitDispatchForLoopBounds()
3154 CodeGenFunction &CGF, const OMPLoopDirective &S, in emitScanBasedDirective() argument
3158 llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast( in emitScanBasedDirective()
3159 NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false); in emitScanBasedDirective()
3194 RedCG.emitSharedOrigLValue(CGF, Count); in emitScanBasedDirective()
3195 RedCG.emitAggregateType(CGF, Count); in emitScanBasedDirective()
3198 CGF, in emitScanBasedDirective()
3204 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl())); in emitScanBasedDirective()
3209 CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S); in emitScanBasedDirective()
3217 CGF.OMPFirstScanLoop = true; in emitScanBasedDirective()
3218 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitScanBasedDirective()
3219 FirstGen(CGF); in emitScanBasedDirective()
3223 llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock(); in emitScanBasedDirective()
3224 llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body"); in emitScanBasedDirective()
3225 llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit"); in emitScanBasedDirective()
3226 llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy); in emitScanBasedDirective()
3228 CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy); in emitScanBasedDirective()
3229 llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg); in emitScanBasedDirective()
3230 F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy); in emitScanBasedDirective()
3231 LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal); in emitScanBasedDirective()
3232 LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy); in emitScanBasedDirective()
3233 llvm::Value *NMin1 = CGF.Builder.CreateNUWSub( in emitScanBasedDirective()
3234 OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1)); in emitScanBasedDirective()
3235 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc()); in emitScanBasedDirective()
3236 CGF.EmitBlock(LoopBB); in emitScanBasedDirective()
3237 auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2); in emitScanBasedDirective()
3239 auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2); in emitScanBasedDirective()
3240 Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB); in emitScanBasedDirective()
3241 Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB); in emitScanBasedDirective()
3245 CGF.createBasicBlock("omp.inner.log.scan.body"); in emitScanBasedDirective()
3247 CGF.createBasicBlock("omp.inner.log.scan.exit"); in emitScanBasedDirective()
3248 llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K); in emitScanBasedDirective()
3249 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB); in emitScanBasedDirective()
3250 CGF.EmitBlock(InnerLoopBB); in emitScanBasedDirective()
3251 auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2); in emitScanBasedDirective()
3254 CodeGenFunction::OMPPrivateScope PrivScope(CGF); in emitScanBasedDirective()
3263 CGF, in emitScanBasedDirective()
3267 LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF); in emitScanBasedDirective()
3272 llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K); in emitScanBasedDirective()
3274 CGF, in emitScanBasedDirective()
3278 RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF); in emitScanBasedDirective()
3285 CGF.CGM.getOpenMPRuntime().emitReduction( in emitScanBasedDirective()
3286 CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps, in emitScanBasedDirective()
3290 CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1)); in emitScanBasedDirective()
3291 IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3292 CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K); in emitScanBasedDirective()
3293 CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB); in emitScanBasedDirective()
3294 CGF.EmitBlock(InnerExitBB); in emitScanBasedDirective()
3296 CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1)); in emitScanBasedDirective()
3297 Counter->addIncoming(Next, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3299 llvm::Value *NextPow2K = CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true); in emitScanBasedDirective()
3300 Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock()); in emitScanBasedDirective()
3301 llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal); in emitScanBasedDirective()
3302 CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB); in emitScanBasedDirective()
3303 auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc()); in emitScanBasedDirective()
3304 CGF.EmitBlock(ExitBB); in emitScanBasedDirective()
3306 CGF.OMPFirstScanLoop = false; in emitScanBasedDirective()
3307 SecondGen(CGF); in emitScanBasedDirective()
3310 static bool emitWorksharingDirective(CodeGenFunction &CGF, in emitWorksharingDirective() argument
3318 const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3319 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF); in emitWorksharingDirective()
3320 OMPLoopScope LoopScope(CGF, S); in emitWorksharingDirective()
3321 return CGF.EmitScalarExpr(S.getNumIterations()); in emitWorksharingDirective()
3323 const auto &&FirstGen = [&S, HasCancel](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3325 CGF, S.getDirectiveKind(), HasCancel); in emitWorksharingDirective()
3326 (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3330 CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(), in emitWorksharingDirective()
3334 &HasLastprivates](CodeGenFunction &CGF) { in emitWorksharingDirective() argument
3336 CGF, S.getDirectiveKind(), HasCancel); in emitWorksharingDirective()
3337 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3341 emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen); in emitWorksharingDirective()
3343 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(), in emitWorksharingDirective()
3345 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), in emitWorksharingDirective()
3354 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, in EmitOMPForDirective()
3356 HasLastprivates = emitWorksharingDirective(CGF, S, S.hasCancel()); in EmitOMPForDirective()
3375 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF, in EmitOMPForSimdDirective() argument
3377 HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPForSimdDirective()
3393 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, in createSectionLVal() argument
3396 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty); in createSectionLVal()
3398 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true); in createSectionLVal()
3407 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) { in EmitSections() argument
3408 const ASTContext &C = CGF.getContext(); in EmitSections()
3412 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.", in EmitSections()
3413 CGF.Builder.getInt32(0)); in EmitSections()
3415 ? CGF.Builder.getInt32(CS->size() - 1) in EmitSections()
3416 : CGF.Builder.getInt32(0); in EmitSections()
3418 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal); in EmitSections()
3419 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.", in EmitSections()
3420 CGF.Builder.getInt32(1)); in EmitSections()
3421 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.", in EmitSections()
3422 CGF.Builder.getInt32(0)); in EmitSections()
3424 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv."); in EmitSections()
3426 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV); in EmitSections()
3428 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB); in EmitSections()
3437 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) { in EmitSections() argument
3449 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit"); in EmitSections()
3451 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()), in EmitSections()
3456 auto CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
3457 CGF.EmitBlock(CaseBB); in EmitSections()
3458 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB); in EmitSections()
3459 CGF.EmitStmt(SubStmt); in EmitSections()
3460 CGF.EmitBranch(ExitBB); in EmitSections()
3464 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case"); in EmitSections()
3465 CGF.EmitBlock(CaseBB); in EmitSections()
3466 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB); in EmitSections()
3467 CGF.EmitStmt(CapturedStmt); in EmitSections()
3468 CGF.EmitBranch(ExitBB); in EmitSections()
3470 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); in EmitSections()
3473 CodeGenFunction::OMPPrivateScope LoopScope(CGF); in EmitSections()
3474 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) { in EmitSections()
3478 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitSections()
3479 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitSections()
3482 CGF.EmitOMPPrivateClause(S, LoopScope); in EmitSections()
3483 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV); in EmitSections()
3484 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitSections()
3485 CGF.EmitOMPReductionClauseInit(S, LoopScope); in EmitSections()
3488 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in EmitSections()
3494 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF), in EmitSections()
3495 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF)); in EmitSections()
3496 CGF.CGM.getOpenMPRuntime().emitForStaticInit( in EmitSections()
3497 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit); in EmitSections()
3499 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc()); in EmitSections()
3500 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect( in EmitSections()
3501 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal); in EmitSections()
3502 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB); in EmitSections()
3504 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV); in EmitSections()
3506 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen, in EmitSections()
3509 auto &&CodeGen = [&S](CodeGenFunction &CGF) { in EmitSections() argument
3510 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), in EmitSections()
3513 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen); in EmitSections()
3514 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitSections()
3516 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) { in EmitSections() argument
3517 return CGF.Builder.CreateIsNotNull( in EmitSections()
3518 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitSections()
3523 CGF.EmitOMPLastprivateClauseFinal( in EmitSections()
3525 CGF.Builder.CreateIsNotNull( in EmitSections()
3526 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()))); in EmitSections()
3588 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPSingleDirective() argument
3589 Action.Enter(CGF); in EmitOMPSingleDirective()
3590 OMPPrivateScope SingleScope(CGF); in EmitOMPSingleDirective()
3591 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope); in EmitOMPSingleDirective()
3592 CGF.EmitOMPPrivateClause(S, SingleScope); in EmitOMPSingleDirective()
3594 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPSingleDirective()
3615 static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) { in emitMaster() argument
3616 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitMaster() argument
3617 Action.Enter(CGF); in emitMaster()
3618 CGF.EmitStmt(S.getRawStmt()); in emitMaster()
3620 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc()); in emitMaster()
3692 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPCriticalDirective() argument
3693 Action.Enter(CGF); in EmitOMPCriticalDirective()
3694 CGF.EmitStmt(S.getAssociatedStmt()); in EmitOMPCriticalDirective()
3710 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForDirective() argument
3711 Action.Enter(CGF); in EmitOMPParallelForDirective()
3712 (void)emitWorksharingDirective(CGF, S, S.hasCancel()); in EmitOMPParallelForDirective()
3728 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelForSimdDirective() argument
3729 Action.Enter(CGF); in EmitOMPParallelForSimdDirective()
3730 (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false); in EmitOMPParallelForSimdDirective()
3746 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterDirective() argument
3747 Action.Enter(CGF); in EmitOMPParallelMasterDirective()
3748 OMPPrivateScope PrivateScope(CGF); in EmitOMPParallelMasterDirective()
3749 bool Copyins = CGF.EmitOMPCopyinClause(S); in EmitOMPParallelMasterDirective()
3750 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPParallelMasterDirective()
3755 CGF.CGM.getOpenMPRuntime().emitBarrierCall( in EmitOMPParallelMasterDirective()
3756 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, in EmitOMPParallelMasterDirective()
3759 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPParallelMasterDirective()
3760 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPParallelMasterDirective()
3762 emitMaster(CGF, S); in EmitOMPParallelMasterDirective()
3763 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in EmitOMPParallelMasterDirective()
3781 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelSectionsDirective() argument
3782 Action.Enter(CGF); in EmitOMPParallelSectionsDirective()
3783 CGF.EmitSections(S); in EmitOMPParallelSectionsDirective()
3939 CapturedRegion](CodeGenFunction &CGF, in EmitOMPTaskBasedDirective()
3944 OMPPrivateScope Scope(CGF); in EmitOMPTaskBasedDirective()
3949 CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true); in EmitOMPTaskBasedDirective()
3951 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
3952 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTaskBasedDirective()
3953 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTaskBasedDirective()
3961 Address PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
3962 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr"); in EmitOMPTaskBasedDirective()
3969 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
3978 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTaskBasedDirective()
3986 Ty = CGF.getContext().getPointerType(Ty); in EmitOMPTaskBasedDirective()
3988 Ty = CGF.getContext().getPointerType(Ty); in EmitOMPTaskBasedDirective()
3989 Address PrivatePtr = CGF.CreateMemTemp( in EmitOMPTaskBasedDirective()
3990 CGF.getContext().getPointerType(Ty), ".local.ptr.addr"); in EmitOMPTaskBasedDirective()
3994 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( in EmitOMPTaskBasedDirective()
3995 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); in EmitOMPTaskBasedDirective()
3998 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD), in EmitOMPTaskBasedDirective()
4000 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr, in EmitOMPTaskBasedDirective()
4003 Scope.addPrivate(Pair.first, [&CGF, &DRE]() { in EmitOMPTaskBasedDirective()
4004 return CGF.EmitLValue(&DRE).getAddress(CGF); in EmitOMPTaskBasedDirective()
4008 Address Replacement(CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
4009 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4016 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first); in EmitOMPTaskBasedDirective()
4017 Address Replacement(Ptr, CGF.getPointerAlign()); in EmitOMPTaskBasedDirective()
4019 Ptr = CGF.Builder.CreateLoad(Replacement); in EmitOMPTaskBasedDirective()
4020 Replacement = Address(Ptr, CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4023 llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first); in EmitOMPTaskBasedDirective()
4024 Address Replacement(Ptr, CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4030 OMPPrivateScope FirstprivateScope(CGF); in EmitOMPTaskBasedDirective()
4032 Address Replacement(CGF.Builder.CreateLoad(Pair.second), in EmitOMPTaskBasedDirective()
4033 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTaskBasedDirective()
4038 OMPLexicalScope LexScope(CGF, S, CapturedRegion); in EmitOMPTaskBasedDirective()
4041 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad( in EmitOMPTaskBasedDirective()
4042 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9))); in EmitOMPTaskBasedDirective()
4044 RedCG.emitSharedOrigLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
4045 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
4049 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
4051 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
4052 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
4054 Address(CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
4055 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
4056 CGF.getContext().getPointerType( in EmitOMPTaskBasedDirective()
4060 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
4087 OMPPrivateScope InRedScope(CGF); in EmitOMPTaskBasedDirective()
4091 RedCG.emitSharedOrigLValue(CGF, Cnt); in EmitOMPTaskBasedDirective()
4092 RedCG.emitAggregateType(CGF, Cnt); in EmitOMPTaskBasedDirective()
4098 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(), in EmitOMPTaskBasedDirective()
4102 ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr), in EmitOMPTaskBasedDirective()
4105 ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); in EmitOMPTaskBasedDirective()
4107 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem( in EmitOMPTaskBasedDirective()
4108 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt)); in EmitOMPTaskBasedDirective()
4110 CGF.EmitScalarConversion( in EmitOMPTaskBasedDirective()
4111 Replacement.getPointer(), CGF.getContext().VoidPtrTy, in EmitOMPTaskBasedDirective()
4112 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()), in EmitOMPTaskBasedDirective()
4115 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement); in EmitOMPTaskBasedDirective()
4122 CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF, in EmitOMPTaskBasedDirective()
4124 Action.Enter(CGF); in EmitOMPTaskBasedDirective()
4125 BodyGen(CGF); in EmitOMPTaskBasedDirective()
4237 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTaskBasedDirective() argument
4239 OMPPrivateScope Scope(CGF); in EmitOMPTargetTaskBasedDirective()
4242 CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true); in EmitOMPTargetTaskBasedDirective()
4244 llvm::Value *CopyFn = CGF.Builder.CreateLoad( in EmitOMPTargetTaskBasedDirective()
4245 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam))); in EmitOMPTargetTaskBasedDirective()
4246 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar( in EmitOMPTargetTaskBasedDirective()
4255 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()), in EmitOMPTargetTaskBasedDirective()
4260 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall( in EmitOMPTargetTaskBasedDirective()
4261 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs); in EmitOMPTargetTaskBasedDirective()
4263 Address Replacement(CGF.Builder.CreateLoad(Pair.second), in EmitOMPTargetTaskBasedDirective()
4264 CGF.getContext().getDeclAlign(Pair.first)); in EmitOMPTargetTaskBasedDirective()
4271 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
4272 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
4273 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
4274 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
4275 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
4276 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
4279 InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP( in EmitOMPTargetTaskBasedDirective()
4280 CGF.GetAddrOfLocalVar(MVD), /*Index=*/0); in EmitOMPTargetTaskBasedDirective()
4283 Action.Enter(CGF); in EmitOMPTargetTaskBasedDirective()
4284 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false); in EmitOMPTargetTaskBasedDirective()
4285 BodyGen(CGF); in EmitOMPTargetTaskBasedDirective()
4316 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskDirective() argument
4317 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPTaskDirective()
4320 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, in EmitOMPTaskDirective()
4322 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn, in EmitOMPTaskDirective()
4346 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTaskgroupDirective() argument
4347 Action.Enter(CGF); in EmitOMPTaskgroupDirective()
4362 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(), in EmitOMPTaskgroupDirective()
4365 CGF.EmitVarDecl(*VD); in EmitOMPTaskgroupDirective()
4366 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD), in EmitOMPTaskgroupDirective()
4369 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTaskgroupDirective()
4780 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeLoop() argument
4782 CGF.EmitOMPSimdInit(S, /*IsMonotonic=*/true); in EmitOMPDistributeLoop()
4785 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeLoop() argument
4786 CGF.EmitOMPInnerLoop( in EmitOMPDistributeLoop()
4788 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
4789 CodeGenLoop(CGF, S, LoopExit); in EmitOMPDistributeLoop()
4791 [&S, StaticChunked](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
4793 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound()); in EmitOMPDistributeLoop()
4794 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound()); in EmitOMPDistributeLoop()
4795 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound()); in EmitOMPDistributeLoop()
4796 CGF.EmitIgnoredExpr(S.getCombinedInit()); in EmitOMPDistributeLoop()
4813 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
4814 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
4815 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
4824 *this, S, [IL, &S](CodeGenFunction &CGF) { in EmitOMPDistributeLoop() argument
4825 return CGF.Builder.CreateIsNotNull( in EmitOMPDistributeLoop()
4826 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); in EmitOMPDistributeLoop()
4847 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPDistributeDirective() argument
4848 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPDistributeDirective()
4857 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true); in emitOutlinedOrderedFunction() local
4859 CGF.CapturedStmtInfo = &CapStmtInfo; in emitOutlinedOrderedFunction()
4860 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, Loc); in emitOutlinedOrderedFunction()
4874 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF, in EmitOMPOrderedDirective()
4879 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in EmitOMPOrderedDirective()
4882 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(), in EmitOMPOrderedDirective()
4885 Action.Enter(CGF); in EmitOMPOrderedDirective()
4886 CGF.EmitStmt(CS->getCapturedStmt()); in EmitOMPOrderedDirective()
4893 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val, in convertToScalarValue() argument
4896 assert(CGF.hasScalarEvaluationKind(DestType) && in convertToScalarValue()
4899 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, in convertToScalarValue()
4901 : CGF.EmitComplexToScalarConversion( in convertToScalarValue()
4906 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, in convertToComplexValue() argument
4908 assert(CGF.getEvaluationKind(DestType) == TEK_Complex && in convertToComplexValue()
4915 llvm::Value *ScalarVal = CGF.EmitScalarConversion( in convertToComplexValue()
4924 ComplexVal.first = CGF.EmitScalarConversion( in convertToComplexValue()
4926 ComplexVal.second = CGF.EmitScalarConversion( in convertToComplexValue()
4932 static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO, in emitSimpleAtomicStore() argument
4935 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal); in emitSimpleAtomicStore()
4937 CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false); in emitSimpleAtomicStore()
4940 static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF, in emitSimpleAtomicLoad() argument
4944 return CGF.EmitLoadOfLValue(LVal, Loc); in emitSimpleAtomicLoad()
4945 return CGF.EmitAtomicLoad( in emitSimpleAtomicLoad()
4968 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO, in emitOMPAtomicReadExpr() argument
4974 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicReadExpr()
4975 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicReadExpr()
4976 RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc); in emitOMPAtomicReadExpr()
4985 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicReadExpr()
4995 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc); in emitOMPAtomicReadExpr()
4996 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V); in emitOMPAtomicReadExpr()
4999 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, in emitOMPAtomicWriteExpr() argument
5004 emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E)); in emitOMPAtomicWriteExpr()
5005 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicWriteExpr()
5014 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicWriteExpr()
5026 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, in emitOMPAtomicRMW() argument
5031 ASTContext &Context = CGF.getContext(); in emitOMPAtomicRMW()
5039 X.getAddress(CGF).getElementType())) || in emitOMPAtomicRMW()
5040 !X.getAddress(CGF).getElementType()->isIntegerTy() || in emitOMPAtomicRMW()
5111 UpdateVal = CGF.Builder.CreateIntCast( in emitOMPAtomicRMW()
5112 IC, X.getAddress(CGF).getElementType(), in emitOMPAtomicRMW()
5116 CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(CGF), UpdateVal, AO); in emitOMPAtomicRMW()
5144 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, in emitOMPAtomicUpdateExpr() argument
5158 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicUpdateExpr()
5159 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicUpdateExpr()
5164 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) { in emitOMPAtomicUpdateExpr()
5165 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicUpdateExpr()
5166 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicUpdateExpr()
5167 return CGF.EmitAnyExpr(UE); in emitOMPAtomicUpdateExpr()
5169 (void)CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicUpdateExpr()
5171 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicUpdateExpr()
5180 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicUpdateExpr()
5192 static RValue convertToType(CodeGenFunction &CGF, RValue Value, in convertToType() argument
5195 switch (CGF.getEvaluationKind(ResType)) { in convertToType()
5198 convertToScalarValue(CGF, Value, SourceType, ResType, Loc)); in convertToType()
5200 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc); in convertToType()
5209 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, in emitOMPAtomicCaptureExpr() argument
5218 LValue VLValue = CGF.EmitLValue(V); in emitOMPAtomicCaptureExpr()
5219 LValue XLValue = CGF.EmitLValue(X); in emitOMPAtomicCaptureExpr()
5220 RValue ExprRValue = CGF.EmitAnyExpr(E); in emitOMPAtomicCaptureExpr()
5238 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr, in emitOMPAtomicCaptureExpr()
5240 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
5241 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue); in emitOMPAtomicCaptureExpr()
5242 RValue Res = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
5246 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
5248 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicCaptureExpr()
5257 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue); in emitOMPAtomicCaptureExpr()
5258 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second); in emitOMPAtomicCaptureExpr()
5259 NewVVal = CGF.EmitAnyExpr(UE); in emitOMPAtomicCaptureExpr()
5265 ExprRValue = convertToType(CGF, ExprRValue, E->getType(), in emitOMPAtomicCaptureExpr()
5272 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr( in emitOMPAtomicCaptureExpr()
5275 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X); in emitOMPAtomicCaptureExpr()
5282 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc); in emitOMPAtomicCaptureExpr()
5283 CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V); in emitOMPAtomicCaptureExpr()
5293 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicCaptureExpr()
5297 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicCaptureExpr()
5302 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc, in emitOMPAtomicCaptureExpr()
5313 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, in emitOMPAtomicExpr() argument
5320 emitOMPAtomicReadExpr(CGF, AO, X, V, Loc); in emitOMPAtomicExpr()
5323 emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc); in emitOMPAtomicExpr()
5327 emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc); in emitOMPAtomicExpr()
5330 emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE, in emitOMPAtomicExpr()
5464 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, in emitCommonOMPTargetDirective() argument
5468 CodeGenModule &CGM = CGF.CGM; in emitCommonOMPTargetDirective()
5472 OMPLexicalScope Scope(CGF, S, OMPD_target); in emitCommonOMPTargetDirective()
5474 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitCommonOMPTargetDirective() argument
5475 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in emitCommonOMPTargetDirective()
5481 CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S); in emitCommonOMPTargetDirective()
5507 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val) in emitCommonOMPTargetDirective()
5513 assert(CGF.CurFuncDecl && "No parent declaration for target region!"); in emitCommonOMPTargetDirective()
5517 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
5519 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl)) in emitCommonOMPTargetDirective()
5523 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl))); in emitCommonOMPTargetDirective()
5528 OMPLexicalScope Scope(CGF, S, OMPD_task); in emitCommonOMPTargetDirective()
5530 [IsOffloadEntry](CodeGenFunction &CGF, in emitCommonOMPTargetDirective()
5533 OMPLoopScope(CGF, D); in emitCommonOMPTargetDirective()
5535 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations()); in emitCommonOMPTargetDirective()
5536 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty, in emitCommonOMPTargetDirective()
5542 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device, in emitCommonOMPTargetDirective()
5546 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, in emitTargetRegion() argument
5548 Action.Enter(CGF); in emitTargetRegion()
5549 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetRegion()
5550 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetRegion()
5551 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetRegion()
5554 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetRegion()
5556 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt()); in emitTargetRegion()
5562 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDeviceFunction() argument
5563 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDeviceFunction()
5574 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDirective() argument
5575 emitTargetRegion(CGF, S, Action); in EmitOMPTargetDirective()
5580 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, in emitCommonOMPTeamsDirective() argument
5586 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction( in emitCommonOMPTeamsDirective()
5595 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit, in emitCommonOMPTeamsDirective()
5599 OMPTeamsScope Scope(CGF, S); in emitCommonOMPTeamsDirective()
5601 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars); in emitCommonOMPTeamsDirective()
5602 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn, in emitCommonOMPTeamsDirective()
5608 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTeamsDirective() argument
5609 Action.Enter(CGF); in EmitOMPTeamsDirective()
5610 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDirective()
5611 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
5612 CGF.EmitOMPPrivateClause(S, PrivateScope); in EmitOMPTeamsDirective()
5613 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDirective()
5615 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt()); in EmitOMPTeamsDirective()
5616 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDirective()
5623 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsRegion() argument
5626 Action.Enter(CGF); in emitTargetTeamsRegion()
5628 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetTeamsRegion() argument
5629 Action.Enter(CGF); in emitTargetTeamsRegion()
5630 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsRegion()
5631 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetTeamsRegion()
5632 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetTeamsRegion()
5633 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsRegion()
5636 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetTeamsRegion()
5637 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetTeamsRegion()
5638 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsRegion()
5640 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen); in emitTargetTeamsRegion()
5641 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsRegion()
5648 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDeviceFunction() argument
5649 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDeviceFunction()
5661 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDirective() argument
5662 emitTargetTeamsRegion(CGF, Action, S); in EmitOMPTargetTeamsDirective()
5668 emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeRegion() argument
5670 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
5671 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeRegion() argument
5672 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeRegion()
5676 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeRegion()
5678 Action.Enter(CGF); in emitTargetTeamsDistributeRegion()
5679 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeRegion()
5680 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeRegion()
5682 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeRegion()
5684 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeRegion()
5686 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen); in emitTargetTeamsDistributeRegion()
5687 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeRegion()
5694 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDeviceFunction() argument
5695 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDeviceFunction()
5707 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeDirective() argument
5708 emitTargetTeamsDistributeRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeDirective()
5714 CodeGenFunction &CGF, PrePostActionTy &Action, in emitTargetTeamsDistributeSimdRegion() argument
5716 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
5717 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeSimdRegion() argument
5718 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in emitTargetTeamsDistributeSimdRegion()
5722 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeSimdRegion()
5724 Action.Enter(CGF); in emitTargetTeamsDistributeSimdRegion()
5725 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeSimdRegion()
5726 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeSimdRegion()
5728 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in emitTargetTeamsDistributeSimdRegion()
5730 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeSimdRegion()
5732 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen); in emitTargetTeamsDistributeSimdRegion()
5733 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeSimdRegion()
5740 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDeviceFunction() argument
5741 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDeviceFunction()
5753 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeSimdDirective() argument
5754 emitTargetTeamsDistributeSimdRegion(CGF, Action, S); in EmitOMPTargetTeamsDistributeSimdDirective()
5762 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeDirective() argument
5763 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeDirective()
5767 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeDirective()
5769 Action.Enter(CGF); in EmitOMPTeamsDistributeDirective()
5770 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeDirective()
5771 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeDirective()
5773 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeDirective()
5775 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeDirective()
5784 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeSimdDirective() argument
5785 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc()); in EmitOMPTeamsDistributeSimdDirective()
5789 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeSimdDirective()
5791 Action.Enter(CGF); in EmitOMPTeamsDistributeSimdDirective()
5792 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeSimdDirective()
5793 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeSimdDirective()
5795 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd, in EmitOMPTeamsDistributeSimdDirective()
5797 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeSimdDirective()
5806 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForDirective() argument
5807 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForDirective()
5812 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForDirective()
5814 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForDirective()
5815 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForDirective()
5816 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForDirective()
5818 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute, in EmitOMPTeamsDistributeParallelForDirective()
5820 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForDirective()
5829 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTeamsDistributeParallelForSimdDirective() argument
5830 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in EmitOMPTeamsDistributeParallelForSimdDirective()
5835 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in EmitOMPTeamsDistributeParallelForSimdDirective()
5837 Action.Enter(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
5838 OMPPrivateScope PrivateScope(CGF); in EmitOMPTeamsDistributeParallelForSimdDirective()
5839 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in EmitOMPTeamsDistributeParallelForSimdDirective()
5841 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in EmitOMPTeamsDistributeParallelForSimdDirective()
5842 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in EmitOMPTeamsDistributeParallelForSimdDirective()
5843 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in EmitOMPTeamsDistributeParallelForSimdDirective()
5852 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, in emitTargetTeamsDistributeParallelForRegion() argument
5854 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
5855 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForRegion() argument
5856 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForRegion()
5861 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForRegion()
5863 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForRegion()
5864 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForRegion()
5865 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForRegion()
5867 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForRegion()
5868 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForRegion()
5869 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForRegion()
5872 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for, in emitTargetTeamsDistributeParallelForRegion()
5874 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForRegion()
5883 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDeviceFunction() argument
5884 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDeviceFunction()
5896 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForDirective() argument
5897 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForDirective()
5903 CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion() argument
5906 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
5907 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in emitTargetTeamsDistributeParallelForSimdRegion() argument
5908 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined, in emitTargetTeamsDistributeParallelForSimdRegion()
5913 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF, in emitTargetTeamsDistributeParallelForSimdRegion()
5915 Action.Enter(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
5916 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetTeamsDistributeParallelForSimdRegion()
5917 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetTeamsDistributeParallelForSimdRegion()
5919 CGF.CGM.getOpenMPRuntime().emitInlinedDirective( in emitTargetTeamsDistributeParallelForSimdRegion()
5920 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false); in emitTargetTeamsDistributeParallelForSimdRegion()
5921 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams); in emitTargetTeamsDistributeParallelForSimdRegion()
5924 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd, in emitTargetTeamsDistributeParallelForSimdRegion()
5926 emitPostUpdateForReductionClause(CGF, S, in emitTargetTeamsDistributeParallelForSimdRegion()
5935 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction() argument
5936 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction()
5948 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetTeamsDistributeParallelForSimdDirective() argument
5949 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetTeamsDistributeParallelForSimdDirective()
6135 void Enter(CodeGenFunction &CGF) override { in EmitOMPTargetDataDirective() argument
6142 CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetDataDirective() argument
6143 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTargetDataDirective() argument
6144 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt()); in EmitOMPTargetDataDirective()
6149 &InnermostCodeGen](CodeGenFunction &CGF, in EmitOMPTargetDataDirective()
6156 Action.Enter(CGF); in EmitOMPTargetDataDirective()
6159 OMPPrivateScope PrivateScope(CGF); in EmitOMPTargetDataDirective()
6162 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
6165 CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope, in EmitOMPTargetDataDirective()
6168 RCG(CGF); in EmitOMPTargetDataDirective()
6170 OMPLexicalScope Scope(CGF, S, OMPD_unknown); in EmitOMPTargetDataDirective()
6171 RCG(CGF); in EmitOMPTargetDataDirective()
6182 OMPLexicalScope Scope(CGF, S); in EmitOMPTargetDataDirective()
6183 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, in EmitOMPTargetDataDirective()
6256 static void emitTargetParallelRegion(CodeGenFunction &CGF, in emitTargetParallelRegion() argument
6261 Action.Enter(CGF); in emitTargetParallelRegion()
6262 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelRegion() argument
6263 Action.Enter(CGF); in emitTargetParallelRegion()
6264 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); in emitTargetParallelRegion()
6265 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope); in emitTargetParallelRegion()
6266 CGF.EmitOMPPrivateClause(S, PrivateScope); in emitTargetParallelRegion()
6267 CGF.EmitOMPReductionClauseInit(S, PrivateScope); in emitTargetParallelRegion()
6270 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S); in emitTargetParallelRegion()
6272 CGF.EmitStmt(CS->getCapturedStmt()); in emitTargetParallelRegion()
6273 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel); in emitTargetParallelRegion()
6275 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen, in emitTargetParallelRegion()
6277 emitPostUpdateForReductionClause(CGF, S, in emitTargetParallelRegion()
6284 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDeviceFunction() argument
6285 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDeviceFunction()
6297 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelDirective() argument
6298 emitTargetParallelRegion(CGF, S, Action); in EmitOMPTargetParallelDirective()
6303 static void emitTargetParallelForRegion(CodeGenFunction &CGF, in emitTargetParallelForRegion() argument
6306 Action.Enter(CGF); in emitTargetParallelForRegion()
6309 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForRegion() argument
6310 Action.Enter(CGF); in emitTargetParallelForRegion()
6312 CGF, OMPD_target_parallel_for, S.hasCancel()); in emitTargetParallelForRegion()
6313 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForRegion()
6316 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen, in emitTargetParallelForRegion()
6324 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDeviceFunction() argument
6325 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDeviceFunction()
6337 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForDirective() argument
6338 emitTargetParallelForRegion(CGF, S, Action); in EmitOMPTargetParallelForDirective()
6344 emitTargetParallelForSimdRegion(CodeGenFunction &CGF, in emitTargetParallelForSimdRegion() argument
6347 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
6350 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in emitTargetParallelForSimdRegion() argument
6351 Action.Enter(CGF); in emitTargetParallelForSimdRegion()
6352 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds, in emitTargetParallelForSimdRegion()
6355 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen, in emitTargetParallelForSimdRegion()
6363 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDeviceFunction() argument
6364 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDeviceFunction()
6376 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPTargetParallelForSimdDirective() argument
6377 emitTargetParallelForSimdRegion(CGF, S, Action); in EmitOMPTargetParallelForSimdDirective()
6383 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, in mapParam() argument
6388 [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); }); in mapParam()
6426 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
6438 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
6439 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) { in EmitOMPTaskLoopBasedDirective()
6443 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then"); in EmitOMPTaskLoopBasedDirective()
6444 ContBlock = CGF.createBasicBlock("taskloop.if.end"); in EmitOMPTaskLoopBasedDirective()
6445 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock, in EmitOMPTaskLoopBasedDirective()
6446 CGF.getProfileCount(&S)); in EmitOMPTaskLoopBasedDirective()
6447 CGF.EmitBlock(ThenBlock); in EmitOMPTaskLoopBasedDirective()
6448 CGF.incrementProfileCounter(&S); in EmitOMPTaskLoopBasedDirective()
6451 (void)CGF.EmitOMPLinearClauseInit(S); in EmitOMPTaskLoopBasedDirective()
6453 OMPPrivateScope LoopScope(CGF); in EmitOMPTaskLoopBasedDirective()
6461 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP, in EmitOMPTaskLoopBasedDirective()
6463 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP, in EmitOMPTaskLoopBasedDirective()
6465 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope); in EmitOMPTaskLoopBasedDirective()
6466 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP, in EmitOMPTaskLoopBasedDirective()
6468 CGF.EmitOMPPrivateLoopCounters(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
6469 CGF.EmitOMPLinearClause(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
6470 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope); in EmitOMPTaskLoopBasedDirective()
6475 CGF.EmitVarDecl(*IVDecl); in EmitOMPTaskLoopBasedDirective()
6476 CGF.EmitIgnoredExpr(S.getInit()); in EmitOMPTaskLoopBasedDirective()
6482 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl())); in EmitOMPTaskLoopBasedDirective()
6484 CGF.EmitIgnoredExpr(S.getCalcLastIteration()); in EmitOMPTaskLoopBasedDirective()
6488 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false); in EmitOMPTaskLoopBasedDirective()
6490 CGF, S, in EmitOMPTaskLoopBasedDirective()
6491 [&S](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
6493 CGF.EmitOMPSimdInit(S); in EmitOMPTaskLoopBasedDirective()
6495 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
6496 CGF.EmitOMPInnerLoop( in EmitOMPTaskLoopBasedDirective()
6498 [&S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
6499 emitOMPLoopBodyWithStopPoint(CGF, S, in EmitOMPTaskLoopBasedDirective()
6507 CGF.EmitBranch(ContBlock); in EmitOMPTaskLoopBasedDirective()
6508 CGF.EmitBlock(ContBlock, true); in EmitOMPTaskLoopBasedDirective()
6512 CGF.EmitOMPLastprivateClauseFinal( in EmitOMPTaskLoopBasedDirective()
6514 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar( in EmitOMPTaskLoopBasedDirective()
6515 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
6518 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) { in EmitOMPTaskLoopBasedDirective() argument
6519 return CGF.Builder.CreateIsNotNull( in EmitOMPTaskLoopBasedDirective()
6520 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false, in EmitOMPTaskLoopBasedDirective()
6525 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn, in EmitOMPTaskLoopBasedDirective()
6528 &Data](CodeGenFunction &CGF, PrePostActionTy &) { in EmitOMPTaskLoopBasedDirective() argument
6529 OMPLoopScope PreInitScope(CGF, S); in EmitOMPTaskLoopBasedDirective()
6530 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S, in EmitOMPTaskLoopBasedDirective()
6534 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop, in EmitOMPTaskLoopBasedDirective()
6542 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF, in EmitOMPTaskLoopBasedDirective()
6544 Action.Enter(CGF); in EmitOMPTaskLoopBasedDirective()
6545 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, in EmitOMPTaskLoopBasedDirective()
6568 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterTaskLoopDirective() argument
6569 Action.Enter(CGF); in EmitOMPMasterTaskLoopDirective()
6580 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPMasterTaskLoopSimdDirective() argument
6581 Action.Enter(CGF); in EmitOMPMasterTaskLoopSimdDirective()
6592 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterTaskLoopDirective() argument
6593 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, in EmitOMPParallelMasterTaskLoopDirective()
6595 Action.Enter(CGF); in EmitOMPParallelMasterTaskLoopDirective()
6596 CGF.EmitOMPTaskLoopBasedDirective(S); in EmitOMPParallelMasterTaskLoopDirective()
6598 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); in EmitOMPParallelMasterTaskLoopDirective()
6599 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, in EmitOMPParallelMasterTaskLoopDirective()
6610 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitOMPParallelMasterTaskLoopSimdDirective() argument
6611 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF, in EmitOMPParallelMasterTaskLoopSimdDirective()
6613 Action.Enter(CGF); in EmitOMPParallelMasterTaskLoopSimdDirective()
6614 CGF.EmitOMPTaskLoopBasedDirective(S); in EmitOMPParallelMasterTaskLoopSimdDirective()
6616 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false); in EmitOMPParallelMasterTaskLoopSimdDirective()
6617 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen, in EmitOMPParallelMasterTaskLoopSimdDirective()
6656 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) { in EmitSimpleOMPExecutableDirective() argument
6657 OMPPrivateScope GlobalsScope(CGF); in EmitSimpleOMPExecutableDirective()
6668 if (!CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
6669 LValue GlobLVal = CGF.EmitLValue(Ref); in EmitSimpleOMPExecutableDirective()
6671 VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); }); in EmitSimpleOMPExecutableDirective()
6678 ParentLoopDirectiveForScanRegion ScanRegion(CGF, D); in EmitSimpleOMPExecutableDirective()
6679 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action); in EmitSimpleOMPExecutableDirective()
6684 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) { in EmitSimpleOMPExecutableDirective()
6685 LValue GlobLVal = CGF.EmitLValue(E); in EmitSimpleOMPExecutableDirective()
6687 VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); }); in EmitSimpleOMPExecutableDirective()
6691 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
6692 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
6704 if (!CGF.LocalDeclMap.count(VD)) in EmitSimpleOMPExecutableDirective()
6705 CGF.EmitVarDecl(*VD); in EmitSimpleOMPExecutableDirective()
6711 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt()); in EmitSimpleOMPExecutableDirective()