24 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" 25 using namespace clang;
26 using namespace CodeGen;
34 for (
const auto *C : S.
clauses()) {
36 if (
const auto *PreInit =
37 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
38 for (
const auto *I : PreInit->decls()) {
39 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
63 const bool EmitPreInitStmt =
true)
67 emitPreInitStmt(CGF, S);
68 if (!CapturedRegion.hasValue())
71 "Expected associated statement for inlined directive.");
73 for (
const auto &C : CS->captures()) {
74 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
75 auto *VD =
C.getCapturedVar();
77 "Canonical decl must be captured.");
94 class OMPParallelScope final :
public OMPLexicalScope {
104 : OMPLexicalScope(CGF, S,
llvm::
None,
105 EmitPreInitStmt(S)) {}
110 class OMPTeamsScope final :
public OMPLexicalScope {
119 : OMPLexicalScope(CGF, S,
llvm::
None,
120 EmitPreInitStmt(S)) {}
129 for (
const auto *E : S.
counters()) {
130 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
137 for (
const Expr *IRef :
C->varlists()) {
138 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
139 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
144 OrigVD->getType().getNonReferenceType()))),
149 (void)PreCondVars.
apply(CGF);
156 if (
auto *For = dyn_cast<ForStmt>(Body)) {
157 Body = For->getBody();
159 assert(isa<CXXForRangeStmt>(Body) &&
160 "Expected canonical for loop or range-based for loop.");
161 auto *CXXFor = cast<CXXForRangeStmt>(Body);
162 if (
const Stmt *Init = CXXFor->getInit())
164 CGF.
EmitStmt(CXXFor->getRangeStmt());
166 Body = CXXFor->getBody();
169 if (
const auto *PreInits = cast_or_null<DeclStmt>(S.
getPreInits())) {
170 for (
const auto *I : PreInits->decls())
179 emitPreInitStmt(CGF, S);
190 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
196 InlinedShareds(CGF) {
197 for (
const auto *C : S.
clauses()) {
199 if (
const auto *PreInit =
200 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
201 for (
const auto *I : PreInit->decls()) {
202 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
211 }
else if (
const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
212 for (
const Expr *E : UDP->varlists()) {
213 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
214 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
221 if (
const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
222 if (
const Expr *E = TG->getReductionRef())
223 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
227 for (
auto &C : CS->captures()) {
228 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
229 auto *VD =
C.getCapturedVar();
231 "Canonical decl must be captured.");
233 isCapturedVar(CGF, VD) ||
256 if (
const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
257 if (
const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
258 OrigVD = OrigVD->getCanonicalDecl();
260 LambdaCaptureFields.lookup(OrigVD) ||
261 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
262 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
263 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
264 OrigDRE->getType(),
VK_LValue, OrigDRE->getExprLoc());
265 return EmitLValue(&DRE);
268 return EmitLValue(E);
275 if (SizeInChars.isZero()) {
280 Size = Size ? Builder.CreateNUWMul(Size, VlaSize.
NumElts)
284 if (SizeInChars.isZero())
285 return llvm::ConstantInt::get(SizeTy, 0);
286 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
288 return CGM.getSize(SizeInChars);
298 I != E; ++I, ++CurField, ++CurCap) {
299 if (CurField->hasCapturedVLAType()) {
302 CapturedVars.push_back(Val);
303 }
else if (CurCap->capturesThis()) {
304 CapturedVars.push_back(CXXThisValue);
305 }
else if (CurCap->capturesVariableByCopy()) {
306 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
310 if (!CurField->getType()->isAnyPointerType()) {
312 Address DstAddr = CreateMemTemp(
314 Twine(CurCap->getCapturedVar()->getName(),
".casted"));
321 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
327 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
329 CapturedVars.push_back(CV);
331 assert(CurCap->capturesVariable() &&
"Expected capture by reference.");
332 CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer());
359 if (
const auto *VLA = dyn_cast<VariableArrayType>(A))
361 if (!A->isVariablyModifiedType())
369 struct FunctionOptions {
374 const bool UIntPtrCastRequired =
true;
377 const bool RegisterCastedArgsOnly =
false;
379 const StringRef FunctionName;
380 explicit FunctionOptions(
const CapturedStmt *S,
bool UIntPtrCastRequired,
381 bool RegisterCastedArgsOnly,
382 StringRef FunctionName)
383 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
384 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
385 FunctionName(FunctionName) {}
391 llvm::MapVector<
const Decl *, std::pair<const VarDecl *, Address>>
393 llvm::DenseMap<
const Decl *, std::pair<const Expr *, llvm::Value *>>
395 llvm::Value *&CXXThisValue,
const FunctionOptions &FO) {
397 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
398 assert(CD->
hasBody() &&
"missing CapturedDecl body");
400 CXXThisValue =
nullptr;
410 auto I = FO.S->captures().begin();
412 if (!FO.UIntPtrCastRequired) {
431 if (FO.UIntPtrCastRequired &&
433 I->capturesVariableArrayType()))
436 if (I->capturesVariable() || I->capturesVariableByCopy()) {
437 CapVar = I->getCapturedVar();
439 }
else if (I->capturesThis()) {
442 assert(I->capturesVariableArrayType());
448 if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
450 Ctx, DebugFunctionDecl,
451 CapVar ? CapVar->
getBeginLoc() : FD->getBeginLoc(),
452 CapVar ? CapVar->
getLocation() : FD->getLocation(), II, ArgType,
458 Args.emplace_back(Arg);
460 TargetArgs.emplace_back(
461 FO.UIntPtrCastRequired
483 F->setDoesNotThrow();
484 F->setDoesNotRecurse();
490 I = FO.S->captures().begin();
494 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
502 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
503 const VarDecl *CurVD = I->getCapturedVar();
504 if (!FO.RegisterCastedArgsOnly)
505 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
513 if (FD->hasCapturedVLAType()) {
514 if (FO.UIntPtrCastRequired) {
517 Args[Cnt]->getName(), ArgLVal),
522 VLASizes.try_emplace(Args[Cnt], VAT->
getSizeExpr(), ExprArg);
523 }
else if (I->capturesVariable()) {
524 const VarDecl *Var = I->getCapturedVar();
534 if (!FO.RegisterCastedArgsOnly) {
539 }
else if (I->capturesVariableByCopy()) {
540 assert(!FD->getType()->isAnyPointerType() &&
541 "Not expecting a captured pointer.");
542 const VarDecl *Var = I->getCapturedVar();
543 LocalAddrs.insert({Args[Cnt],
544 {Var, FO.UIntPtrCastRequired
546 CGF, I->getLocation(), FD->getType(),
547 Args[Cnt]->getName(), ArgLVal)
551 assert(I->capturesThis());
553 LocalAddrs.insert({Args[Cnt], {
nullptr, ArgLVal.
getAddress(CGF)}});
566 "CapturedStmtInfo should be set when generating the captured function");
569 bool NeedWrapperFunction =
570 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
572 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
573 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
575 llvm::raw_svector_ostream Out(Buffer);
576 Out << CapturedStmtInfo->getHelperName();
577 if (NeedWrapperFunction)
579 FunctionOptions FO(&S, !NeedWrapperFunction,
false,
582 VLASizes, CXXThisValue, FO);
584 for (
const auto &LocalAddrPair : LocalAddrs) {
585 if (LocalAddrPair.second.first) {
586 LocalScope.
addPrivate(LocalAddrPair.second.first, [&LocalAddrPair]() {
587 return LocalAddrPair.second.second;
592 for (
const auto &VLASizePair : VLASizes)
593 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
595 CapturedStmtInfo->EmitBody(*
this, CD->
getBody());
598 if (!NeedWrapperFunction)
601 FunctionOptions WrapperFO(&S,
true,
603 CapturedStmtInfo->getHelperName());
605 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
609 llvm::Function *WrapperF =
611 WrapperCGF.CXXThisValue, WrapperFO);
613 for (
const auto *Arg : Args) {
615 auto I = LocalAddrs.find(Arg);
616 if (I != LocalAddrs.end()) {
617 LValue LV = WrapperCGF.MakeAddrLValue(
619 I->second.first ? I->second.first->getType() : Arg->getType(),
621 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.
getBeginLoc());
623 auto EI = VLASizes.find(Arg);
624 if (EI != VLASizes.end()) {
625 CallArg = EI->second.second;
627 LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
630 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.
getBeginLoc());
633 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
635 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.
getBeginLoc(),
637 WrapperCGF.FinishFunction();
652 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
653 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.
getElementType());
658 llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
660 llvm::BasicBlock *BodyBB = createBasicBlock(
"omp.arraycpy.body");
661 llvm::BasicBlock *DoneBB = createBasicBlock(
"omp.arraycpy.done");
663 Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arraycpy.isempty");
664 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
667 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
670 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
672 llvm::PHINode *SrcElementPHI =
673 Builder.CreatePHI(SrcBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
674 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
679 llvm::PHINode *DestElementPHI =
680 Builder.CreatePHI(DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
681 DestElementPHI->addIncoming(DestBegin, EntryBB);
687 CopyGen(DestElementCurrent, SrcElementCurrent);
690 llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
691 DestElementPHI, 1,
"omp.arraycpy.dest.element");
692 llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
693 SrcElementPHI, 1,
"omp.arraycpy.src.element");
696 Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
697 Builder.CreateCondBr(Done, DoneBB, BodyBB);
698 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
699 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
702 EmitBlock(DoneBB,
true);
710 if (BO && BO->getOpcode() == BO_Assign) {
712 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
713 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
714 EmitAggregateAssign(Dest, Src, OriginalType);
718 EmitOMPAggregateAssign(
719 DestAddr, SrcAddr, OriginalType,
720 [
this, Copy, SrcVD, DestVD](
Address DestElement,
Address SrcElement) {
725 Remap.
addPrivate(DestVD, [DestElement]() {
return DestElement; });
726 Remap.
addPrivate(SrcVD, [SrcElement]() {
return SrcElement; });
728 EmitIgnoredExpr(Copy);
734 Remap.
addPrivate(SrcVD, [SrcAddr]() {
return SrcAddr; });
735 Remap.
addPrivate(DestVD, [DestAddr]() {
return DestAddr; });
738 EmitIgnoredExpr(Copy);
744 if (!HaveInsertPoint())
746 bool DeviceConstTarget =
747 getLangOpts().OpenMPIsDevice &&
749 bool FirstprivateIsLastprivate =
false;
752 for (
const auto *D : C->varlists())
761 bool MustEmitFirstprivateCopy =
762 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
764 auto IRef = C->varlist_begin();
765 auto InitsRef = C->inits().begin();
766 for (
const Expr *IInit : C->private_copies()) {
767 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
768 bool ThisFirstprivateIsLastprivate =
769 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
770 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
771 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
772 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
774 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
775 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
782 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
784 (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())) {
785 (void)CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(*
this,
791 FirstprivateIsLastprivate =
792 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
793 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
795 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
797 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
813 assert(!CE &&
"Expected non-constant firstprivate.");
814 OriginalLVal = EmitLValue(&DRE);
817 OriginalLVal = EmitLValue(&DRE);
825 OrigVD, [
this, VD, Type, OriginalLVal, VDInit]() {
828 if (!isa<CXXConstructExpr>(Init) ||
829 isTrivialInitializer(Init)) {
833 EmitAggregateAssign(Dest, OriginalLVal, Type);
835 EmitOMPAggregateAssign(
837 OriginalLVal.getAddress(*
this),
Type,
838 [
this, VDInit, Init](
Address DestElement,
844 setAddrOfLocalVar(VDInit, SrcElement);
845 EmitAnyExprToMem(Init, DestElement,
848 LocalDeclMap.erase(VDInit);
851 EmitAutoVarCleanups(Emission);
855 Address OriginalAddr = OriginalLVal.getAddress(*
this);
857 OrigVD, [
this, VDInit, OriginalAddr, VD]() {
861 setAddrOfLocalVar(VDInit, OriginalAddr);
863 LocalDeclMap.erase(VDInit);
864 return GetAddrOfLocalVar(VD);
867 assert(IsRegistered &&
868 "firstprivate var already registered as private");
876 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
882 if (!HaveInsertPoint())
886 auto IRef = C->varlist_begin();
887 for (
const Expr *IInit : C->private_copies()) {
888 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
889 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
890 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
891 bool IsRegistered = PrivateScope.
addPrivate(OrigVD, [
this, VD]() {
894 return GetAddrOfLocalVar(VD);
896 assert(IsRegistered &&
"private var already registered as private");
906 if (!HaveInsertPoint())
913 llvm::BasicBlock *CopyBegin =
nullptr, *CopyEnd =
nullptr;
915 auto IRef = C->varlist_begin();
916 auto ISrcRef = C->source_exprs().begin();
917 auto IDestRef = C->destination_exprs().begin();
918 for (
const Expr *AssignOp : C->assignment_ops()) {
919 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
926 if (getLangOpts().OpenMPUseTLS &&
927 getContext().getTargetInfo().isTLSSupported()) {
928 assert(CapturedStmtInfo->lookup(VD) &&
929 "Copyin threadprivates should have been captured!");
930 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
true,
931 (*IRef)->getType(),
VK_LValue, (*IRef)->getExprLoc());
932 MasterAddr = EmitLValue(&DRE).getAddress(*
this);
933 LocalDeclMap.erase(VD);
937 : CGM.GetAddrOfGlobal(VD),
938 getContext().getDeclAlign(VD));
941 Address PrivateAddr = EmitLValue(*IRef).getAddress(*
this);
942 if (CopiedVars.size() == 1) {
945 CopyBegin = createBasicBlock(
"copyin.not.master");
946 CopyEnd = createBasicBlock(
"copyin.not.master.end");
947 Builder.CreateCondBr(
948 Builder.CreateICmpNE(
949 Builder.CreatePtrToInt(MasterAddr.
getPointer(), CGM.IntPtrTy),
950 Builder.CreatePtrToInt(PrivateAddr.getPointer(),
953 EmitBlock(CopyBegin);
956 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
958 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
959 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
968 EmitBlock(CopyEnd,
true);
976 if (!HaveInsertPoint())
978 bool HasAtLeastOneLastprivate =
false;
981 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
982 for (
const Expr *C : LoopDirective->counters()) {
989 HasAtLeastOneLastprivate =
true;
991 !getLangOpts().OpenMPSimd)
993 auto IRef = C->varlist_begin();
994 auto IDestRef = C->destination_exprs().begin();
995 for (
const Expr *IInit : C->private_copies()) {
998 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1001 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1002 const auto *DestVD =
1003 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1004 PrivateScope.
addPrivate(DestVD, [
this, OrigVD, IRef]() {
1005 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1007 CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
1008 (*IRef)->getType(),
VK_LValue, (*IRef)->getExprLoc());
1009 return EmitLValue(&DRE).getAddress(*
this);
1014 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1015 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1016 bool IsRegistered = PrivateScope.
addPrivate(OrigVD, [
this, VD]() {
1019 return GetAddrOfLocalVar(VD);
1021 assert(IsRegistered &&
1022 "lastprivate var already registered as private");
1030 return HasAtLeastOneLastprivate;
1036 if (!HaveInsertPoint())
1044 llvm::BasicBlock *ThenBB =
nullptr;
1045 llvm::BasicBlock *DoneBB =
nullptr;
1046 if (IsLastIterCond) {
1049 if (!getLangOpts().OpenMPSimd &&
1052 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1054 CGM.getOpenMPRuntime().emitBarrierCall(*
this, D.
getBeginLoc(),
1059 ThenBB = createBasicBlock(
".omp.lastprivate.then");
1060 DoneBB = createBasicBlock(
".omp.lastprivate.done");
1061 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1065 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1066 if (
const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1067 auto IC = LoopDirective->counters().begin();
1068 for (
const Expr *F : LoopDirective->finals()) {
1072 AlreadyEmittedVars.insert(D);
1074 LoopCountersAndUpdates[D] = F;
1079 auto IRef = C->varlist_begin();
1080 auto ISrcRef = C->source_exprs().begin();
1081 auto IDestRef = C->destination_exprs().begin();
1082 for (
const Expr *AssignOp : C->assignment_ops()) {
1083 const auto *PrivateVD =
1084 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1086 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1087 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1091 if (
const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1092 EmitIgnoredExpr(FinalExpr);
1094 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1095 const auto *DestVD =
1096 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1098 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1099 if (
const auto *RefTy = PrivateVD->getType()->getAs<
ReferenceType>())
1101 Address(Builder.CreateLoad(PrivateAddr),
1102 getNaturalTypeAlignment(RefTy->getPointeeType()));
1104 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1105 CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
1106 *
this, MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1107 (*IRef)->getExprLoc());
1109 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1110 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1116 if (
const Expr *PostUpdate = C->getPostUpdateExpr())
1117 EmitIgnoredExpr(PostUpdate);
1120 EmitBlock(DoneBB,
true);
1126 if (!HaveInsertPoint())
1134 auto IPriv = C->privates().begin();
1135 auto IRed = C->reduction_ops().begin();
1136 auto ILHS = C->lhs_exprs().begin();
1137 auto IRHS = C->rhs_exprs().begin();
1138 for (
const Expr *Ref : C->varlists()) {
1139 Shareds.emplace_back(Ref);
1140 Privates.emplace_back(*IPriv);
1141 ReductionOps.emplace_back(*IRed);
1142 LHSs.emplace_back(*ILHS);
1143 RHSs.emplace_back(*IRHS);
1144 std::advance(IPriv, 1);
1145 std::advance(IRed, 1);
1146 std::advance(ILHS, 1);
1147 std::advance(IRHS, 1);
1152 auto ILHS = LHSs.begin();
1153 auto IRHS = RHSs.begin();
1154 auto IPriv = Privates.begin();
1155 for (
const Expr *IRef : Shareds) {
1156 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1158 RedCG.emitSharedLValue(*
this, Count);
1167 EmitAutoVarCleanups(Emission);
1171 RedCG.
getBaseDecl(Count), [BaseAddr]() {
return BaseAddr; });
1172 assert(IsRegistered &&
"private var already registered as private");
1176 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1177 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1179 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1180 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1183 PrivateScope.
addPrivate(LHSVD, [&RedCG, Count,
this]() {
1187 RHSVD, [
this, PrivateVD]() {
return GetAddrOfLocalVar(PrivateVD); });
1188 }
else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1189 isa<ArraySubscriptExpr>(IRef)) {
1192 PrivateScope.
addPrivate(LHSVD, [&RedCG, Count,
this]() {
1195 PrivateScope.
addPrivate(RHSVD, [
this, PrivateVD, RHSVD]() {
1196 return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1197 ConvertTypeForMem(RHSVD->getType()),
1201 QualType Type = PrivateVD->getType();
1202 bool IsArray = getContext().getAsArrayType(Type) !=
nullptr;
1207 OriginalAddr = Builder.CreateElementBitCast(
1208 OriginalAddr, ConvertTypeForMem(LHSVD->getType()),
"lhs.begin");
1210 PrivateScope.
addPrivate(LHSVD, [OriginalAddr]() {
return OriginalAddr; });
1212 RHSVD, [
this, PrivateVD, RHSVD, IsArray]() {
1214 ? Builder.CreateElementBitCast(
1215 GetAddrOfLocalVar(PrivateVD),
1216 ConvertTypeForMem(RHSVD->getType()),
"rhs.begin")
1217 : GetAddrOfLocalVar(PrivateVD);
1229 if (!HaveInsertPoint())
1235 bool HasAtLeastOneReduction =
false;
1237 HasAtLeastOneReduction =
true;
1238 Privates.append(C->privates().begin(), C->privates().end());
1239 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1240 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1241 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1243 if (HasAtLeastOneReduction) {
1246 ReductionKind == OMPD_simd;
1247 bool SimpleReduction = ReductionKind == OMPD_simd;
1250 CGM.getOpenMPRuntime().emitReduction(
1252 {WithNowait, SimpleReduction, ReductionKind});
1261 llvm::BasicBlock *DoneBB =
nullptr;
1263 if (
const Expr *PostUpdate = C->getPostUpdateExpr()) {
1270 CGF.
Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1288 CodeGenBoundParametersTy;
1294 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1296 llvm::Function *OutlinedFn =
1305 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1310 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1312 const Expr *IfCond =
nullptr;
1314 if (C->getNameModifier() == OMPD_unknown ||
1315 C->getNameModifier() == OMPD_parallel) {
1316 IfCond = C->getCondition();
1321 OMPParallelScope
Scope(CGF, S);
1327 CodeGenBoundParameters(CGF, S, CapturedVars);
1330 CapturedVars, IfCond);
1339 if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder()) {
1343 IfCond = EmitScalarExpr(C->getCondition(),
1348 NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(),
1351 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1353 ProcBind = ProcBindClause->getProcBindKind();
1355 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1359 auto FiniCB = [
this](InsertPointTy IP) {
1360 CGBuilderTy::InsertPointGuard IPG(Builder);
1361 assert(IP.getBlock()->end() != IP.getPoint() &&
1362 "OpenMP IR Builder should cause terminated block!");
1363 llvm::BasicBlock *IPBB = IP.getBlock();
1364 llvm::BasicBlock *DestBB = IPBB->splitBasicBlock(IP.getPoint());
1365 IPBB->getTerminator()->eraseFromParent();
1366 Builder.SetInsertPoint(IPBB);
1368 EmitBranchThroughCleanup(Dest);
1375 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1387 auto BodyGenCB = [ParallelRegionBodyStmt,
1388 this](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1389 llvm::BasicBlock &ContinuationBB) {
1390 auto OldAllocaIP = AllocaInsertPt;
1391 AllocaInsertPt = &*AllocaIP.getPoint();
1393 auto OldReturnBlock = ReturnBlock;
1394 ReturnBlock = getJumpDestInCurrentScope(&ContinuationBB);
1396 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
1397 CodeGenIPBB->splitBasicBlock(CodeGenIP.getPoint());
1398 llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator();
1399 CodeGenIPBBTI->removeFromParent();
1401 Builder.SetInsertPoint(CodeGenIPBB);
1403 EmitStmt(ParallelRegionBodyStmt);
1405 Builder.Insert(CodeGenIPBBTI);
1407 AllocaInsertPt = OldAllocaIP;
1408 ReturnBlock = OldReturnBlock;
1413 Builder.restoreIP(OMPBuilder->CreateParallel(Builder, BodyGenCB, PrivCB,
1414 FiniCB, IfCond, NumThreads,
1446 int MaxLevel,
int Level = 0) {
1447 assert(
Level < MaxLevel &&
"Too deep lookup during loop body codegen.");
1449 if (
const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1452 "LLVM IR generation of compound statement ('{}')");
1456 for (
const Stmt *CurStmt : CS->body())
1460 if (SimplifiedS == NextLoop) {
1461 if (
const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1464 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1465 "Expected canonical for loop or range-based for loop.");
1466 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1467 CGF.
EmitStmt(CXXFor->getLoopVarStmt());
1468 S = CXXFor->getBody();
1470 if (
Level + 1 < MaxLevel) {
1485 EmitIgnoredExpr(UE);
1491 for (
const Expr *UE : C->updates())
1492 EmitIgnoredExpr(UE);
1497 JumpDest Continue = getJumpDestInCurrentScope(
"omp.body.continue");
1498 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1504 llvm::BasicBlock *NextBB = createBasicBlock(
"omp.body.next");
1505 EmitBranchOnBoolExpr(E, NextBB, Continue.
getBlock(),
1506 getProfileCount(D.
getBody()));
1520 BreakContinueStack.pop_back();
1524 const Stmt &S,
bool RequiresCleanup,
const Expr *LoopCond,
1525 const Expr *IncExpr,
1528 auto LoopExit = getJumpDestInCurrentScope(
"omp.inner.for.end");
1531 auto CondBlock = createBasicBlock(
"omp.inner.for.cond");
1532 EmitBlock(CondBlock);
1534 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.
getBegin()),
1535 SourceLocToDebugLoc(R.
getEnd()));
1539 llvm::BasicBlock *ExitBlock =
LoopExit.getBlock();
1540 if (RequiresCleanup)
1541 ExitBlock = createBasicBlock(
"omp.inner.for.cond.cleanup");
1543 llvm::BasicBlock *LoopBody = createBasicBlock(
"omp.inner.for.body");
1546 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1547 if (ExitBlock !=
LoopExit.getBlock()) {
1548 EmitBlock(ExitBlock);
1549 EmitBranchThroughCleanup(
LoopExit);
1552 EmitBlock(LoopBody);
1553 incrementProfileCounter(&S);
1556 JumpDest Continue = getJumpDestInCurrentScope(
"omp.inner.for.inc");
1557 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
1563 EmitIgnoredExpr(IncExpr);
1565 BreakContinueStack.pop_back();
1566 EmitBranch(CondBlock);
1573 if (!HaveInsertPoint())
1576 bool HasLinears =
false;
1578 for (
const Expr *Init : C->inits()) {
1580 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1581 if (
const auto *Ref =
1584 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1585 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1586 CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
1592 EmitAutoVarCleanups(Emission);
1599 if (
const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1600 if (
const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1601 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1603 EmitIgnoredExpr(CS);
1612 if (!HaveInsertPoint())
1614 llvm::BasicBlock *DoneBB =
nullptr;
1617 auto IC = C->varlist_begin();
1618 for (
const Expr *F : C->finals()) {
1623 llvm::BasicBlock *ThenBB = createBasicBlock(
".omp.linear.pu");
1624 DoneBB = createBasicBlock(
".omp.linear.pu.done");
1625 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1629 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1630 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1631 CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
1632 (*IC)->getType(),
VK_LValue, (*IC)->getExprLoc());
1633 Address OrigAddr = EmitLValue(&DRE).getAddress(*
this);
1635 VarScope.
addPrivate(OrigVD, [OrigAddr]() {
return OrigAddr; });
1640 if (
const Expr *PostUpdate = C->getPostUpdateExpr())
1641 EmitIgnoredExpr(PostUpdate);
1644 EmitBlock(DoneBB,
true);
1653 if (
const Expr *AlignmentExpr = Clause->getAlignment()) {
1656 ClauseAlignment = AlignmentCI->getValue();
1658 for (
const Expr *E : Clause->varlists()) {
1660 if (Alignment == 0) {
1667 E->getType()->getPointeeType()))
1670 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
1671 "alignment is not power of 2");
1672 if (Alignment != 0) {
1684 if (!HaveInsertPoint())
1688 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1689 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1692 EmitAutoVarCleanups(VarEmission);
1693 LocalDeclMap.erase(PrivateVD);
1694 (void)LoopScope.
addPrivate(VD, [&VarEmission]() {
1695 return VarEmission.getAllocatedAddress();
1697 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1699 (void)LoopScope.
addPrivate(PrivateVD, [
this, VD, E]() {
1700 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
1701 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1702 E->getType(), VK_LValue, E->getExprLoc());
1703 return EmitLValue(&DRE).getAddress(*this);
1706 (void)LoopScope.
addPrivate(PrivateVD, [&VarEmission]() {
1707 return VarEmission.getAllocatedAddress();
1714 if (!C->getNumForLoops())
1717 E = C->getLoopNumIterations().size();
1719 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
1720 const auto *VD = cast<VarDecl>(DRE->getDecl());
1723 if (DRE->refersToEnclosingVariableOrCapture()) {
1724 (void)LoopScope.
addPrivate(VD, [
this, DRE, VD]() {
1725 return CreateMemTemp(DRE->getType(), VD->getName());
1733 const Expr *Cond, llvm::BasicBlock *TrueBlock,
1734 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1752 assert(!E->getType().getNonReferenceType()->isRecordType() &&
1753 "dependent counter must not be an iterator.");
1754 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1757 (void)PreCondVars.
setVarAddr(CGF, VD, CounterAddr);
1759 (void)PreCondVars.
apply(CGF);
1772 if (!HaveInsertPoint())
1776 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1777 for (
const Expr *C : LoopDirective->counters()) {
1783 auto CurPrivate = C->privates().begin();
1784 for (
const Expr *E : C->varlists()) {
1785 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1786 const auto *PrivateVD =
1787 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1789 bool IsRegistered = PrivateScope.
addPrivate(VD, [
this, PrivateVD]() {
1791 EmitVarDecl(*PrivateVD);
1792 return GetAddrOfLocalVar(PrivateVD);
1794 assert(IsRegistered &&
"linear var already registered as private");
1798 EmitVarDecl(*PrivateVD);
1813 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
1823 auto *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
1835 LoopStack.setParallel(!IsMonotonic);
1836 LoopStack.setVectorizeEnable();
1843 if (!HaveInsertPoint())
1845 llvm::BasicBlock *DoneBB =
nullptr;
1849 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1850 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1852 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1853 OrigVD->hasGlobalStorage() || CED) {
1858 llvm::BasicBlock *ThenBB = createBasicBlock(
".omp.final.then");
1859 DoneBB = createBasicBlock(
".omp.final.done");
1860 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1867 EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(*
this);
1869 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
1871 (*IPC)->getType(),
VK_LValue, (*IPC)->getExprLoc());
1872 OrigAddr = EmitLValue(&DRE).getAddress(*
this);
1875 VarScope.
addPrivate(OrigVD, [OrigAddr]() {
return OrigAddr; });
1883 EmitBlock(DoneBB,
true);
1897 auto VDecl = cast<VarDecl>(Helper->
getDecl());
1905 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](
CodeGenFunction &CGF,
1919 const Expr *IfCond =
nullptr;
1922 (C->getNameModifier() == OMPD_unknown ||
1923 C->getNameModifier() == OMPD_simd)) {
1924 IfCond = C->getCondition();
1940 "Expected simd directive");
1941 OMPLoopScope PreInitScope(CGF, S);
1958 llvm::BasicBlock *ContBlock =
nullptr;
1973 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1981 CGF.
EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2007 CGF.EmitOMPInnerLoop(
2010 CGF.CGM.getOpenMPRuntime().initLastprivateConditionalCounter(
2013 CGF.EmitStopPoint(&S);
2019 if (HasLastprivateClause)
2020 CGF.EmitOMPLastprivateClauseFinal(S,
true);
2021 CGF.EmitOMPReductionClauseFinal(S, OMPD_simd);
2037 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2038 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd, CodeGen);
2041 void CodeGenFunction::EmitOMPOuterLoop(
2044 const CodeGenFunction::OMPLoopArguments &LoopArgs,
2050 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
2056 llvm::BasicBlock *CondBlock = createBasicBlock(
"omp.dispatch.cond");
2057 EmitBlock(CondBlock);
2059 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.
getBegin()),
2060 SourceLocToDebugLoc(R.
getEnd()));
2063 if (!DynamicOrOrdered) {
2067 EmitIgnoredExpr(LoopArgs.EUB);
2069 EmitIgnoredExpr(LoopArgs.Init);
2071 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
2074 RT.emitForNext(*
this, S.
getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
2075 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
2080 llvm::BasicBlock *ExitBlock = LoopExit.
getBlock();
2082 ExitBlock = createBasicBlock(
"omp.dispatch.cleanup");
2084 llvm::BasicBlock *LoopBody = createBasicBlock(
"omp.dispatch.body");
2085 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
2086 if (ExitBlock != LoopExit.
getBlock()) {
2087 EmitBlock(ExitBlock);
2088 EmitBranchThroughCleanup(LoopExit);
2090 EmitBlock(LoopBody);
2094 if (DynamicOrOrdered)
2095 EmitIgnoredExpr(LoopArgs.Init);
2098 JumpDest Continue = getJumpDestInCurrentScope(
"omp.dispatch.inc");
2099 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
2111 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
2119 CGF.EmitOMPInnerLoop(
2122 CodeGenLoop(CGF, S, LoopExit);
2125 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2130 BreakContinueStack.pop_back();
2131 if (!DynamicOrOrdered) {
2133 EmitIgnoredExpr(LoopArgs.NextLB);
2134 EmitIgnoredExpr(LoopArgs.NextUB);
2137 EmitBranch(CondBlock);
2140 EmitBlock(LoopExit.getBlock());
2144 if (!DynamicOrOrdered)
2145 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.
getEndLoc(),
2151 void CodeGenFunction::EmitOMPForOuterLoop(
2154 const OMPLoopArguments &LoopArgs,
2159 const bool DynamicOrOrdered =
2160 Ordered || RT.isDynamic(ScheduleKind.
Schedule);
2163 !RT.isStaticNonchunked(ScheduleKind.
Schedule,
2164 LoopArgs.Chunk !=
nullptr)) &&
2165 "static non-chunked schedule does not need outer loop");
2218 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
2221 if (DynamicOrOrdered) {
2222 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
2223 CGDispatchBounds(*
this, S, LoopArgs.LB, LoopArgs.UB);
2226 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
2228 RT.emitForDispatchInit(*
this, S.
getBeginLoc(), ScheduleKind, IVSize,
2229 IVSigned, Ordered, DipatchRTInputValues);
2231 CGOpenMPRuntime::StaticRTInput StaticInit(
2232 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
2233 LoopArgs.ST, LoopArgs.Chunk);
2235 ScheduleKind, StaticInit);
2239 const unsigned IVSize,
2240 const bool IVSigned) {
2247 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
2248 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
2249 OuterLoopArgs.IncExpr = S.
getInc();
2250 OuterLoopArgs.Init = S.
getInit();
2251 OuterLoopArgs.Cond = S.
getCond();
2254 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
2259 const unsigned IVSize,
const bool IVSigned) {}
2261 void CodeGenFunction::EmitOMPDistributeOuterLoop(
2263 OMPPrivateScope &LoopScope,
const OMPLoopArguments &LoopArgs,
2264 const CodeGenLoopTy &CodeGenLoopContent) {
2274 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
2277 CGOpenMPRuntime::StaticRTInput StaticInit(
2278 IVSize, IVSigned,
false, LoopArgs.IL, LoopArgs.LB,
2279 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2280 RT.emitDistributeStaticInit(*
this, S.
getBeginLoc(), ScheduleKind, StaticInit);
2293 OMPLoopArguments OuterLoopArgs;
2294 OuterLoopArgs.LB = LoopArgs.LB;
2295 OuterLoopArgs.UB = LoopArgs.UB;
2296 OuterLoopArgs.ST = LoopArgs.ST;
2297 OuterLoopArgs.IL = LoopArgs.IL;
2298 OuterLoopArgs.Chunk = LoopArgs.Chunk;
2302 OuterLoopArgs.IncExpr = IncExpr;
2316 EmitOMPOuterLoop(
false,
false, S,
2317 LoopScope, OuterLoopArgs, CodeGenLoopContent,
2321 static std::pair<LValue, LValue>
2364 static std::pair<llvm::Value *, llvm::Value *>
2374 QualType IteratorTy = IVExpr->getType();
2379 return {LBVal, UBVal};
2385 const auto &Dir = cast<OMPLoopDirective>(S);
2387 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2391 CapturedVars.push_back(LBCast);
2393 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2398 CapturedVars.push_back(UBCast);
2408 bool HasCancel =
false;
2410 if (
const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2411 HasCancel = D->hasCancel();
2412 else if (
const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2413 HasCancel = D->hasCancel();
2414 else if (
const auto *D =
2415 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2416 HasCancel = D->hasCancel();
2428 CGInlinedWorksharingLoop,
2438 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
2439 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute, CodeGen);
2448 OMPLexicalScope
Scope(*
this, S, OMPD_parallel);
2449 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute, CodeGen);
2457 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2458 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd, CodeGen);
2468 llvm::Constant *Addr;
2471 S, ParentName, Fn, Addr,
true, CodeGen);
2472 assert(Fn && Addr &&
"Target device function emission failed.");
2484 struct ScheduleKindModifiersTy {
2491 :
Kind(Kind), M1(M1), M2(M2) {}
2501 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2502 EmitVarDecl(*IVDecl);
2508 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2515 bool HasLastprivateClause;
2518 OMPLoopScope PreInitScope(*
this, S);
2523 llvm::BasicBlock *ContBlock =
nullptr;
2524 if (ConstantFoldsToSimpleInteger(S.
getPreCond(), CondConstant)) {
2528 llvm::BasicBlock *ThenBlock = createBasicBlock(
"omp.precond.then");
2529 ContBlock = createBasicBlock(
"omp.precond.end");
2531 getProfileCount(&S));
2532 EmitBlock(ThenBlock);
2533 incrementProfileCounter(&S);
2537 bool Ordered =
false;
2539 if (OrderedClause->getNumForLoops())
2540 RT.emitDoacrossInit(*
this, S, OrderedClause->getLoopNumIterations());
2547 bool HasLinears = EmitOMPLinearClauseInit(S);
2550 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*
this, S);
2551 LValue LB = Bounds.first;
2552 LValue UB = Bounds.second;
2561 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2565 CGM.getOpenMPRuntime().emitBarrierCall(
2569 EmitOMPPrivateClause(S, LoopScope);
2572 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2573 EmitOMPReductionClauseInit(S, LoopScope);
2574 EmitOMPPrivateLoopCounters(S, LoopScope);
2575 EmitOMPLinearClause(S, LoopScope);
2578 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*
this, S);
2581 const Expr *ChunkExpr =
nullptr;
2584 ScheduleKind.
Schedule = C->getScheduleKind();
2585 ScheduleKind.
M1 = C->getFirstScheduleModifier();
2586 ScheduleKind.
M2 = C->getSecondScheduleModifier();
2587 ChunkExpr = C->getChunkSize();
2590 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
2591 *
this, S, ScheduleKind.
Schedule, ChunkExpr);
2593 bool HasChunkSizeOne =
false;
2596 Chunk = EmitScalarExpr(ChunkExpr);
2597 Chunk = EmitScalarConversion(Chunk, ChunkExpr->
getType(),
2603 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
2606 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2607 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2612 bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.
Schedule,
2613 Chunk !=
nullptr) && HasChunkSizeOne &&
2615 if ((RT.isStaticNonchunked(ScheduleKind.
Schedule,
2616 Chunk !=
nullptr) ||
2617 StaticChunkedOne) &&
2620 getJumpDestInCurrentScope(createBasicBlock(
"omp.loop.exit"));
2627 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
2628 &S, ScheduleKind, LoopExit,
2635 CGOpenMPRuntime::StaticRTInput StaticInit(
2636 IVSize, IVSigned, Ordered, IL.getAddress(CGF),
2637 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF),
2638 StaticChunkedOne ? Chunk :
nullptr);
2639 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2640 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
2643 if (!StaticChunkedOne)
2644 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
2646 CGF.EmitIgnoredExpr(S.getInit());
2660 CGF.EmitOMPInnerLoop(
2662 StaticChunkedOne ? S.getCombinedParForInDistCond()
2664 StaticChunkedOne ? S.getDistInc() : S.getInc(),
2666 CGF.CGM.getOpenMPRuntime()
2667 .initLastprivateConditionalCounter(CGF, S);
2668 CGF.EmitOMPLoopBody(S, LoopExit);
2669 CGF.EmitStopPoint(&S);
2673 EmitBlock(LoopExit.getBlock());
2676 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2677 S.getDirectiveKind());
2679 OMPCancelStack.emitExit(*
this, S.getDirectiveKind(), CodeGen);
2681 const bool IsMonotonic =
2682 Ordered || ScheduleKind.
Schedule == OMPC_SCHEDULE_static ||
2684 ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2685 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2688 const OMPLoopArguments LoopArguments(
2691 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2692 LoopArguments, CGDispatchBounds);
2696 return CGF.
Builder.CreateIsNotNull(
2700 EmitOMPReductionClauseFinal(
2702 ? OMPD_parallel_for_simd
2707 return CGF.
Builder.CreateIsNotNull(
2711 if (HasLastprivateClause)
2712 EmitOMPLastprivateClauseFinal(
2714 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.
getBeginLoc())));
2717 return CGF.
Builder.CreateIsNotNull(
2723 EmitBranch(ContBlock);
2724 EmitBlock(ContBlock,
true);
2727 return HasLastprivateClause;
2733 static std::pair<LValue, LValue>
2735 const auto &LS = cast<OMPLoopDirective>(S);
2747 static std::pair<llvm::Value *, llvm::Value *>
2750 const auto &LS = cast<OMPLoopDirective>(S);
2751 const Expr *IVExpr = LS.getIterationVariable();
2755 return {LBVal, UBVal};
2759 bool HasLastprivates =
false;
2768 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2769 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_for, CodeGen,
2775 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getBeginLoc(), OMPD_for);
2779 bool HasLastprivates =
false;
2787 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2788 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd, CodeGen);
2793 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getBeginLoc(), OMPD_for);
2808 bool HasLastprivates =
false;
2809 auto &&CodeGen = [&S, CapturedStmt, CS,
2816 CGF.Builder.getInt32(0));
2817 llvm::ConstantInt *GlobalUBVal = CS !=
nullptr 2818 ? CGF.Builder.getInt32(CS->size() - 1)
2819 : CGF.Builder.getInt32(0);
2823 CGF.Builder.getInt32(1));
2825 CGF.Builder.getInt32(0));
2850 CGF.CGM.getOpenMPRuntime().initLastprivateConditionalCounter(CGF, S);
2851 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".omp.sections.exit");
2853 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.
getBeginLoc()),
2854 ExitBB, CS ==
nullptr ? 1 : CS->size());
2856 unsigned CaseNumber = 0;
2858 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
2859 CGF.EmitBlock(CaseBB);
2860 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2861 CGF.EmitStmt(SubStmt);
2862 CGF.EmitBranch(ExitBB);
2866 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(
".omp.sections.case");
2867 CGF.EmitBlock(CaseBB);
2868 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2869 CGF.EmitStmt(CapturedStmt);
2870 CGF.EmitBranch(ExitBB);
2872 CGF.EmitBlock(ExitBB,
true);
2876 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2880 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2884 CGF.EmitOMPPrivateClause(S, LoopScope);
2886 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2887 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2890 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2894 ScheduleKind.
Schedule = OMPC_SCHEDULE_static;
2895 CGOpenMPRuntime::StaticRTInput StaticInit(
2896 32,
true,
false, IL.getAddress(CGF),
2898 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2899 CGF, S.
getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2902 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2903 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2904 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2906 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.
getBeginLoc()), IV);
2908 CGF.EmitOMPInnerLoop(S,
false, &Cond, &Inc, BodyGen,
2912 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.
getEndLoc(),
2913 S.getDirectiveKind());
2915 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2916 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
2919 return CGF.
Builder.CreateIsNotNull(
2924 if (HasLastprivates)
2931 bool HasCancel =
false;
2932 if (
auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2933 HasCancel = OSD->hasCancel();
2934 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2935 HasCancel = OPSD->hasCancel();
2937 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_sections, CodeGen,
2945 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getBeginLoc(),
2952 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2957 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getBeginLoc(),
2966 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2967 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_section, CodeGen,
2981 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2982 DestExprs.append(C->destination_exprs().begin(),
2983 C->destination_exprs().end());
2984 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2985 AssignmentOps.append(C->assignment_ops().begin(),
2986 C->assignment_ops().end());
2998 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
2999 CGM.getOpenMPRuntime().emitSingleRegion(*
this, CodeGen, S.
getBeginLoc(),
3000 CopyprivateVars, DestExprs,
3001 SrcExprs, AssignmentOps);
3006 CGM.getOpenMPRuntime().emitBarrierCall(
3021 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3030 const Expr *Hint =
nullptr;
3032 Hint = HintClause->getHint();
3033 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3034 CGM.getOpenMPRuntime().emitCriticalRegion(*
this,
3101 CGF.EmitSections(S);
3114 auto PartId = std::next(I);
3115 auto TaskT = std::next(I, 4);
3120 const Expr *Cond = Clause->getCondition();
3122 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
3123 Data.
Final.setInt(CondConstant);
3125 Data.
Final.setPointer(EvaluateExprAsBool(Cond));
3128 Data.
Final.setInt(
false);
3132 const Expr *Prio = Clause->getPriority();
3134 Data.
Priority.setPointer(EmitScalarConversion(
3135 EmitScalarExpr(Prio), Prio->
getType(),
3136 getContext().getIntTypeForBitwidth(32, 1),
3144 auto IRef = C->varlist_begin();
3145 for (
const Expr *IInit : C->private_copies()) {
3146 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
3147 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
3154 EmittedAsPrivate.clear();
3157 auto IRef = C->varlist_begin();
3158 auto IElemInitRef = C->inits().begin();
3159 for (
const Expr *IInit : C->private_copies()) {
3160 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
3161 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
3171 llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
3173 auto IRef = C->varlist_begin();
3174 auto ID = C->destination_exprs().begin();
3175 for (
const Expr *IInit : C->private_copies()) {
3176 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
3177 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
3181 LastprivateDstsOrigs.insert(
3182 {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
3183 cast<DeclRefExpr>(*IRef)});
3191 auto IPriv = C->privates().begin();
3192 auto IRed = C->reduction_ops().begin();
3193 auto ILHS = C->lhs_exprs().begin();
3194 auto IRHS = C->rhs_exprs().begin();
3195 for (
const Expr *Ref : C->varlists()) {
3199 LHSs.emplace_back(*ILHS);
3200 RHSs.emplace_back(*IRHS);
3201 std::advance(IPriv, 1);
3202 std::advance(IRed, 1);
3203 std::advance(ILHS, 1);
3204 std::advance(IRHS, 1);
3207 Data.
Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
3211 for (
const Expr *IRef : C->varlists())
3212 Data.
Dependences.emplace_back(C->getDependencyKind(), IRef);
3213 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
3220 llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3221 CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()},
true);
3222 enum { PrivatesParam = 2, CopyFnParam = 3 };
3224 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3225 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3226 CS->getCapturedDecl()->getParam(PrivatesParam)));
3230 CallArgs.push_back(PrivatesPtr);
3232 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3233 Address PrivatePtr = CGF.CreateMemTemp(
3234 CGF.getContext().getPointerType(E->
getType()),
".priv.ptr.addr");
3235 PrivatePtrs.emplace_back(VD, PrivatePtr);
3236 CallArgs.push_back(PrivatePtr.getPointer());
3239 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3241 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
3242 ".firstpriv.ptr.addr");
3243 PrivatePtrs.emplace_back(VD, PrivatePtr);
3244 CallArgs.push_back(PrivatePtr.getPointer());
3247 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3249 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
3250 ".lastpriv.ptr.addr");
3251 PrivatePtrs.emplace_back(VD, PrivatePtr);
3252 CallArgs.push_back(PrivatePtr.getPointer());
3254 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3255 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
3256 for (
const auto &Pair : LastprivateDstsOrigs) {
3257 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
3260 CGF.CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
3262 Pair.second->getExprLoc());
3263 Scope.
addPrivate(Pair.first, [&CGF, &DRE]() {
3264 return CGF.EmitLValue(&DRE).getAddress(CGF);
3267 for (
const auto &Pair : PrivatePtrs) {
3268 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3269 CGF.getContext().getDeclAlign(Pair.first));
3270 Scope.
addPrivate(Pair.first, [Replacement]() { return Replacement; });
3274 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
3277 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
3278 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
3279 for (
unsigned Cnt = 0, E = Data.
ReductionVars.size(); Cnt < E; ++Cnt) {
3280 RedCG.emitSharedLValue(CGF, Cnt);
3285 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3287 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3290 Address(CGF.EmitScalarConversion(
3291 Replacement.
getPointer(), CGF.getContext().VoidPtrTy,
3292 CGF.getContext().getPointerType(
3298 [Replacement]() {
return Replacement; });
3308 auto IPriv = C->privates().begin();
3309 auto IRed = C->reduction_ops().begin();
3310 auto ITD = C->taskgroup_descriptors().begin();
3311 for (
const Expr *Ref : C->varlists()) {
3312 InRedVars.emplace_back(Ref);
3313 InRedPrivs.emplace_back(*IPriv);
3314 InRedOps.emplace_back(*IRed);
3315 TaskgroupDescriptors.emplace_back(*ITD);
3316 std::advance(IPriv, 1);
3317 std::advance(IRed, 1);
3318 std::advance(ITD, 1);
3324 if (!InRedVars.empty()) {
3326 for (
unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
3327 RedCG.emitSharedLValue(CGF, Cnt);
3334 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3337 CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
3338 TaskgroupDescriptors[Cnt]->getExprLoc());
3339 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3342 CGF.EmitScalarConversion(
3343 Replacement.
getPointer(), CGF.getContext().VoidPtrTy,
3344 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
3345 InRedPrivs[Cnt]->getExprLoc()),
3349 [Replacement]() {
return Replacement; });
3357 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3358 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.
Tied,
3363 TaskGen(*
this, OutlinedFn, Data);
3401 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3404 auto PartId = std::next(I);
3405 auto TaskT = std::next(I, 4);
3408 Data.
Final.setInt(
false);
3411 auto IRef = C->varlist_begin();
3412 auto IElemInitRef = C->inits().begin();
3413 for (
auto *IInit : C->private_copies()) {
3427 getContext(), getContext().getTranslationUnitDecl(), 0);
3429 QualType BaseAndPointersType = getContext().getConstantArrayType(
3433 getContext(), Data, BaseAndPointersType, CD, S.
getBeginLoc());
3435 getContext(), Data, BaseAndPointersType, CD, S.
getBeginLoc());
3436 QualType SizesType = getContext().getConstantArrayType(
3437 getContext().getIntTypeForBitwidth(64, 1),
3447 [&InputInfo]() {
return InputInfo.
SizesArray; });
3452 for (
const Expr *IRef : C->varlists())
3453 Data.
Dependences.emplace_back(C->getDependencyKind(), IRef);
3454 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3459 llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3460 CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()},
true);
3461 enum { PrivatesParam = 2, CopyFnParam = 3 };
3463 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3464 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3465 CS->getCapturedDecl()->getParam(PrivatesParam)));
3469 CallArgs.push_back(PrivatesPtr);
3471 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3473 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->
getType()),
3474 ".firstpriv.ptr.addr");
3475 PrivatePtrs.emplace_back(VD, PrivatePtr);
3476 CallArgs.push_back(PrivatePtr.getPointer());
3478 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3479 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
3480 for (
const auto &Pair : PrivatePtrs) {
3481 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3482 CGF.getContext().getDeclAlign(Pair.first));
3483 Scope.
addPrivate(Pair.first, [Replacement]() { return Replacement; });
3490 CGF.GetAddrOfLocalVar(BPVD), 0);
3492 CGF.GetAddrOfLocalVar(PVD), 0);
3493 InputInfo.
SizesArray = CGF.Builder.CreateConstArrayGEP(
3494 CGF.GetAddrOfLocalVar(SVD), 0);
3498 OMPLexicalScope LexScope(CGF, S, OMPD_task,
false);
3501 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3502 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen,
true,
3506 getContext().getIntTypeForBitwidth(32, 0),
3509 CGM.getOpenMPRuntime().emitTaskCall(*
this, S.getBeginLoc(), S, OutlinedFn,
3510 SharedsTy, CapturedStruct, &IfCond, Data);
3516 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3518 const Expr *IfCond =
nullptr;
3520 if (C->getNameModifier() == OMPD_unknown ||
3521 C->getNameModifier() == OMPD_task) {
3522 IfCond = C->getCondition();
3533 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3536 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.
getBeginLoc(), S, OutlinedFn,
3537 SharedsTy, CapturedStruct, IfCond,
3540 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3545 CGM.getOpenMPRuntime().emitTaskyieldCall(*
this, S.
getBeginLoc());
3549 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getBeginLoc(), OMPD_barrier);
3553 CGM.getOpenMPRuntime().emitTaskwaitCall(*
this, S.
getBeginLoc());
3565 auto IPriv = C->privates().begin();
3566 auto IRed = C->reduction_ops().begin();
3567 auto ILHS = C->lhs_exprs().begin();
3568 auto IRHS = C->rhs_exprs().begin();
3569 for (
const Expr *Ref : C->varlists()) {
3573 LHSs.emplace_back(*ILHS);
3574 RHSs.emplace_back(*IRHS);
3575 std::advance(IPriv, 1);
3576 std::advance(IRed, 1);
3577 std::advance(ILHS, 1);
3578 std::advance(IRHS, 1);
3584 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3591 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3592 CGM.getOpenMPRuntime().emitTaskgroupRegion(*
this, CodeGen, S.
getBeginLoc());
3596 CGM.getOpenMPRuntime().emitFlush(
3600 return llvm::makeArrayRef(FlushClause->varlist_begin(),
3601 FlushClause->varlist_end());
3612 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3613 EmitVarDecl(*IVDecl);
3619 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3626 bool HasLastprivateClause =
false;
3629 OMPLoopScope PreInitScope(*
this, S);
3634 llvm::BasicBlock *ContBlock =
nullptr;
3635 if (ConstantFoldsToSimpleInteger(S.
getPreCond(), CondConstant)) {
3639 llvm::BasicBlock *ThenBlock = createBasicBlock(
"omp.precond.then");
3640 ContBlock = createBasicBlock(
"omp.precond.end");
3642 getProfileCount(&S));
3643 EmitBlock(ThenBlock);
3644 incrementProfileCounter(&S);
3653 *
this, cast<DeclRefExpr>(
3658 *
this, cast<DeclRefExpr>(
3668 if (EmitOMPFirstprivateClause(S, LoopScope)) {
3672 CGM.getOpenMPRuntime().emitBarrierCall(
3676 EmitOMPPrivateClause(S, LoopScope);
3680 EmitOMPReductionClauseInit(S, LoopScope);
3681 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3682 EmitOMPPrivateLoopCounters(S, LoopScope);
3685 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*
this, S);
3691 ScheduleKind = C->getDistScheduleKind();
3692 if (
const Expr *Ch = C->getChunkSize()) {
3693 Chunk = EmitScalarExpr(Ch);
3694 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3700 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
3701 *
this, S, ScheduleKind, Chunk);
3703 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3704 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3714 bool StaticChunked = RT.isStaticChunked(
3715 ScheduleKind, Chunk !=
nullptr) &&
3717 if (RT.isStaticNonchunked(ScheduleKind,
3718 Chunk !=
nullptr) ||
3720 CGOpenMPRuntime::StaticRTInput StaticInit(
3721 IVSize, IVSigned,
false, IL.
getAddress(*
this),
3722 LB.
getAddress(*
this), UB.getAddress(*
this), ST.getAddress(*
this),
3723 StaticChunked ? Chunk :
nullptr);
3724 RT.emitDistributeStaticInit(*
this, S.
getBeginLoc(), ScheduleKind,
3727 getJumpDestInCurrentScope(createBasicBlock(
"omp.loop.exit"));
3775 [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,
3777 CGF.EmitOMPInnerLoop(
3778 S, LoopScope.requiresCleanups(), Cond, IncExpr,
3780 CodeGenLoop(CGF, S, LoopExit);
3783 if (StaticChunked) {
3784 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
3785 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
3786 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
3787 CGF.EmitIgnoredExpr(S.getCombinedInit());
3791 EmitBlock(LoopExit.getBlock());
3797 const OMPLoopArguments LoopArguments = {
3798 LB.
getAddress(*
this), UB.getAddress(*
this), ST.getAddress(*
this),
3800 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3805 return CGF.
Builder.CreateIsNotNull(
3812 EmitOMPReductionClauseFinal(S, OMPD_simd);
3816 return CGF.
Builder.CreateIsNotNull(
3821 if (HasLastprivateClause) {
3822 EmitOMPLastprivateClauseFinal(
3824 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.
getBeginLoc())));
3830 EmitBranch(ContBlock);
3831 EmitBlock(ContBlock,
true);
3841 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3842 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute, CodeGen);
3851 Fn->setDoesNotRecurse();
3858 "No associated statement must be in ordered depend construct.");
3860 CGM.getOpenMPRuntime().emitDoacrossOrdered(*
this, DC);
3869 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3871 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.
getBeginLoc(),
3872 OutlinedFn, CapturedVars);
3878 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
3879 CGM.getOpenMPRuntime().emitOrderedRegion(*
this, CodeGen, S.
getBeginLoc(), !C);
3886 "DestType must have scalar evaluation kind.");
3887 assert(!Val.
isAggregate() &&
"Must be a scalar or complex.");
3898 "DestType must have complex evaluation kind.");
3907 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3909 assert(Val.
isComplex() &&
"Must be a scalar or complex.");
3914 Val.
getComplexVal().first, SrcElementType, DestElementType, Loc);
3916 Val.
getComplexVal().second, SrcElementType, DestElementType, Loc);
3927 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3928 : llvm::AtomicOrdering::Monotonic,
3935 switch (getEvaluationKind(LVal.
getType())) {
3938 *
this, RVal, RValTy, LVal.
getType(), Loc)),
3947 llvm_unreachable(
"Must be a scalar or complex.");
3955 assert(V->
isLValue() &&
"V of 'omp atomic read' is not lvalue");
3956 assert(X->
isLValue() &&
"X of 'omp atomic read' is not lvalue");
3959 RValue Res = XLValue.isGlobalReg()
3963 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3964 : llvm::AtomicOrdering::Monotonic,
3965 XLValue.isVolatile());
3979 assert(X->
isLValue() &&
"X of 'omp atomic write' is not lvalue");
3992 llvm::AtomicOrdering AO,
3993 bool IsXLHSInRHSPart) {
3998 if (BO == BO_Comma || !Update.
isScalar() ||
4006 return std::make_pair(
false,
RValue::get(
nullptr));
4008 llvm::AtomicRMWInst::BinOp RMWOp;
4014 if (!IsXLHSInRHSPart)
4015 return std::make_pair(
false,
RValue::get(
nullptr));
4022 RMWOp = llvm::AtomicRMWInst::Or;
4025 RMWOp = llvm::AtomicRMWInst::Xor;
4029 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
4030 : llvm::AtomicRMWInst::Max)
4031 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
4032 : llvm::AtomicRMWInst::UMax);
4036 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
4037 : llvm::AtomicRMWInst::Min)
4038 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
4039 : llvm::AtomicRMWInst::UMin);
4042 RMWOp = llvm::AtomicRMWInst::Xchg;
4051 return std::make_pair(
false,
RValue::get(
nullptr));
4070 llvm_unreachable(
"Unsupported atomic update operation");
4073 if (
auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
4074 UpdateVal = CGF.
Builder.CreateIntCast(
4098 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
4109 const Expr *UE,
bool IsXLHSInRHSPart,
4112 "Update expr in 'atomic update' must be a binary operator.");
4120 assert(X->
isLValue() &&
"X of 'omp atomic update' is not lvalue");
4123 llvm::AtomicOrdering AO = IsSeqCst
4124 ? llvm::AtomicOrdering::SequentiallyConsistent
4125 : llvm::AtomicOrdering::Monotonic;
4126 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
4127 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
4130 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](
RValue XRValue) {
4136 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
4159 llvm_unreachable(
"Must be a scalar or complex.");
4163 bool IsPostfixUpdate,
const Expr *
V,
4165 const Expr *UE,
bool IsXLHSInRHSPart,
4167 assert(X->
isLValue() &&
"X of 'omp atomic capture' is not lvalue");
4168 assert(V->
isLValue() &&
"V of 'omp atomic capture' is not lvalue");
4173 llvm::AtomicOrdering AO = IsSeqCst
4174 ? llvm::AtomicOrdering::SequentiallyConsistent
4175 : llvm::AtomicOrdering::Monotonic;
4180 "Update expr in 'atomic capture' must be a binary operator.");
4188 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
4189 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
4191 NewVValType = XRValExpr->getType();
4193 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
4194 IsPostfixUpdate](
RValue XRValue) {
4198 NewVVal = IsPostfixUpdate ? XRValue : Res;
4202 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
4205 if (IsPostfixUpdate) {
4207 NewVVal = Res.second;
4221 auto &&Gen = [&NewVVal, ExprRValue](
RValue XRValue) {
4227 XLValue, ExprRValue, BO_Assign,
false, AO,
4231 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
4245 bool IsSeqCst,
bool IsPostfixUpdate,
4247 const Expr *UE,
bool IsXLHSInRHSPart,
4262 IsXLHSInRHSPart, Loc);
4266 case OMPC_num_threads:
4268 case OMPC_firstprivate:
4269 case OMPC_lastprivate:
4270 case OMPC_reduction:
4271 case OMPC_task_reduction:
4272 case OMPC_in_reduction:
4275 case OMPC_allocator:
4284 case OMPC_copyprivate:
4286 case OMPC_proc_bind:
4293 case OMPC_mergeable:
4298 case OMPC_num_teams:
4299 case OMPC_thread_limit:
4301 case OMPC_grainsize:
4303 case OMPC_num_tasks:
4305 case OMPC_dist_schedule:
4306 case OMPC_defaultmap:
4310 case OMPC_use_device_ptr:
4311 case OMPC_is_device_ptr:
4312 case OMPC_unified_address:
4313 case OMPC_unified_shared_memory:
4314 case OMPC_reverse_offload:
4315 case OMPC_dynamic_allocators:
4316 case OMPC_atomic_default_mem_order:
4319 case OMPC_nontemporal:
4320 llvm_unreachable(
"Clause is not allowed in 'omp atomic'.");
4329 if (C->getClauseKind() != OMPC_seq_cst) {
4330 Kind = C->getClauseKind();
4336 if (
const auto *FE = dyn_cast<FullExpr>(CS))
4337 enterFullExpression(FE);
4339 if (
const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
4340 for (
const Stmt *C : Compound->body()) {
4341 if (
const auto *FE = dyn_cast<FullExpr>(C))
4342 enterFullExpression(FE);
4348 CGF.EmitStopPoint(CS);
4353 OMPLexicalScope
Scope(*
this, S, OMPD_unknown);
4354 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_atomic, CodeGen);
4365 OMPLexicalScope
Scope(CGF, S, OMPD_target);
4373 llvm::Function *Fn =
nullptr;
4374 llvm::Constant *FnID =
nullptr;
4376 const Expr *IfCond =
nullptr;
4379 if (C->getNameModifier() == OMPD_unknown ||
4380 C->getNameModifier() == OMPD_target) {
4381 IfCond = C->getCondition();
4387 const Expr *Device =
nullptr;
4389 Device = C->getDevice();
4394 bool IsOffloadEntry =
true;
4398 IsOffloadEntry =
false;
4401 IsOffloadEntry =
false;
4403 assert(CGF.
CurFuncDecl &&
"No parent declaration for target region!");
4404 StringRef ParentName;
4407 if (
const auto *D = dyn_cast<CXXConstructorDecl>(CGF.
CurFuncDecl))
4409 else if (
const auto *D = dyn_cast<CXXDestructorDecl>(CGF.
CurFuncDecl))
4417 IsOffloadEntry, CodeGen);
4418 OMPLexicalScope
Scope(CGF, S, OMPD_task);
4419 auto &&SizeEmitter =
4422 if (IsOffloadEntry) {
4423 OMPLoopScope(CGF, D);
4426 NumIterations = CGF.
Builder.CreateIntCast(NumIterations, CGF.
Int64Ty,
4428 return NumIterations;
4450 StringRef ParentName,
4456 llvm::Constant *Addr;
4459 S, ParentName, Fn, Addr,
true, CodeGen);
4460 assert(Fn && Addr &&
"Target device function emission failed.");
4475 llvm::Function *OutlinedFn =
4482 const Expr *NumTeams = NT ? NT->getNumTeams() :
nullptr;
4483 const Expr *ThreadLimit = TL ? TL->getThreadLimit() :
nullptr;
4489 OMPTeamsScope
Scope(CGF, S);
4521 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4522 CGF.EmitOMPPrivateClause(S, PrivateScope);
4523 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4526 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4527 CGF.EmitStmt(CS->getCapturedStmt());
4528 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4542 llvm::Constant *Addr;
4545 S, ParentName, Fn, Addr,
true, CodeGen);
4546 assert(Fn && Addr &&
"Target device function emission failed.");
4570 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4572 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4574 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4588 llvm::Constant *Addr;
4591 S, ParentName, Fn, Addr,
true, CodeGen);
4592 assert(Fn && Addr &&
"Target device function emission failed.");
4616 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4618 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4620 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4634 llvm::Constant *Addr;
4637 S, ParentName, Fn, Addr,
true, CodeGen);
4638 assert(Fn && Addr &&
"Target device function emission failed.");
4661 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4663 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4665 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4683 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4685 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4687 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4706 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4708 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4710 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4729 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4731 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4732 CGF, OMPD_distribute, CodeGenDistribute,
false);
4733 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4755 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4757 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4758 CGF, OMPD_distribute, CodeGenDistribute,
false);
4759 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4777 llvm::Constant *Addr;
4780 S, ParentName, Fn, Addr,
true, CodeGen);
4781 assert(Fn && Addr &&
"Target device function emission failed.");
4807 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4809 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4810 CGF, OMPD_distribute, CodeGenDistribute,
false);
4811 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4829 llvm::Constant *Addr;
4832 S, ParentName, Fn, Addr,
true, CodeGen);
4833 assert(Fn && Addr &&
"Target device function emission failed.");
4846 CGM.getOpenMPRuntime().emitCancellationPointCall(*
this, S.
getBeginLoc(),
4851 const Expr *IfCond =
nullptr;
4853 if (C->getNameModifier() == OMPD_unknown ||
4854 C->getNameModifier() == OMPD_cancel) {
4855 IfCond = C->getCondition();
4859 if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder()) {
4865 IfCondition = EmitScalarExpr(IfCond,
4867 return Builder.restoreIP(
4872 CGM.getOpenMPRuntime().emitCancelCall(*
this, S.
getBeginLoc(), IfCond,
4878 if (Kind == OMPD_parallel || Kind == OMPD_task ||
4879 Kind == OMPD_target_parallel)
4881 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4882 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4883 Kind == OMPD_distribute_parallel_for ||
4884 Kind == OMPD_target_parallel_for ||
4885 Kind == OMPD_teams_distribute_parallel_for ||
4886 Kind == OMPD_target_teams_distribute_parallel_for);
4887 return OMPCancelStack.getExitBlock();
4892 const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4893 const auto &C = cast<OMPUseDevicePtrClause>(NC);
4894 auto OrigVarIt = C.varlist_begin();
4895 auto InitIt = C.inits().begin();
4896 for (
const Expr *PvtVarIt : C.private_copies()) {
4897 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4898 const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4899 const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4905 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4908 const auto *ME = cast<MemberExpr>(OED->getInit());
4909 assert(isa<CXXThisExpr>(ME->getBase()) &&
4910 "Base should be the current struct!");
4911 MatchingVD = ME->getMemberDecl();
4916 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4917 if (InitAddrIt == CaptureDeviceAddrMap.end())
4920 bool IsRegistered = PrivateScope.
addPrivate(OrigVD, [
this, OrigVD,
4929 getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4930 llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4931 Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4932 setAddrOfLocalVar(InitVD, InitAddr);
4940 LocalDeclMap.erase(InitVD);
4943 return GetAddrOfLocalVar(PvtVD);
4945 assert(IsRegistered &&
"firstprivate var already registered as private");
4957 CGOpenMPRuntime::TargetDataInfo Info(
true);
4962 bool PrivatizeDevicePointers =
false;
4964 bool &PrivatizeDevicePointers;
4967 explicit DevicePointerPrivActionTy(
bool &PrivatizeDevicePointers)
4968 :
PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4970 PrivatizeDevicePointers =
true;
4973 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4975 auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4982 auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4986 PrivatizeDevicePointers =
false;
4992 if (PrivatizeDevicePointers) {
4996 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4997 Info.CaptureDeviceAddrMap);
5012 OMPLexicalScope
Scope(CGF, S);
5013 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
5021 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
5027 const Expr *IfCond =
nullptr;
5029 IfCond = C->getCondition();
5032 const Expr *Device =
nullptr;
5034 Device = C->getDevice();
5040 CGM.getOpenMPRuntime().emitTargetDataCalls(*
this, S, IfCond, Device, RCG,
5048 if (CGM.getLangOpts().OMPTargetTriples.empty())
5052 const Expr *IfCond =
nullptr;
5054 IfCond = C->getCondition();
5057 const Expr *Device =
nullptr;
5059 Device = C->getDevice();
5061 OMPLexicalScope
Scope(*
this, S, OMPD_task);
5062 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond, Device);
5069 if (CGM.getLangOpts().OMPTargetTriples.empty())
5073 const Expr *IfCond =
nullptr;
5075 IfCond = C->getCondition();
5078 const Expr *Device =
nullptr;
5080 Device = C->getDevice();
5082 OMPLexicalScope
Scope(*
this, S, OMPD_task);
5083 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond, Device);
5095 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
5096 CGF.EmitOMPPrivateClause(S, PrivateScope);
5097 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
5100 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
5103 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
5118 llvm::Constant *Addr;
5121 S, ParentName, Fn, Addr,
true, CodeGen);
5122 assert(Fn && Addr &&
"Target device function emission failed.");
5142 CGF, OMPD_target_parallel_for, S.
hasCancel());
5158 llvm::Constant *Addr;
5161 S, ParentName, Fn, Addr,
true, CodeGen);
5162 assert(Fn && Addr &&
"Target device function emission failed.");
5197 llvm::Constant *Addr;
5200 S, ParentName, Fn, Addr,
true, CodeGen);
5201 assert(Fn && Addr &&
"Target device function emission failed.");
5216 const auto *VDecl = cast<VarDecl>(Helper->
getDecl());
5225 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
5227 const Expr *IfCond =
nullptr;
5229 if (C->getNameModifier() == OMPD_unknown ||
5230 C->getNameModifier() == OMPD_taskloop) {
5231 IfCond = C->getCondition();
5245 Data.
Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
5249 Data.
Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
5263 llvm::BasicBlock *ContBlock =
nullptr;
5264 OMPLoopScope PreInitScope(CGF, S);
5265 if (CGF.ConstantFoldsToSimpleInteger(S.
getPreCond(), CondConstant)) {
5269 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(
"taskloop.if.then");
5270 ContBlock = CGF.createBasicBlock(
"taskloop.if.end");
5272 CGF.getProfileCount(&S));
5273 CGF.EmitBlock(ThenBlock);
5274 CGF.incrementProfileCounter(&S);
5277 (void)CGF.EmitOMPLinearClauseInit(S);
5281 enum { LowerBound = 5, UpperBound, Stride, LastIter };
5283 auto *LBP = std::next(I, LowerBound);
5284 auto *UBP = std::next(I, UpperBound);
5285 auto *STP = std::next(I, Stride);
5286 auto *LIP = std::next(I, LastIter);
5294 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
5295 CGF.EmitOMPLinearClause(S, LoopScope);
5296 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
5300 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
5301 CGF.EmitVarDecl(*IVDecl);
5302 CGF.EmitIgnoredExpr(S.
getInit());
5308 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
5314 OMPLexicalScope
Scope(CGF, S, OMPD_taskloop,
false);
5322 CGF.EmitOMPInnerLoop(
5326 CGF.EmitStopPoint(&S);
5333 CGF.EmitBranch(ContBlock);
5334 CGF.EmitBlock(ContBlock,
true);
5337 if (HasLastprivateClause) {
5338 CGF.EmitOMPLastprivateClauseFinal(
5340 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
5341 CGF.GetAddrOfLocalVar(*LIP),
false,
5345 return CGF.
Builder.CreateIsNotNull(
5350 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5353 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
5355 OMPLoopScope PreInitScope(CGF, S);
5356 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.
getBeginLoc(), S,
5357 OutlinedFn, SharedsTy,
5358 CapturedStruct, IfCond, Data);
5360 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
5364 EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
5366 CGM.getOpenMPRuntime().emitTaskgroupRegion(
5379 EmitOMPTaskLoopBasedDirective(S);
5384 OMPLexicalScope
Scope(*
this, S);
5385 EmitOMPTaskLoopBasedDirective(S);
5392 EmitOMPTaskLoopBasedDirective(S);
5395 CGM.getOpenMPRuntime().emitMasterRegion(*
this, CodeGen, S.
getBeginLoc());
5402 EmitOMPTaskLoopBasedDirective(S);
5404 OMPLexicalScope
Scope(*
this, S);
5405 CGM.getOpenMPRuntime().emitMasterRegion(*
this, CodeGen, S.
getBeginLoc());
5417 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
5432 OMPLexicalScope
Scope(CGF, S, OMPD_parallel,
false);
5433 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
5445 if (CGM.getLangOpts().OMPTargetTriples.empty())
5449 const Expr *IfCond =
nullptr;
5451 IfCond = C->getCondition();
5454 const Expr *Device =
nullptr;
5456 Device = C->getDevice();
5458 OMPLexicalScope
Scope(*
this, S, OMPD_task);
5459 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond, Device);
5471 if (
const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
5472 for (
const Expr *E : LD->counters()) {
5473 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5477 VD, [&GlobLVal, &CGF]() {
return GlobLVal.
getAddress(CGF); });
5479 if (isa<OMPCapturedExprDecl>(VD)) {
5481 if (!CGF.LocalDeclMap.count(VD))
5486 if (!C->getNumForLoops())
5488 for (
unsigned I = LD->getCollapsedNumber(),
5489 E = C->getLoopNumIterations().size();
5491 if (
const auto *VD = dyn_cast<OMPCapturedExprDecl>(
5492 cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
5494 if (!CGF.LocalDeclMap.count(VD))
5504 OMPSimdLexicalScope
Scope(*
this, D);
5505 CGM.getOpenMPRuntime().emitInlinedDirective(
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
bool hasCancel() const
Return true if current directive has inner cancel directive.
void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S)
Defines the clang::ASTContext interface.
This represents '#pragma omp distribute simd' composite directive.
Expr * getNextUpperBound() const
This represents '#pragma omp master' directive.
This represents '#pragma omp task' directive.
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
Expr * getUpperBoundVariable() const
Other implicit parameter.
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
LValue getReferenceLValue(CodeGenFunction &CGF, Expr *refExpr) const
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
This represents clause 'copyin' in the '#pragma omp ...' directives.
PointerType - C99 6.7.5.1 - Pointer Declarators.
Scheduling data for loop-based OpenMP directives.
A (possibly-)qualified type.
static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S)
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
CodeGenTypes & getTypes()
ArrayRef< OMPClause * > clauses()
llvm::Type * ConvertTypeForMem(QualType T)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, int MaxLevel, int Level=0)
ArrayRef< Expr * > dependent_counters()
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Manages list of lastprivate conditional decls for the specified directive.
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
This represents clause 'in_reduction' in the '#pragma omp task' directives.
Expr * getLowerBoundVariable() const
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents '#pragma omp for simd' directive.
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
This represents 'grainsize' clause in the '#pragma omp ...' directive.
This represents '#pragma omp teams distribute parallel for' composite directive.
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
Defines the PrettyStackTraceEntry class, which is used to make crashes give more contextual informati...
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents 'if' clause in the '#pragma omp ...' directive.
This represents '#pragma omp parallel master' directive.
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
This represents 'priority' clause in the '#pragma omp ...' directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
This represents '#pragma omp target teams distribute' combined directive.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Floating point control options.
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=CSK_unspecified, Expr *TrailingRequiresClause=nullptr)
This represents '#pragma omp parallel for' directive.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
This represents '#pragma omp target teams distribute parallel for' combined directive.
Expr * getCombinedEnsureUpperBound() const
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Represents a point when we exit a loop.
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
SmallVector< const Expr *, 4 > ReductionCopies
SourceLocation getEndLoc() const
Returns ending location of directive.
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
This represents '#pragma omp target exit data' directive.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
CanQualType getCanonicalParamType(QualType T) const
Return the canonical parameter type corresponding to the specific potentially non-canonical one...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
Represents a variable declaration or definition.
This represents clause 'private' in the '#pragma omp ...' directives.
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
ArrayRef< Expr * > dependent_inits()
Extra information about a function prototype.
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
Expr * getCombinedUpperBoundVariable() const
Expr * getCalcLastIteration() const
llvm::Value * getPointer() const
unsigned getContextParamPosition() const
void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
This represents implicit clause 'flush' for the '#pragma omp flush' directive.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPParallelMasterTaskLoopSimdDirective(const OMPParallelMasterTaskLoopSimdDirective &S)
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
bool isXLHSInRHSPart() const
Return true if helper update expression has form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' and...
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
This represents 'nogroup' clause in the '#pragma omp ...' directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
This represents 'safelen' clause in the '#pragma omp ...' directive.
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
SmallVector< const Expr *, 4 > PrivateVars
Represents a struct/union/class.
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
Save/restore original map of previously emitted local vars in case when we need to duplicate emission...
One of these records is kept for each identifier that is lexed.
void EmitOMPSimdDirective(const OMPSimdDirective &S)
This represents '#pragma omp parallel' directive.
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
void EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S)
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
A C++ nested-name-specifier augmented with source location information.
This represents 'simd' clause in the '#pragma omp ...' directive.
llvm::IntegerType * Int64Ty
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
field_range fields() const
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
Expr * getEnsureUpperBound() const
SmallVector< const Expr *, 4 > LastprivateVars
SourceLocation getBeginLoc() const LLVM_READONLY
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
Represents a member of a struct/union/class.
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
CharUnits getAlignment() const
llvm::IntegerType * SizeTy
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, bool IsMonotonic)
ArrayRef< Expr * > updates()
bool isReferenceType() const
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
This represents '#pragma omp target simd' directive.
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself...
OpenMPDirectiveKind getDirectiveKind() const
SourceLocation getBeginLoc() const LLVM_READONLY
This represents '#pragma omp barrier' directive.
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc...
This represents '#pragma omp critical' directive.
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
This represents clause 'copyprivate' in the '#pragma omp ...' directives.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
This represents '#pragma omp distribute parallel for' composite directive.
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
This represents '#pragma omp teams distribute parallel for simd' composite directive.
static bool hasScalarEvaluationKind(QualType T)
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
ArrayRef< Expr * > finals()
Expr * getIsLastIterVariable() const
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
Manages list of nontemporal decls for the specified directive.
OpenMPScheduleClauseKind Schedule
CharUnits - This is an opaque type for sizes expressed in character units.
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value *> &CapturedVars)
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
llvm::function_ref< std::pair< LValue, LValue > CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
Expr * getX()
Get 'x' part of the associated expression/statement.
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
RAII for correct setting/restoring of CapturedStmtInfo.
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
CharUnits getAlignment() const
Return the alignment of this pointer.
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
Expr * getIterationVariable() const
A builtin binary operation expression such as "x + y" or "x <= y".
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
OpenMPScheduleClauseModifier M2
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
SmallVector< const Expr *, 4 > PrivateCopies
void restore(CodeGenFunction &CGF)
Restores original addresses of the variables.
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S)
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
This represents '#pragma omp cancellation point' directive.
If a crash happens while one of these objects are live, the message is printed out along with the spe...
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations...
Expr * getSizeExpr() const
Scope - A scope is a transient data structure that is used while parsing the program.
field_iterator field_begin() const
This represents 'final' clause in the '#pragma omp ...' directive.
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
void EmitOMPParallelDirective(const OMPParallelDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
This represents '#pragma omp teams' directive.
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
SmallVector< const Expr *, 4 > FirstprivateCopies
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
This represents clause 'reduction' in the '#pragma omp ...' directives.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
This represents '#pragma omp teams distribute simd' combined directive.
void ForceCleanup(std::initializer_list< llvm::Value **> ValuesToReload={})
Force the emission of cleanups now, instead of waiting until this object is destroyed.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
An ordinary object is located at an address in memory.
SmallVector< const Expr *, 4 > ReductionOps
Controls insertion of cancellation exit blocks in worksharing constructs.
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * > CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
Represents the body of a CapturedStmt, and serves as its DeclContext.
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
SmallVector< const Expr *, 4 > ReductionVars
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address >> &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value *>> &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point...
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
This represents '#pragma omp target parallel for simd' directive.
ArrayRef< Expr * > private_counters()
CompoundStmt - This represents a group of statements like { stmt stmt }.
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
ASTContext & getContext() const
bool hasCancel() const
Return true if current directive has inner cancel directive.
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
This represents '#pragma omp taskgroup' directive.
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
This represents clause 'aligned' in the '#pragma omp ...' directives.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat)
Class intended to support codegen of all kind of the reduction clauses.
bool isGlobalVarCaptured(const VarDecl *VD) const
Checks if the global variable is captured in current function.
Expr * getCombinedLowerBoundVariable() const
The class used to assign some variables some temporarily addresses.
This represents clause 'task_reduction' in the '#pragma omp taskgroup' directives.
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
This represents '#pragma omp distribute' directive.
void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S)
This represents implicit clause 'depend' for the '#pragma omp task' directive.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
unsigned NumberOfTargetItems
const Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
bool addPrivate(const VarDecl *LocalVD, const llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'proc_bind' clause in the '#pragma omp ...' directive.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
This represents one expression.
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
void EmitAutoVarInit(const AutoVarEmission &emission)
Address getAddress(CodeGenFunction &CGF) const
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
This represents 'simdlen' clause in the '#pragma omp ...' directive.
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top, if IgnoreCaptured is true.
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type, where the destination type is an LLVM scalar type.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
This represents '#pragma omp master taskloop' directive.
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
const T * castAs() const
Member-template castAs<specific type>.
llvm::BasicBlock * getBlock() const
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
bool hasCancel() const
Return true if current directive has inner cancel directive.
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
This represents '#pragma omp target teams distribute parallel for simd' combined directive.
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
OpenMPClauseKind
OpenMP clauses.
llvm::PointerType * getType() const
Return the type of the pointer value.
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause *> Clauses)
This represents '#pragma omp target teams distribute simd' combined directive.
void EmitOMPSimdInit(const OMPLoopDirective &D, bool IsMonotonic=false)
Helpers for the OpenMP loop directives.
This represents 'ordered' clause in the '#pragma omp ...' directive.
Expr * getDistInc() const
llvm::LLVMContext & getLLVMContext()
Expr * getNextLowerBound() const
Expr * getPrevEnsureUpperBound() const
This represents '#pragma omp for' directive.
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
SmallVector< const Expr *, 4 > FirstprivateVars
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
void EmitOMPMasterDirective(const OMPMasterDirective &S)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the 'for' loop has a dispatch schedule (e.g.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
This represents '#pragma omp target teams' directive.
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
SourceLocation getEnd() const
UnaryOperator - This represents the unary-expression's (except sizeof and alignof), the postinc/postdec operators from postfix-expression, and various extensions.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
This represents '#pragma omp cancel' directive.
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
const LangOptions & getLangOpts() const
ASTContext & getContext() const
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
static Stmt * tryToFindNextInnerLoop(Stmt *CurStmt, bool TryImperfectlyNestedLoops)
Try to find the next loop sub-statement in the specified statement CurStmt.
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
GlobalDecl - represents a global declaration.
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
This represents '#pragma omp flush' directive.
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind. ...
This represents '#pragma omp parallel for simd' directive.
This represents 'seq_cst' clause in the '#pragma omp atomic' directive.
This represents 'untied' clause in the '#pragma omp ...' directive.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
The l-value was considered opaque, so the alignment was determined from a type.
This represents '#pragma omp parallel master taskloop' directive.
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
Expr * getLastIteration() const
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
bool isPostfixUpdate() const
Return true if 'v' expression must be updated to original value of 'x', false if 'v' must be updated ...
This represents '#pragma omp target enter data' directive.
void EmitOMPFlushDirective(const OMPFlushDirective &S)
Expr * getStrideVariable() const
This represents '#pragma omp master taskloop simd' directive.
This represents 'num_teams' clause in the '#pragma omp ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
Expr * getCombinedDistCond() const
const Stmt * getPreInits() const
This captures a statement into a function.
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
This represents '#pragma omp single' directive.
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
This represents 'hint' clause in the '#pragma omp ...' directive.
void EmitOMPForDirective(const OMPForDirective &S)
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
Pseudo declaration for capturing expressions.
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
This is a basic class for representing single OpenMP executable directive.
This represents 'schedule' clause in the '#pragma omp ...' directive.
Expr * getExpr()
Get 'expr' part of the associated expression/statement.
llvm::Value * getPointer(CodeGenFunction &CGF) const
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
DeclarationNameInfo getDirectiveName() const
Return name of the directive.
void EmitStmt(const Stmt *S, ArrayRef< const Attr *> Attrs=None)
EmitStmt - Emit the code for the statement.
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
C-style initialization with assignment.
This represents '#pragma omp taskwait' directive.
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
This file defines OpenMP nodes for declarative directives.
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
This is a basic class for representing single OpenMP clause.
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
bool isAnyPointerType() const
virtual void Enter(CodeGenFunction &CGF)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, const RegionCodeGenTy &SimdInitGen, const RegionCodeGenTy &BodyCodeGen)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
bool isLValue() const
isLValue - True if this expression is an "l-value" according to the rules of the current language...
This represents '#pragma omp target' directive.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
Expr * getV()
Get 'v' part of the associated expression/statement.
constexpr XRayInstrMask None
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
This represents '#pragma omp ordered' directive.
This represents '#pragma omp target update' directive.
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
This represents '#pragma omp parallel master taskloop simd' directive.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
ArrayRef< Expr * > finals_conditions()
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
void setAction(PrePostActionTy &Action) const
CGFunctionInfo - Class to encapsulate the information about a function definition.
This class organizes the cross-function state that is used while generating LLVM code.
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
Dataflow Directional Tag Classes.
std::string getAsString() const
getAsString - Retrieve the human-readable string for this name.
Expr * getPrevUpperBoundVariable() const
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
This represents 'device' clause in the '#pragma omp ...' directive.
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value *> &)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
EvalResult is a struct with detailed info about an evaluated expression.
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV)
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
This represents '#pragma omp section' directive.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
OpenMPScheduleClauseModifier M1
This represents '#pragma omp teams distribute' directive.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
const Expr * getInit() const
This represents '#pragma omp simd' directive.
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
The name of a declaration.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
const Expr * getReductionRef() const
Returns reference to the task_reduction return variable.
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
SourceLocation getEndLoc() const LLVM_READONLY
This represents clause 'linear' in the '#pragma omp ...' directives.
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPCancelDirective(const OMPCancelDirective &S)
Expr * getUpdateExpr()
Get helper expression of the form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' or 'OpaqueValueExp...
SourceLocation getBeginLoc() const LLVM_READONLY
llvm::Module & getModule() const
This represents '#pragma omp atomic' directive.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
Expr * getCombinedInit() const
unsigned getCollapsedNumber() const
Get number of collapsed loops.
Expr * getCombinedNextLowerBound() const
ArrayRef< Expr * > counters()
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
SwitchStmt - This represents a 'switch' stmt.
Expr * getCombinedNextUpperBound() const
void EmitAutoVarCleanups(const AutoVarEmission &emission)
API for captured statement code generation.
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
Complex values, per C99 6.2.5p11.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, LValue LVal, RValue RVal)
This file defines OpenMP AST classes for executable directives and clauses.
ArrayRef< Expr * > inits()
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr, NonOdrUseReason NOUR=NOUR_None)
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
void EmitOMPParallelMasterTaskLoopDirective(const OMPParallelMasterTaskLoopDirective &S)
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Expr * getNumIterations() const
Base for LValueReferenceType and RValueReferenceType.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
StringRef getMangledName(GlobalDecl GD)
Address BasePointersArray
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD, Address TempAddr)
Sets the address of the variable LocalVD to be TempAddr in function CGF.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
bool hasAssociatedStmt() const
Returns true if directive has associated statement.
Expr * getPrevLowerBoundVariable() const
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
SourceManager & getSourceManager()
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
CanQualType getCanonicalType(QualType T) const
Return the canonical (structural) type corresponding to the specified potentially non-canonical type ...
bool isLValueReferenceType() const
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
TranslationUnitDecl * getTranslationUnitDecl() const
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for reduction variables.
This represents '#pragma omp target parallel' directive.
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
This represents 'nowait' clause in the '#pragma omp ...' directive.
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g., it is an signed integer type or a vector.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
This represents 'num_tasks' clause in the '#pragma omp ...' directive.
Privates[]
Gets the list of initial values for linear variables.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Expr * getCombinedCond() const
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
bool Add(InterpState &S, CodePtr OpPC)
void EmitOMPUseDevicePtrClause(const OMPClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, Address > &CaptureDeviceAddrMap)
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
This represents '#pragma omp taskloop simd' directive.
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
Expr * getPreCond() const
This represents 'dist_schedule' clause in the '#pragma omp ...' directive.
CGCapturedStmtInfo * CapturedStmtInfo
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
const VariableArrayType * getAsVariableArrayType(QualType T) const
This represents '#pragma omp sections' directive.
This represents '#pragma omp target data' directive.
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
A reference to a declared variable, function, enum, etc.
static RValue get(llvm::Value *V)
bool isPointerType() const
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
void EmitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
An l-value expression is a reference to an object with independent storage.
A trivial tuple used to represent a source range.
LValue - This represents an lvalue references.
This represents '#pragma omp taskyield' directive.
This represents '#pragma omp distribute parallel for simd' composite directive.
Represents a C array with a specified size that is not an integer-constant-expression.
This represents '#pragma omp parallel sections' directive.
const LangOptions & getLangOpts() const
bool apply(CodeGenFunction &CGF)
Applies new addresses to the list of the variables.
SourceLocation getBegin() const
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
void EmitOMPTaskDirective(const OMPTaskDirective &S)
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
bool Sub(InterpState &S, CodePtr OpPC)
This represents '#pragma omp target parallel for' directive.
SmallVector< const Expr *, 4 > FirstprivateInits
SourceLocation getLocation() const
This represents clause 'use_device_ptr' in the '#pragma omp ...' directives.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool Privatize()
Privatizes local variables previously registered as private.
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
This represents '#pragma omp taskloop' directive.