22 #include "llvm/IR/CallSite.h" 23 using namespace clang;
24 using namespace CodeGen;
31 for (
const auto *C : S.
clauses()) {
33 if (
auto *PreInit = cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
34 for (
const auto *I : PreInit->decls()) {
35 if (!I->hasAttr<OMPCaptureNoInitAttr>())
57 bool AsInlined =
false,
bool EmitPreInitStmt =
true)
61 emitPreInitStmt(CGF, S);
65 for (
auto &C : CS->captures()) {
66 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
67 auto *VD =
C.getCapturedVar();
69 "Canonical decl must be captured.");
71 isCapturedVar(CGF, VD) ||
89 class OMPParallelScope final :
public OMPLexicalScope {
99 : OMPLexicalScope(CGF, S,
101 EmitPreInitStmt(S)) {}
106 class OMPTeamsScope final :
public OMPLexicalScope {
115 : OMPLexicalScope(CGF, S,
117 EmitPreInitStmt(S)) {}
126 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
127 (void)PreCondScope.addPrivate(VD, [&CGF, VD]() {
128 return CGF.CreateMemTemp(VD->getType().getNonReferenceType());
131 (void)PreCondScope.Privatize();
132 if (
auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
133 if (
auto *PreInits = cast_or_null<DeclStmt>(LD->getPreInits())) {
134 for (
const auto *I : PreInits->decls())
143 emitPreInitStmt(CGF, S);
154 cast<BlockDecl>(CGF.
CurCodeDecl)->capturesVariable(VD));
160 InlinedShareds(CGF) {
161 for (
const auto *C : S.
clauses()) {
163 if (
auto *PreInit = cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
164 for (
const auto *I : PreInit->decls()) {
165 if (!I->hasAttr<OMPCaptureNoInitAttr>())
174 }
else if (
const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
175 for (
const Expr *E : UDP->varlists()) {
176 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
177 if (
const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
184 if (
const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
185 if (
const Expr *E = TG->getReductionRef())
186 CGF.
EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
190 for (
auto &C : CS->captures()) {
191 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
192 auto *VD =
C.getCapturedVar();
194 "Canonical decl must be captured.");
196 isCapturedVar(CGF, VD) ||
219 if (
auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
220 if (
auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
221 OrigVD = OrigVD->getCanonicalDecl();
223 LambdaCaptureFields.lookup(OrigVD) ||
224 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
225 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
226 DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), IsCaptured,
227 OrigDRE->getType(),
VK_LValue, OrigDRE->getExprLoc());
228 return EmitLValue(&DRE);
231 return EmitLValue(E);
235 auto &C = getContext();
237 auto SizeInChars = C.getTypeSizeInChars(Ty);
238 if (SizeInChars.isZero()) {
240 while (
auto *VAT = C.getAsVariableArrayType(Ty)) {
242 std::tie(ArraySize, Ty) = getVLASize(VAT);
243 Size = Size ? Builder.CreateNUWMul(Size, ArraySize) : ArraySize;
245 SizeInChars = C.getTypeSizeInChars(Ty);
246 if (SizeInChars.isZero())
247 return llvm::ConstantInt::get(SizeTy, 0);
248 Size = Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
250 Size = CGM.getSize(SizeInChars);
261 I != E; ++I, ++CurField, ++CurCap) {
262 if (CurField->hasCapturedVLAType()) {
263 auto VAT = CurField->getCapturedVLAType();
264 auto *Val = VLASizeMap[VAT->getSizeExpr()];
265 CapturedVars.push_back(Val);
266 }
else if (CurCap->capturesThis())
267 CapturedVars.push_back(CXXThisValue);
268 else if (CurCap->capturesVariableByCopy()) {
270 EmitLoadOfLValue(EmitLValue(*I),
SourceLocation()).getScalarVal();
274 if (!CurField->getType()->isAnyPointerType()) {
275 auto &Ctx = getContext();
276 auto DstAddr = CreateMemTemp(
277 Ctx.getUIntPtrType(),
278 Twine(CurCap->getCapturedVar()->getName()) +
".casted");
279 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
281 auto *SrcAddrVal = EmitScalarConversion(
282 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
285 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
293 CapturedVars.push_back(CV);
295 assert(CurCap->capturesVariable() &&
"Expected capture by reference.");
296 CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
302 StringRef Name,
LValue AddrLV,
303 bool isReferenceType =
false) {
315 if (isReferenceType) {
317 auto *RefVal = TmpAddr.getPointer();
335 if (
auto *VLA = dyn_cast<VariableArrayType>(A))
337 else if (!A->isVariablyModifiedType())
345 struct FunctionOptions {
350 const bool UIntPtrCastRequired =
true;
353 const bool RegisterCastedArgsOnly =
false;
355 const StringRef FunctionName;
356 explicit FunctionOptions(
const CapturedStmt *S,
bool UIntPtrCastRequired,
357 bool RegisterCastedArgsOnly,
358 StringRef FunctionName)
359 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
360 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
361 FunctionName(FunctionName) {}
367 llvm::MapVector<
const Decl *, std::pair<const VarDecl *, Address>>
369 llvm::DenseMap<
const Decl *, std::pair<const Expr *, llvm::Value *>>
371 llvm::Value *&CXXThisValue,
const FunctionOptions &FO) {
373 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
374 assert(CD->
hasBody() &&
"missing CapturedDecl body");
376 CXXThisValue =
nullptr;
386 auto I = FO.S->captures().begin();
388 if (!FO.UIntPtrCastRequired) {
397 for (
auto *FD : RD->
fields()) {
408 I->capturesVariableArrayType()) {
409 if (FO.UIntPtrCastRequired)
413 if (I->capturesVariable() || I->capturesVariableByCopy()) {
414 CapVar = I->getCapturedVar();
416 }
else if (I->capturesThis())
419 assert(I->capturesVariableArrayType());
425 if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
427 Ctx, DebugFunctionDecl,
428 CapVar ? CapVar->
getLocStart() : FD->getLocStart(),
429 CapVar ? CapVar->
getLocation() : FD->getLocation(), II, ArgType,
435 Args.emplace_back(Arg);
437 TargetArgs.emplace_back(
438 FO.UIntPtrCastRequired
460 F->setDoesNotThrow();
466 I = FO.S->captures().begin();
467 for (
auto *FD : RD->
fields()) {
470 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
478 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
479 const VarDecl *CurVD = I->getCapturedVar();
488 if (!FO.RegisterCastedArgsOnly)
489 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
497 if (FD->hasCapturedVLAType()) {
498 if (FO.UIntPtrCastRequired) {
500 Args[Cnt]->getName(),
506 auto VAT = FD->getCapturedVLAType();
507 VLASizes.insert({Args[Cnt], {VAT->getSizeExpr(), ExprArg}});
508 }
else if (I->capturesVariable()) {
509 auto *Var = I->getCapturedVar();
521 if (!FO.RegisterCastedArgsOnly) {
526 }
else if (I->capturesVariableByCopy()) {
527 assert(!FD->getType()->isAnyPointerType() &&
528 "Not expecting a captured pointer.");
529 auto *Var = I->getCapturedVar();
534 FO.UIntPtrCastRequired
540 assert(I->capturesThis());
543 LocalAddrs.insert({Args[Cnt], {
nullptr, ArgLVal.
getAddress()}});
556 "CapturedStmtInfo should be set when generating the captured function");
559 bool NeedWrapperFunction =
563 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
564 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
566 llvm::raw_svector_ostream Out(Buffer);
567 Out << CapturedStmtInfo->getHelperName();
568 if (NeedWrapperFunction)
570 FunctionOptions FO(&S, !NeedWrapperFunction,
false,
573 VLASizes, CXXThisValue, FO);
574 for (
const auto &LocalAddrPair : LocalAddrs) {
575 if (LocalAddrPair.second.first) {
576 setAddrOfLocalVar(LocalAddrPair.second.first,
577 LocalAddrPair.second.second);
580 for (
const auto &VLASizePair : VLASizes)
581 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
583 CapturedStmtInfo->EmitBody(*
this, CD->
getBody());
585 if (!NeedWrapperFunction)
588 FunctionOptions WrapperFO(&S,
true,
590 CapturedStmtInfo->getHelperName());
595 llvm::Function *WrapperF =
597 WrapperCGF.CXXThisValue, WrapperFO);
599 for (
const auto *Arg : Args) {
601 auto I = LocalAddrs.find(Arg);
602 if (I != LocalAddrs.end()) {
603 LValue LV = WrapperCGF.MakeAddrLValue(
605 I->second.first ? I->second.first->getType() : Arg->getType(),
609 auto EI = VLASizes.find(Arg);
610 if (EI != VLASizes.end())
611 CallArg = EI->second.second;
613 LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
619 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(
CallArg, Arg->getType()));
621 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.
getLocStart(),
623 WrapperCGF.FinishFunction();
638 auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
639 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.
getElementType());
644 auto DestEnd = Builder.CreateGEP(DestBegin, NumElements);
646 auto BodyBB = createBasicBlock(
"omp.arraycpy.body");
647 auto DoneBB = createBasicBlock(
"omp.arraycpy.done");
649 Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arraycpy.isempty");
650 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
653 auto EntryBB = Builder.GetInsertBlock();
656 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
658 llvm::PHINode *SrcElementPHI =
659 Builder.CreatePHI(SrcBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
660 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
665 llvm::PHINode *DestElementPHI =
666 Builder.CreatePHI(DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
667 DestElementPHI->addIncoming(DestBegin, EntryBB);
673 CopyGen(DestElementCurrent, SrcElementCurrent);
676 auto DestElementNext = Builder.CreateConstGEP1_32(
677 DestElementPHI, 1,
"omp.arraycpy.dest.element");
678 auto SrcElementNext = Builder.CreateConstGEP1_32(
679 SrcElementPHI, 1,
"omp.arraycpy.src.element");
682 Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
683 Builder.CreateCondBr(Done, DoneBB, BodyBB);
684 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
685 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
688 EmitBlock(DoneBB,
true);
696 if (BO && BO->getOpcode() == BO_Assign) {
698 EmitAggregateAssign(DestAddr, SrcAddr, OriginalType);
702 EmitOMPAggregateAssign(
703 DestAddr, SrcAddr, OriginalType,
704 [
this, Copy, SrcVD, DestVD](
Address DestElement,
Address SrcElement) {
713 SrcVD, [SrcElement]() ->
Address {
return SrcElement; });
715 EmitIgnoredExpr(Copy);
725 EmitIgnoredExpr(Copy);
731 if (!HaveInsertPoint())
733 bool FirstprivateIsLastprivate =
false;
736 for (
const auto *D : C->varlists())
743 auto IRef = C->varlist_begin();
744 auto InitsRef = C->inits().begin();
745 for (
auto IInit : C->private_copies()) {
746 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
747 bool ThisFirstprivateIsLastprivate =
748 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
749 auto *CapFD = CapturesInfo.
lookup(OrigVD);
750 auto *FD = CapturedStmtInfo->lookup(OrigVD);
751 if (!ThisFirstprivateIsLastprivate && FD && (FD == CapFD) &&
752 !FD->getType()->isReferenceType()) {
753 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
758 FirstprivateIsLastprivate =
759 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
760 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
761 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
762 auto *VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
766 (*IRef)->getType(),
VK_LValue, (*IRef)->getExprLoc());
767 Address OriginalAddr = EmitLValue(&DRE).getAddress();
774 auto Emission = EmitAutoVarAlloca(*VD);
776 if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
778 EmitAggregateAssign(Emission.getAllocatedAddress(), OriginalAddr,
781 EmitOMPAggregateAssign(
782 Emission.getAllocatedAddress(), OriginalAddr,
Type,
783 [
this, VDInit, Init](
Address DestElement,
788 setAddrOfLocalVar(VDInit, SrcElement);
789 EmitAnyExprToMem(Init, DestElement,
790 Init->getType().getQualifiers(),
792 LocalDeclMap.erase(VDInit);
795 EmitAutoVarCleanups(Emission);
796 return Emission.getAllocatedAddress();
804 setAddrOfLocalVar(VDInit, OriginalAddr);
806 LocalDeclMap.erase(VDInit);
807 return GetAddrOfLocalVar(VD);
810 assert(IsRegistered &&
811 "firstprivate var already registered as private");
819 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
825 if (!HaveInsertPoint())
829 auto IRef = C->varlist_begin();
830 for (
auto IInit : C->private_copies()) {
831 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
832 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
833 auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
838 return GetAddrOfLocalVar(VD);
840 assert(IsRegistered &&
"private var already registered as private");
850 if (!HaveInsertPoint())
857 llvm::BasicBlock *CopyBegin =
nullptr, *CopyEnd =
nullptr;
859 auto IRef = C->varlist_begin();
860 auto ISrcRef = C->source_exprs().begin();
861 auto IDestRef = C->destination_exprs().begin();
862 for (
auto *AssignOp : C->assignment_ops()) {
863 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
870 if (getLangOpts().OpenMPUseTLS &&
871 getContext().getTargetInfo().isTLSSupported()) {
872 assert(CapturedStmtInfo->lookup(VD) &&
873 "Copyin threadprivates should have been captured!");
874 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
true, (*IRef)->getType(),
876 MasterAddr = EmitLValue(&DRE).getAddress();
877 LocalDeclMap.erase(VD);
881 : CGM.GetAddrOfGlobal(VD),
882 getContext().getDeclAlign(VD));
885 Address PrivateAddr = EmitLValue(*IRef).getAddress();
886 if (CopiedVars.size() == 1) {
889 CopyBegin = createBasicBlock(
"copyin.not.master");
890 CopyEnd = createBasicBlock(
"copyin.not.master.end");
891 Builder.CreateCondBr(
892 Builder.CreateICmpNE(
893 Builder.CreatePtrToInt(MasterAddr.
getPointer(), CGM.IntPtrTy),
894 Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)),
896 EmitBlock(CopyBegin);
898 auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
899 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
900 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
909 EmitBlock(CopyEnd,
true);
917 if (!HaveInsertPoint())
919 bool HasAtLeastOneLastprivate =
false;
922 auto *LoopDirective = cast<OMPLoopDirective>(&D);
923 for (
auto *C : LoopDirective->counters()) {
930 HasAtLeastOneLastprivate =
true;
932 !getLangOpts().OpenMPSimd)
934 auto IRef = C->varlist_begin();
935 auto IDestRef = C->destination_exprs().begin();
936 for (
auto *IInit : C->private_copies()) {
939 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
942 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
943 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
946 const_cast<VarDecl *>(OrigVD),
947 CapturedStmtInfo->lookup(
949 (*IRef)->getType(),
VK_LValue, (*IRef)->getExprLoc());
950 return EmitLValue(&DRE).getAddress();
955 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
956 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
960 return GetAddrOfLocalVar(VD);
962 assert(IsRegistered &&
963 "lastprivate var already registered as private");
971 return HasAtLeastOneLastprivate;
977 if (!HaveInsertPoint())
985 llvm::BasicBlock *ThenBB =
nullptr;
986 llvm::BasicBlock *DoneBB =
nullptr;
987 if (IsLastIterCond) {
988 ThenBB = createBasicBlock(
".omp.lastprivate.then");
989 DoneBB = createBasicBlock(
".omp.lastprivate.done");
990 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
994 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
995 if (
auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
996 auto IC = LoopDirective->counters().begin();
997 for (
auto F : LoopDirective->finals()) {
1001 AlreadyEmittedVars.insert(D);
1003 LoopCountersAndUpdates[D] = F;
1008 auto IRef = C->varlist_begin();
1009 auto ISrcRef = C->source_exprs().begin();
1010 auto IDestRef = C->destination_exprs().begin();
1011 for (
auto *AssignOp : C->assignment_ops()) {
1012 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1014 auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1015 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1019 if (
auto *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1020 EmitIgnoredExpr(FinalExpr);
1021 auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1022 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1024 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1026 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1027 if (
auto RefTy = PrivateVD->getType()->getAs<
ReferenceType>())
1029 Address(Builder.CreateLoad(PrivateAddr),
1030 getNaturalTypeAlignment(RefTy->getPointeeType()));
1031 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1037 if (
auto *PostUpdate = C->getPostUpdateExpr())
1038 EmitIgnoredExpr(PostUpdate);
1041 EmitBlock(DoneBB,
true);
1047 if (!HaveInsertPoint())
1055 auto IPriv = C->privates().begin();
1056 auto IRed = C->reduction_ops().begin();
1057 auto ILHS = C->lhs_exprs().begin();
1058 auto IRHS = C->rhs_exprs().begin();
1059 for (
const auto *Ref : C->varlists()) {
1060 Shareds.emplace_back(Ref);
1061 Privates.emplace_back(*IPriv);
1062 ReductionOps.emplace_back(*IRed);
1063 LHSs.emplace_back(*ILHS);
1064 RHSs.emplace_back(*IRHS);
1065 std::advance(IPriv, 1);
1066 std::advance(IRed, 1);
1067 std::advance(ILHS, 1);
1068 std::advance(IRHS, 1);
1073 auto ILHS = LHSs.begin();
1074 auto IRHS = RHSs.begin();
1075 auto IPriv = Privates.begin();
1076 for (
const auto *IRef : Shareds) {
1077 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1079 RedCG.emitSharedLValue(*
this, Count);
1081 auto Emission = EmitAutoVarAlloca(*PrivateVD);
1088 EmitAutoVarCleanups(Emission);
1090 *
this, Count, Emission.getAllocatedAddress());
1093 assert(IsRegistered &&
"private var already registered as private");
1097 auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1098 auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1100 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1101 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1108 return GetAddrOfLocalVar(PrivateVD);
1110 }
else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1111 isa<ArraySubscriptExpr>(IRef)) {
1118 return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1119 ConvertTypeForMem(RHSVD->getType()),
1123 QualType Type = PrivateVD->getType();
1124 bool IsArray = getContext().getAsArrayType(Type) !=
nullptr;
1129 OriginalAddr = Builder.CreateElementBitCast(
1130 OriginalAddr, ConvertTypeForMem(LHSVD->getType()),
"lhs.begin");
1133 LHSVD, [OriginalAddr]() ->
Address {
return OriginalAddr; });
1135 RHSVD, [
this, PrivateVD, RHSVD, IsArray]() ->
Address {
1137 ? Builder.CreateElementBitCast(
1138 GetAddrOfLocalVar(PrivateVD),
1139 ConvertTypeForMem(RHSVD->getType()),
"rhs.begin")
1140 : GetAddrOfLocalVar(PrivateVD);
1152 if (!HaveInsertPoint())
1158 bool HasAtLeastOneReduction =
false;
1160 HasAtLeastOneReduction =
true;
1161 Privates.append(C->privates().begin(), C->privates().end());
1162 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1163 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1164 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1166 if (HasAtLeastOneReduction) {
1169 ReductionKind == OMPD_simd;
1170 bool SimpleReduction = ReductionKind == OMPD_simd;
1173 CGM.getOpenMPRuntime().emitReduction(
1175 {WithNowait, SimpleReduction, ReductionKind});
1184 llvm::BasicBlock *DoneBB =
nullptr;
1186 if (
auto *PostUpdate = C->getPostUpdateExpr()) {
1188 if (
auto *Cond = CondGen(CGF)) {
1193 CGF.
Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1211 CodeGenBoundParametersTy;
1217 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1223 auto NumThreads = CGF.
EmitScalarExpr(NumThreadsClause->getNumThreads(),
1226 CGF, NumThreads, NumThreadsClause->getLocStart());
1231 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
1233 const Expr *IfCond =
nullptr;
1236 C->getNameModifier() == OMPD_parallel) {
1237 IfCond = C->getCondition();
1242 OMPParallelScope
Scope(CGF, S);
1248 CodeGenBoundParameters(CGF, S, CapturedVars);
1251 CapturedVars, IfCond);
1296 for (
auto *U : C->updates())
1302 auto Continue = getJumpDestInCurrentScope(
"omp.body.continue");
1303 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1307 EmitBlock(Continue.getBlock());
1308 BreakContinueStack.pop_back();
1312 const Stmt &S,
bool RequiresCleanup,
const Expr *LoopCond,
1313 const Expr *IncExpr,
1316 auto LoopExit = getJumpDestInCurrentScope(
"omp.inner.for.end");
1319 auto CondBlock = createBasicBlock(
"omp.inner.for.cond");
1320 EmitBlock(CondBlock);
1322 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.
getBegin()),
1323 SourceLocToDebugLoc(R.
getEnd()));
1327 auto ExitBlock =
LoopExit.getBlock();
1328 if (RequiresCleanup)
1329 ExitBlock = createBasicBlock(
"omp.inner.for.cond.cleanup");
1331 auto LoopBody = createBasicBlock(
"omp.inner.for.body");
1334 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1335 if (ExitBlock !=
LoopExit.getBlock()) {
1336 EmitBlock(ExitBlock);
1337 EmitBranchThroughCleanup(
LoopExit);
1340 EmitBlock(LoopBody);
1341 incrementProfileCounter(&S);
1344 auto Continue = getJumpDestInCurrentScope(
"omp.inner.for.inc");
1345 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
1350 EmitBlock(Continue.getBlock());
1351 EmitIgnoredExpr(IncExpr);
1353 BreakContinueStack.pop_back();
1354 EmitBranch(CondBlock);
1361 if (!HaveInsertPoint())
1364 bool HasLinears =
false;
1366 for (
auto *Init : C->inits()) {
1368 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1371 auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1373 CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
1379 EmitAutoVarCleanups(Emission);
1385 if (
auto CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1386 if (
auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1387 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1389 EmitIgnoredExpr(CS);
1398 if (!HaveInsertPoint())
1400 llvm::BasicBlock *DoneBB =
nullptr;
1403 auto IC = C->varlist_begin();
1404 for (
auto *F : C->finals()) {
1406 if (
auto *Cond = CondGen(*
this)) {
1409 auto *ThenBB = createBasicBlock(
".omp.linear.pu");
1410 DoneBB = createBasicBlock(
".omp.linear.pu.done");
1411 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1415 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1417 CapturedStmtInfo->lookup(OrigVD) !=
nullptr,
1418 (*IC)->getType(),
VK_LValue, (*IC)->getExprLoc());
1419 Address OrigAddr = EmitLValue(&DRE).getAddress();
1426 if (
auto *PostUpdate = C->getPostUpdateExpr())
1427 EmitIgnoredExpr(PostUpdate);
1430 EmitBlock(DoneBB,
true);
1438 unsigned ClauseAlignment = 0;
1439 if (
auto AlignmentExpr = Clause->getAlignment()) {
1442 ClauseAlignment =
static_cast<unsigned>(AlignmentCI->getZExtValue());
1444 for (
auto E : Clause->varlists()) {
1445 unsigned Alignment = ClauseAlignment;
1446 if (Alignment == 0) {
1453 E->getType()->getPointeeType()))
1456 assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1457 "alignment is not power of 2");
1458 if (Alignment != 0) {
1468 if (!HaveInsertPoint())
1472 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1473 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1476 if (!LocalDeclMap.count(PrivateVD)) {
1477 auto VarEmission = EmitAutoVarAlloca(*PrivateVD);
1478 EmitAutoVarCleanups(VarEmission);
1482 (*I)->getType(),
VK_LValue, (*I)->getExprLoc());
1483 return EmitLValue(&DRE).getAddress();
1485 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1488 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
1489 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1490 E->getType(), VK_LValue, E->getExprLoc());
1491 return EmitLValue(&DRE).getAddress();
1499 const Expr *Cond, llvm::BasicBlock *TrueBlock,
1500 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1508 for (
auto I : S.
inits()) {
1518 if (!HaveInsertPoint())
1522 auto *LoopDirective = cast<OMPLoopDirective>(&D);
1523 for (
auto *C : LoopDirective->counters()) {
1529 auto CurPrivate = C->privates().begin();
1530 for (
auto *E : C->varlists()) {
1531 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1533 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1537 EmitVarDecl(*PrivateVD);
1538 return GetAddrOfLocalVar(PrivateVD);
1540 assert(IsRegistered &&
"linear var already registered as private");
1544 EmitVarDecl(*PrivateVD);
1558 llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
1568 llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
1580 LoopStack.setParallel(!IsMonotonic);
1581 LoopStack.setVectorizeEnable(
true);
1588 if (!HaveInsertPoint())
1590 llvm::BasicBlock *DoneBB =
nullptr;
1593 for (
auto F : D.
finals()) {
1594 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1595 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1597 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1598 OrigVD->hasGlobalStorage() || CED) {
1600 if (
auto *Cond = CondGen(*
this)) {
1603 auto *ThenBB = createBasicBlock(
".omp.final.then");
1604 DoneBB = createBasicBlock(
".omp.final.done");
1605 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1611 OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1615 (*IPC)->getType(),
VK_LValue, (*IPC)->getExprLoc());
1616 OrigAddr = EmitLValue(&DRE).getAddress();
1620 [OrigAddr]() ->
Address {
return OrigAddr; });
1628 EmitBlock(DoneBB,
true);
1641 auto VDecl = cast<VarDecl>(Helper->
getDecl());
1650 "Expected simd directive");
1651 OMPLoopScope PreInitScope(CGF, S);
1668 llvm::BasicBlock *ContBlock =
nullptr;
1683 const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1691 CGF.
EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1718 if (HasLastprivateClause)
1737 OMPLexicalScope
Scope(*
this, S,
true);
1738 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd, CodeGen);
1741 void CodeGenFunction::EmitOMPOuterLoop(
1744 const CodeGenFunction::OMPLoopArguments &LoopArgs,
1747 auto &RT = CGM.getOpenMPRuntime();
1750 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
1753 auto LoopExit = getJumpDestInCurrentScope(
"omp.dispatch.end");
1756 auto CondBlock = createBasicBlock(
"omp.dispatch.cond");
1757 EmitBlock(CondBlock);
1759 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.
getBegin()),
1760 SourceLocToDebugLoc(R.
getEnd()));
1763 if (!DynamicOrOrdered) {
1767 EmitIgnoredExpr(LoopArgs.EUB);
1769 EmitIgnoredExpr(LoopArgs.Init);
1771 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1774 RT.emitForNext(*
this, S.
getLocStart(), IVSize, IVSigned, LoopArgs.IL,
1775 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1780 auto ExitBlock =
LoopExit.getBlock();
1782 ExitBlock = createBasicBlock(
"omp.dispatch.cleanup");
1784 auto LoopBody = createBasicBlock(
"omp.dispatch.body");
1785 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1786 if (ExitBlock !=
LoopExit.getBlock()) {
1787 EmitBlock(ExitBlock);
1788 EmitBranchThroughCleanup(
LoopExit);
1790 EmitBlock(LoopBody);
1794 if (DynamicOrOrdered)
1795 EmitIgnoredExpr(LoopArgs.Init);
1798 auto Continue = getJumpDestInCurrentScope(
"omp.dispatch.inc");
1799 BreakContinueStack.push_back(BreakContinue(
LoopExit, Continue));
1804 LoopStack.setParallel(!IsMonotonic);
1806 EmitOMPSimdInit(S, IsMonotonic);
1818 CodeGenLoop(CGF, S, LoopExit);
1821 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1824 EmitBlock(Continue.getBlock());
1825 BreakContinueStack.pop_back();
1826 if (!DynamicOrOrdered) {
1828 EmitIgnoredExpr(LoopArgs.NextLB);
1829 EmitIgnoredExpr(LoopArgs.NextUB);
1832 EmitBranch(CondBlock);
1835 EmitBlock(LoopExit.getBlock());
1839 if (!DynamicOrOrdered)
1846 void CodeGenFunction::EmitOMPForOuterLoop(
1849 const OMPLoopArguments &LoopArgs,
1851 auto &RT = CGM.getOpenMPRuntime();
1854 const bool DynamicOrOrdered =
1855 Ordered || RT.isDynamic(ScheduleKind.
Schedule);
1858 !RT.isStaticNonchunked(ScheduleKind.
Schedule,
1859 LoopArgs.Chunk !=
nullptr)) &&
1860 "static non-chunked schedule does not need outer loop");
1913 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
1916 if (DynamicOrOrdered) {
1917 auto DispatchBounds = CGDispatchBounds(*
this, S, LoopArgs.LB, LoopArgs.UB);
1920 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1922 RT.emitForDispatchInit(*
this, S.
getLocStart(), ScheduleKind, IVSize,
1923 IVSigned, Ordered, DipatchRTInputValues);
1925 CGOpenMPRuntime::StaticRTInput StaticInit(
1926 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1927 LoopArgs.ST, LoopArgs.Chunk);
1929 ScheduleKind, StaticInit);
1933 const unsigned IVSize,
1934 const bool IVSigned) {
1941 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1942 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1943 OuterLoopArgs.IncExpr = S.
getInc();
1944 OuterLoopArgs.Init = S.
getInit();
1945 OuterLoopArgs.Cond = S.
getCond();
1948 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1953 const unsigned IVSize,
const bool IVSigned) {}
1955 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1957 OMPPrivateScope &LoopScope,
const OMPLoopArguments &LoopArgs,
1958 const CodeGenLoopTy &CodeGenLoopContent) {
1960 auto &RT = CGM.getOpenMPRuntime();
1968 const unsigned IVSize = getContext().getTypeSize(IVExpr->
getType());
1971 CGOpenMPRuntime::StaticRTInput StaticInit(
1972 IVSize, IVSigned,
false, LoopArgs.IL, LoopArgs.LB,
1973 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
1974 RT.emitDistributeStaticInit(*
this, S.
getLocStart(), ScheduleKind, StaticInit);
1987 OMPLoopArguments OuterLoopArgs;
1988 OuterLoopArgs.LB = LoopArgs.LB;
1989 OuterLoopArgs.UB = LoopArgs.UB;
1990 OuterLoopArgs.ST = LoopArgs.ST;
1991 OuterLoopArgs.IL = LoopArgs.IL;
1992 OuterLoopArgs.Chunk = LoopArgs.Chunk;
1996 OuterLoopArgs.IncExpr = IncExpr;
2010 EmitOMPOuterLoop(
false,
false, S,
2011 LoopScope, OuterLoopArgs, CodeGenLoopContent,
2015 static std::pair<LValue, LValue>
2054 static std::pair<llvm::Value *, llvm::Value *>
2064 QualType IteratorTy = IVExpr->getType();
2069 return {LBVal, UBVal};
2075 const auto &Dir = cast<OMPLoopDirective>(S);
2077 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2078 auto LBCast = CGF.
Builder.CreateIntCast(
2080 CapturedVars.push_back(LBCast);
2082 CGF.
EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2084 auto UBCast = CGF.
Builder.CreateIntCast(
2086 CapturedVars.push_back(UBCast);
2095 bool HasCancel =
false;
2097 if (
const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2098 HasCancel = D->hasCancel();
2099 else if (
const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2100 HasCancel = D->hasCancel();
2101 else if (
const auto *D =
2102 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2103 HasCancel = D->hasCancel();
2115 CGInlinedWorksharingLoop,
2125 OMPLexicalScope
Scope(*
this, S,
true);
2126 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute, CodeGen);
2135 OMPLexicalScope
Scope(*
this, S,
true);
2136 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute, CodeGen);
2144 OMPLexicalScope
Scope(*
this, S,
true);
2145 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd, CodeGen);
2155 llvm::Constant *Addr;
2158 S, ParentName, Fn, Addr,
true, CodeGen);
2159 assert(Fn && Addr &&
"Target device function emission failed.");
2172 OMPLexicalScope
Scope(*
this, S,
true);
2173 CGM.getOpenMPRuntime().emitInlinedDirective(
2174 *
this, OMPD_target_teams_distribute_parallel_for,
2183 OMPLexicalScope
Scope(*
this, S,
true);
2184 CGM.getOpenMPRuntime().emitInlinedDirective(
2185 *
this, OMPD_target_teams_distribute_parallel_for_simd,
2193 struct ScheduleKindModifiersTy {
2200 :
Kind(Kind), M1(M1), M2(M2) {}
2210 auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
2211 EmitVarDecl(*IVDecl);
2217 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2222 auto &RT = CGM.getOpenMPRuntime();
2224 bool HasLastprivateClause;
2227 OMPLoopScope PreInitScope(*
this, S);
2232 llvm::BasicBlock *ContBlock =
nullptr;
2233 if (ConstantFoldsToSimpleInteger(S.
getPreCond(), CondConstant)) {
2237 auto *ThenBlock = createBasicBlock(
"omp.precond.then");
2238 ContBlock = createBasicBlock(
"omp.precond.end");
2240 getProfileCount(&S));
2241 EmitBlock(ThenBlock);
2242 incrementProfileCounter(&S);
2245 bool Ordered =
false;
2247 if (OrderedClause->getNumForLoops())
2248 RT.emitDoacrossInit(*
this, S);
2255 bool HasLinears = EmitOMPLinearClauseInit(S);
2258 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*
this, S);
2259 LValue LB = Bounds.first;
2260 LValue UB = Bounds.second;
2269 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2273 CGM.getOpenMPRuntime().emitBarrierCall(
2277 EmitOMPPrivateClause(S, LoopScope);
2278 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2279 EmitOMPReductionClauseInit(S, LoopScope);
2280 EmitOMPPrivateLoopCounters(S, LoopScope);
2281 EmitOMPLinearClause(S, LoopScope);
2288 ScheduleKind.
Schedule = C->getScheduleKind();
2289 ScheduleKind.
M1 = C->getFirstScheduleModifier();
2290 ScheduleKind.
M2 = C->getSecondScheduleModifier();
2291 if (
const auto *Ch = C->getChunkSize()) {
2292 Chunk = EmitScalarExpr(Ch);
2293 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
2298 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2299 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2304 if (RT.isStaticNonchunked(ScheduleKind.
Schedule,
2305 Chunk !=
nullptr) &&
2308 EmitOMPSimdInit(S,
true);
2314 CGOpenMPRuntime::StaticRTInput StaticInit(
2318 ScheduleKind, StaticInit);
2320 getJumpDestInCurrentScope(createBasicBlock(
"omp.loop.exit"));
2341 const bool IsMonotonic =
2342 Ordered || ScheduleKind.
Schedule == OMPC_SCHEDULE_static ||
2344 ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2345 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2351 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2352 LoopArguments, CGDispatchBounds);
2357 return CGF.
Builder.CreateIsNotNull(
2361 EmitOMPReductionClauseFinal(
2363 ? OMPD_parallel_for_simd
2368 return CGF.
Builder.CreateIsNotNull(
2372 if (HasLastprivateClause)
2373 EmitOMPLastprivateClauseFinal(
2375 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.
getLocStart())));
2378 return CGF.
Builder.CreateIsNotNull(
2383 EmitBranch(ContBlock);
2384 EmitBlock(ContBlock,
true);
2387 return HasLastprivateClause;
2393 static std::pair<LValue, LValue>
2407 static std::pair<llvm::Value *, llvm::Value *>
2415 return {LBVal, UBVal};
2419 bool HasLastprivates =
false;
2428 OMPLexicalScope
Scope(*
this, S,
true);
2429 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_for, CodeGen,
2435 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getLocStart(), OMPD_for);
2440 bool HasLastprivates =
false;
2448 OMPLexicalScope
Scope(*
this, S,
true);
2449 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_simd, CodeGen);
2454 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getLocStart(), OMPD_for);
2470 bool HasLastprivates =
false;
2473 auto &C = CGF.CGM.getContext();
2474 auto KmpInt32Ty = C.getIntTypeForBitwidth(32, 1);
2477 CGF.Builder.getInt32(0));
2478 auto *GlobalUBVal = CS !=
nullptr ? CGF.Builder.getInt32(CS->size() - 1)
2479 : CGF.Builder.getInt32(0);
2483 CGF.Builder.getInt32(1));
2485 CGF.Builder.getInt32(0));
2510 auto *ExitBB = CGF.createBasicBlock(
".omp.sections.exit");
2512 CGF.EmitLoadOfLValue(IV, S.
getLocStart()).getScalarVal(), ExitBB,
2513 CS ==
nullptr ? 1 : CS->size());
2515 unsigned CaseNumber = 0;
2516 for (
auto *SubStmt : CS->children()) {
2517 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
2518 CGF.EmitBlock(CaseBB);
2519 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2520 CGF.EmitStmt(SubStmt);
2521 CGF.EmitBranch(ExitBB);
2525 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
2526 CGF.EmitBlock(CaseBB);
2527 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2529 CGF.EmitBranch(ExitBB);
2531 CGF.EmitBlock(ExitBB,
true);
2535 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2539 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2543 CGF.EmitOMPPrivateClause(S, LoopScope);
2544 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2545 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2550 ScheduleKind.
Schedule = OMPC_SCHEDULE_static;
2551 CGOpenMPRuntime::StaticRTInput StaticInit(
2552 32,
true,
false, IL.getAddress(),
2554 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2555 CGF, S.
getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2557 auto *UBVal = CGF.EmitLoadOfScalar(UB, S.
getLocStart());
2558 auto *MinUBGlobalUB = CGF.Builder.CreateSelect(
2559 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2560 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2562 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.
getLocStart()), IV);
2564 CGF.EmitOMPInnerLoop(S,
false, &Cond, &Inc, BodyGen,
2568 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.
getLocEnd(),
2569 S.getDirectiveKind());
2571 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2572 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
2576 return CGF.
Builder.CreateIsNotNull(
2581 if (HasLastprivates)
2588 bool HasCancel =
false;
2589 if (
auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2590 HasCancel = OSD->hasCancel();
2591 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2592 HasCancel = OPSD->hasCancel();
2594 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_sections, CodeGen,
2602 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getLocStart(),
2609 OMPLexicalScope
Scope(*
this, S,
true);
2614 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getLocStart(),
2623 OMPLexicalScope
Scope(*
this, S,
true);
2624 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_section, CodeGen,
2638 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2639 DestExprs.append(C->destination_exprs().begin(),
2640 C->destination_exprs().end());
2641 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2642 AssignmentOps.append(C->assignment_ops().begin(),
2643 C->assignment_ops().end());
2655 OMPLexicalScope
Scope(*
this, S,
true);
2656 CGM.getOpenMPRuntime().emitSingleRegion(*
this, CodeGen, S.
getLocStart(),
2657 CopyprivateVars, DestExprs,
2658 SrcExprs, AssignmentOps);
2663 CGM.getOpenMPRuntime().emitBarrierCall(
2674 OMPLexicalScope
Scope(*
this, S,
true);
2675 CGM.getOpenMPRuntime().emitMasterRegion(*
this, CodeGen, S.
getLocStart());
2683 Expr *Hint =
nullptr;
2685 Hint = HintClause->getHint();
2686 OMPLexicalScope
Scope(*
this, S,
true);
2687 CGM.getOpenMPRuntime().emitCriticalRegion(*
this,
2722 CGF.EmitSections(S);
2734 auto *I = CS->getCapturedDecl()->param_begin();
2735 auto *PartId = std::next(I);
2736 auto *TaskT = std::next(I, 4);
2741 auto *Cond = Clause->getCondition();
2743 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2744 Data.
Final.setInt(CondConstant);
2746 Data.
Final.setPointer(EvaluateExprAsBool(Cond));
2749 Data.
Final.setInt(
false);
2753 auto *Prio = Clause->getPriority();
2755 Data.
Priority.setPointer(EmitScalarConversion(
2756 EmitScalarExpr(Prio), Prio->getType(),
2757 getContext().getIntTypeForBitwidth(32, 1),
2758 Prio->getExprLoc()));
2765 auto IRef = C->varlist_begin();
2766 for (
auto *IInit : C->private_copies()) {
2767 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2768 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2775 EmittedAsPrivate.clear();
2778 auto IRef = C->varlist_begin();
2779 auto IElemInitRef = C->inits().begin();
2780 for (
auto *IInit : C->private_copies()) {
2781 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2782 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2792 llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2794 auto IRef = C->varlist_begin();
2795 auto ID = C->destination_exprs().begin();
2796 for (
auto *IInit : C->private_copies()) {
2797 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2798 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2802 LastprivateDstsOrigs.insert(
2803 {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2804 cast<DeclRefExpr>(*IRef)});
2812 auto IPriv = C->privates().begin();
2813 auto IRed = C->reduction_ops().begin();
2814 auto ILHS = C->lhs_exprs().begin();
2815 auto IRHS = C->rhs_exprs().begin();
2816 for (
const auto *Ref : C->varlists()) {
2820 LHSs.emplace_back(*ILHS);
2821 RHSs.emplace_back(*IRHS);
2822 std::advance(IPriv, 1);
2823 std::advance(IRed, 1);
2824 std::advance(ILHS, 1);
2825 std::advance(IRHS, 1);
2828 Data.
Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2832 for (
auto *IRef : C->varlists())
2833 Data.
Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
2834 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs](
2840 enum { PrivatesParam = 2, CopyFnParam = 3 };
2841 auto *CopyFn = CGF.Builder.CreateLoad(
2842 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
2843 auto *PrivatesPtr = CGF.Builder.CreateLoad(
2844 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
2848 CallArgs.push_back(PrivatesPtr);
2850 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2851 Address PrivatePtr = CGF.CreateMemTemp(
2852 CGF.getContext().getPointerType(E->getType()),
".priv.ptr.addr");
2853 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2854 CallArgs.push_back(PrivatePtr.getPointer());
2857 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2859 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2860 ".firstpriv.ptr.addr");
2861 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2862 CallArgs.push_back(PrivatePtr.getPointer());
2865 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2867 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2868 ".lastpriv.ptr.addr");
2869 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2870 CallArgs.push_back(PrivatePtr.getPointer());
2872 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
2874 for (
auto &&Pair : LastprivateDstsOrigs) {
2875 auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2877 const_cast<VarDecl *>(OrigVD),
2878 CGF.CapturedStmtInfo->lookup(
2880 Pair.second->getType(),
VK_LValue, Pair.second->getExprLoc());
2881 Scope.
addPrivate(Pair.first, [&CGF, &DRE]() {
2882 return CGF.EmitLValue(&DRE).getAddress();
2885 for (
auto &&Pair : PrivatePtrs) {
2887 CGF.getContext().getDeclAlign(Pair.first));
2892 OMPLexicalScope LexScope(CGF, S,
true);
2895 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2896 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2897 for (
unsigned Cnt = 0, E = Data.
ReductionVars.size(); Cnt < E; ++Cnt) {
2898 RedCG.emitSharedLValue(CGF, Cnt);
2903 Address(CGF.EmitScalarConversion(
2904 Replacement.
getPointer(), CGF.getContext().VoidPtrTy,
2905 CGF.getContext().getPointerType(
2915 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2926 auto IPriv = C->privates().begin();
2927 auto IRed = C->reduction_ops().begin();
2928 auto ITD = C->taskgroup_descriptors().begin();
2929 for (
const auto *Ref : C->varlists()) {
2930 InRedVars.emplace_back(Ref);
2931 InRedPrivs.emplace_back(*IPriv);
2932 InRedOps.emplace_back(*IRed);
2933 TaskgroupDescriptors.emplace_back(*ITD);
2934 std::advance(IPriv, 1);
2935 std::advance(IRed, 1);
2936 std::advance(ITD, 1);
2942 if (!InRedVars.empty()) {
2944 for (
unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
2945 RedCG.emitSharedLValue(CGF, Cnt);
2949 llvm::Value *ReductionsPtr = CGF.EmitLoadOfScalar(
2954 CGF.EmitScalarConversion(
2955 Replacement.
getPointer(), CGF.getContext().VoidPtrTy,
2956 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
2965 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2974 auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
2975 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.
Tied,
2977 OMPLexicalScope
Scope(*
this, S);
2978 TaskGen(*
this, OutlinedFn, Data);
3019 auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
3020 auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3021 auto *I = CS->getCapturedDecl()->param_begin();
3022 auto *PartId = std::next(I);
3023 auto *TaskT = std::next(I, 4);
3026 Data.
Final.setInt(
false);
3029 auto IRef = C->varlist_begin();
3030 auto IElemInitRef = C->inits().begin();
3031 for (
auto *IInit : C->private_copies()) {
3045 getContext(), getContext().getTranslationUnitDecl(), 0);
3047 QualType BaseAndPointersType = getContext().getConstantArrayType(
3051 BaseAndPointersType, CD);
3053 BaseAndPointersType, CD);
3054 QualType SizesType = getContext().getConstantArrayType(
3063 [&InputInfo]() {
return InputInfo.
SizesArray; });
3068 for (
auto *IRef : C->varlists())
3069 Data.
Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
3070 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3075 enum { PrivatesParam = 2, CopyFnParam = 3 };
3076 auto *CopyFn = CGF.Builder.CreateLoad(
3077 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
3078 auto *PrivatesPtr = CGF.Builder.CreateLoad(
3079 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
3083 CallArgs.push_back(PrivatesPtr);
3085 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3087 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3088 ".firstpriv.ptr.addr");
3089 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
3090 CallArgs.push_back(PrivatePtr.getPointer());
3092 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
3094 for (
auto &&Pair : PrivatePtrs) {
3096 CGF.getContext().getDeclAlign(Pair.first));
3103 CGF.GetAddrOfLocalVar(BPVD), 0, CGF.getPointerSize());
3105 CGF.GetAddrOfLocalVar(PVD), 0, CGF.getPointerSize());
3106 InputInfo.
SizesArray = CGF.Builder.CreateConstArrayGEP(
3107 CGF.GetAddrOfLocalVar(SVD), 0, CGF.getSizeSize());
3110 OMPLexicalScope LexScope(CGF, S,
true,
3114 auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3115 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen,
true,
3117 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<
OMPNowaitClause>() ? 1 : 0);
3119 getContext().getIntTypeForBitwidth(32, 0),
3122 CGM.getOpenMPRuntime().emitTaskCall(*
this, S.getLocStart(), S, OutlinedFn,
3123 SharedsTy, CapturedStruct, &IfCond, Data);
3129 auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
3130 auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3131 const Expr *IfCond =
nullptr;
3134 C->getNameModifier() == OMPD_task) {
3135 IfCond = C->getCondition();
3144 CGF.
EmitStmt(CS->getCapturedStmt());
3146 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3149 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.
getLocStart(), S, OutlinedFn,
3150 SharedsTy, CapturedStruct, IfCond,
3153 EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
3158 CGM.getOpenMPRuntime().emitTaskyieldCall(*
this, S.
getLocStart());
3162 CGM.getOpenMPRuntime().emitBarrierCall(*
this, S.
getLocStart(), OMPD_barrier);
3166 CGM.getOpenMPRuntime().emitTaskwaitCall(*
this, S.
getLocStart());
3178 auto IPriv = C->privates().begin();
3179 auto IRed = C->reduction_ops().begin();
3180 auto ILHS = C->lhs_exprs().begin();
3181 auto IRHS = C->rhs_exprs().begin();
3182 for (
const auto *Ref : C->varlists()) {
3186 LHSs.emplace_back(*ILHS);
3187 RHSs.emplace_back(*IRHS);
3188 std::advance(IPriv, 1);
3189 std::advance(IRed, 1);
3190 std::advance(ILHS, 1);
3191 std::advance(IRHS, 1);
3197 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3200 false, E->getType());
3204 OMPLexicalScope
Scope(*
this, S,
true);
3205 CGM.getOpenMPRuntime().emitTaskgroupRegion(*
this, CodeGen, S.
getLocStart());
3211 return llvm::makeArrayRef(FlushClause->varlist_begin(),
3212 FlushClause->varlist_end());
3223 auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
3224 EmitVarDecl(*IVDecl);
3230 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3235 auto &RT = CGM.getOpenMPRuntime();
3237 bool HasLastprivateClause =
false;
3240 OMPLoopScope PreInitScope(*
this, S);
3245 llvm::BasicBlock *ContBlock =
nullptr;
3246 if (ConstantFoldsToSimpleInteger(S.
getPreCond(), CondConstant)) {
3250 auto *ThenBlock = createBasicBlock(
"omp.precond.then");
3251 ContBlock = createBasicBlock(
"omp.precond.end");
3253 getProfileCount(&S));
3254 EmitBlock(ThenBlock);
3255 incrementProfileCounter(&S);
3264 *
this, cast<DeclRefExpr>(
3269 *
this, cast<DeclRefExpr>(
3279 if (EmitOMPFirstprivateClause(S, LoopScope)) {
3283 CGM.getOpenMPRuntime().emitBarrierCall(
3287 EmitOMPPrivateClause(S, LoopScope);
3291 EmitOMPReductionClauseInit(S, LoopScope);
3292 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3293 EmitOMPPrivateLoopCounters(S, LoopScope);
3300 ScheduleKind = C->getDistScheduleKind();
3301 if (
const auto *Ch = C->getChunkSize()) {
3302 Chunk = EmitScalarExpr(Ch);
3303 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3308 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3309 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3319 if (RT.isStaticNonchunked(ScheduleKind,
3320 Chunk !=
nullptr)) {
3322 EmitOMPSimdInit(S,
true);
3323 CGOpenMPRuntime::StaticRTInput StaticInit(
3325 LB.
getAddress(), UB.getAddress(), ST.getAddress());
3326 RT.emitDistributeStaticInit(*
this, S.
getLocStart(), ScheduleKind,
3329 getJumpDestInCurrentScope(createBasicBlock(
"omp.loop.exit"));
3349 CodeGenLoop(CGF, S, LoopExit);
3352 EmitBlock(LoopExit.getBlock());
3354 RT.emitForStaticFinish(*
this, S.getLocStart(), S.getDirectiveKind());
3358 const OMPLoopArguments LoopArguments = {
3361 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3366 return CGF.
Builder.CreateIsNotNull(
3373 ReductionKind = OMPD_parallel_for_simd;
3375 ReductionKind = OMPD_parallel_for;
3377 ReductionKind = OMPD_simd;
3381 "No reduction clauses is allowed in distribute directive.");
3383 EmitOMPReductionClauseFinal(S, ReductionKind);
3387 return CGF.
Builder.CreateIsNotNull(
3391 if (HasLastprivateClause) {
3392 EmitOMPLastprivateClauseFinal(
3394 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.
getLocStart())));
3400 EmitBranch(ContBlock);
3401 EmitBlock(ContBlock,
true);
3412 OMPLexicalScope
Scope(*
this, S,
true);
3413 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_distribute, CodeGen);
3422 Fn->addFnAttr(llvm::Attribute::NoInline);
3429 CGM.getOpenMPRuntime().emitDoacrossOrdered(*
this, DC);
3438 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3440 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.
getLocStart(),
3441 OutlinedFn, CapturedVars);
3448 OMPLexicalScope
Scope(*
this, S,
true);
3449 CGM.getOpenMPRuntime().emitOrderedRegion(*
this, CodeGen, S.
getLocStart(), !C);
3456 "DestType must have scalar evaluation kind.");
3457 assert(!Val.
isAggregate() &&
"Must be a scalar or complex.");
3469 "DestType must have complex evaluation kind.");
3475 DestElementType, Loc);
3477 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3479 assert(Val.
isComplex() &&
"Must be a scalar or complex.");
3483 Val.
getComplexVal().first, SrcElementType, DestElementType, Loc);
3485 Val.
getComplexVal().second, SrcElementType, DestElementType, Loc);
3496 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3497 : llvm::AtomicOrdering::Monotonic,
3504 switch (getEvaluationKind(LVal.
getType())) {
3507 *
this, RVal, RValTy, LVal.
getType(), Loc)),
3516 llvm_unreachable(
"Must be a scalar or complex.");
3524 assert(V->
isLValue() &&
"V of 'omp atomic read' is not lvalue");
3525 assert(X->
isLValue() &&
"X of 'omp atomic read' is not lvalue");
3528 RValue Res = XLValue.isGlobalReg()
3532 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3533 : llvm::AtomicOrdering::Monotonic,
3534 XLValue.isVolatile());
3548 assert(X->
isLValue() &&
"X of 'omp atomic write' is not lvalue");
3561 llvm::AtomicOrdering AO,
3562 bool IsXLHSInRHSPart) {
3567 if (BO == BO_Comma || !Update.
isScalar() ||
3575 return std::make_pair(
false,
RValue::get(
nullptr));
3577 llvm::AtomicRMWInst::BinOp RMWOp;
3580 RMWOp = llvm::AtomicRMWInst::Add;
3583 if (!IsXLHSInRHSPart)
3584 return std::make_pair(
false,
RValue::get(
nullptr));
3585 RMWOp = llvm::AtomicRMWInst::Sub;
3591 RMWOp = llvm::AtomicRMWInst::Or;
3594 RMWOp = llvm::AtomicRMWInst::Xor;
3598 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3599 : llvm::AtomicRMWInst::Max)
3600 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3601 : llvm::AtomicRMWInst::UMax);
3605 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3606 : llvm::AtomicRMWInst::Min)
3607 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3608 : llvm::AtomicRMWInst::UMin);
3611 RMWOp = llvm::AtomicRMWInst::Xchg;
3620 return std::make_pair(
false,
RValue::get(
nullptr));
3639 llvm_unreachable(
"Unsupported atomic update operation");
3642 if (
auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3643 UpdateVal = CGF.
Builder.CreateIntCast(
3666 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3677 const Expr *UE,
bool IsXLHSInRHSPart,
3680 "Update expr in 'atomic update' must be a binary operator.");
3688 assert(X->
isLValue() &&
"X of 'omp atomic update' is not lvalue");
3691 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3692 : llvm::AtomicOrdering::Monotonic;
3693 auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3694 auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3695 auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3696 auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3698 [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](
RValue XRValue) ->
RValue {
3704 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3727 llvm_unreachable(
"Must be a scalar or complex.");
3731 bool IsPostfixUpdate,
const Expr *V,
3733 const Expr *UE,
bool IsXLHSInRHSPart,
3735 assert(X->
isLValue() &&
"X of 'omp atomic capture' is not lvalue");
3736 assert(V->
isLValue() &&
"V of 'omp atomic capture' is not lvalue");
3741 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3742 : llvm::AtomicOrdering::Monotonic;
3747 "Update expr in 'atomic capture' must be a binary operator.");
3755 auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3756 auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3757 auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3758 NewVValType = XRValExpr->getType();
3759 auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3760 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3765 NewVVal = IsPostfixUpdate ? XRValue : Res;
3769 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3772 if (IsPostfixUpdate) {
3774 NewVVal = Res.second;
3788 auto &&Gen = [&NewVVal, ExprRValue](
RValue XRValue) ->
RValue {
3794 XLValue, ExprRValue, BO_Assign,
false, AO,
3798 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3812 bool IsSeqCst,
bool IsPostfixUpdate,
3814 const Expr *UE,
bool IsXLHSInRHSPart,
3829 IsXLHSInRHSPart, Loc);
3833 case OMPC_num_threads:
3835 case OMPC_firstprivate:
3836 case OMPC_lastprivate:
3837 case OMPC_reduction:
3838 case OMPC_task_reduction:
3839 case OMPC_in_reduction:
3849 case OMPC_copyprivate:
3851 case OMPC_proc_bind:
3858 case OMPC_mergeable:
3863 case OMPC_num_teams:
3864 case OMPC_thread_limit:
3866 case OMPC_grainsize:
3868 case OMPC_num_tasks:
3870 case OMPC_dist_schedule:
3871 case OMPC_defaultmap:
3875 case OMPC_use_device_ptr:
3876 case OMPC_is_device_ptr:
3877 llvm_unreachable(
"Clause is not allowed in 'omp atomic'.");
3886 if (C->getClauseKind() != OMPC_seq_cst) {
3887 Kind = C->getClauseKind();
3894 if (
const auto *EWC = dyn_cast<ExprWithCleanups>(CS)) {
3895 enterFullExpression(EWC);
3898 if (
const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3899 for (
const auto *C : Compound->body()) {
3900 if (
const auto *EWC = dyn_cast<ExprWithCleanups>(C)) {
3901 enterFullExpression(EWC);
3908 CGF.EmitStopPoint(CS);
3913 OMPLexicalScope
Scope(*
this, S,
true);
3914 CGM.getOpenMPRuntime().emitInlinedDirective(*
this, OMPD_atomic, CodeGen);
3924 llvm::Function *Fn =
nullptr;
3925 llvm::Constant *FnID =
nullptr;
3927 const Expr *IfCond =
nullptr;
3931 C->getNameModifier() == OMPD_target) {
3932 IfCond = C->getCondition();
3938 const Expr *Device =
nullptr;
3940 Device = C->getDevice();
3946 bool IsOffloadEntry =
true;
3950 IsOffloadEntry =
false;
3953 IsOffloadEntry =
false;
3955 assert(CGF.
CurFuncDecl &&
"No parent declaration for target region!");
3956 StringRef ParentName;
3959 if (
auto *D = dyn_cast<CXXConstructorDecl>(CGF.
CurFuncDecl))
3961 else if (
auto *D = dyn_cast<CXXDestructorDecl>(CGF.
CurFuncDecl))
3969 IsOffloadEntry, CodeGen);
3970 OMPLexicalScope
Scope(CGF, S);
3989 StringRef ParentName,
3995 llvm::Constant *Addr;
3998 S, ParentName, Fn, Addr,
true, CodeGen);
3999 assert(Fn && Addr &&
"Target device function emission failed.");
4021 Expr *ThreadLimit = (TL) ? TL->getThreadLimit() :
nullptr;
4027 OMPTeamsScope
Scope(CGF, S);
4057 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4058 CGF.EmitOMPPrivateClause(S, PrivateScope);
4059 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4062 CGF.EmitStmt(CS->getCapturedStmt());
4063 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4077 llvm::Constant *Addr;
4080 S, ParentName, Fn, Addr,
true, CodeGen);
4081 assert(Fn && Addr &&
"Target device function emission failed.");
4104 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4106 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4108 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4122 llvm::Constant *Addr;
4125 S, ParentName, Fn, Addr,
true, CodeGen);
4126 assert(Fn && Addr &&
"Target device function emission failed.");
4149 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4151 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4153 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4167 llvm::Constant *Addr;
4170 S, ParentName, Fn, Addr,
true, CodeGen);
4171 assert(Fn && Addr &&
"Target device function emission failed.");
4193 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4195 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4197 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4214 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4216 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4218 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4236 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4238 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4240 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4258 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4260 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4261 CGF, OMPD_distribute, CodeGenDistribute,
false);
4262 CGF.EmitOMPReductionClauseFinal(S, OMPD_teams);
4271 CGM.getOpenMPRuntime().emitCancellationPointCall(*
this, S.
getLocStart(),
4276 const Expr *IfCond =
nullptr;
4279 C->getNameModifier() == OMPD_cancel) {
4280 IfCond = C->getCondition();
4284 CGM.getOpenMPRuntime().emitCancelCall(*
this, S.
getLocStart(), IfCond,
4290 if (Kind == OMPD_parallel || Kind == OMPD_task ||
4291 Kind == OMPD_target_parallel)
4293 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4294 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4295 Kind == OMPD_distribute_parallel_for ||
4296 Kind == OMPD_target_parallel_for ||
4297 Kind == OMPD_teams_distribute_parallel_for ||
4298 Kind == OMPD_target_teams_distribute_parallel_for);
4299 return OMPCancelStack.getExitBlock();
4304 const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4305 const auto &C = cast<OMPUseDevicePtrClause>(NC);
4306 auto OrigVarIt = C.varlist_begin();
4307 auto InitIt = C.inits().begin();
4308 for (
auto PvtVarIt : C.private_copies()) {
4309 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4310 auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4311 auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4317 if (
auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4320 auto *ME = cast<MemberExpr>(OED->getInit());
4321 assert(isa<CXXThisExpr>(ME->getBase()) &&
4322 "Base should be the current struct!");
4323 MatchingVD = ME->getMemberDecl();
4328 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4329 if (InitAddrIt == CaptureDeviceAddrMap.end())
4339 getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4340 llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4341 Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4342 setAddrOfLocalVar(InitVD, InitAddr);
4350 LocalDeclMap.erase(InitVD);
4353 return GetAddrOfLocalVar(PvtVD);
4355 assert(IsRegistered &&
"firstprivate var already registered as private");
4367 CGOpenMPRuntime::TargetDataInfo Info(
true);
4372 bool PrivatizeDevicePointers =
false;
4374 bool &PrivatizeDevicePointers;
4377 explicit DevicePointerPrivActionTy(
bool &PrivatizeDevicePointers)
4378 :
PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4380 PrivatizeDevicePointers =
true;
4383 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4385 auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4393 auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4397 PrivatizeDevicePointers =
false;
4403 if (PrivatizeDevicePointers) {
4407 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4408 Info.CaptureDeviceAddrMap);
4422 OMPLexicalScope
Scope(CGF, S);
4423 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4431 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4437 const Expr *IfCond =
nullptr;
4439 IfCond = C->getCondition();
4442 const Expr *Device =
nullptr;
4444 Device = C->getDevice();
4450 CGM.getOpenMPRuntime().emitTargetDataCalls(*
this, S, IfCond, Device, RCG,
4458 if (CGM.getLangOpts().OMPTargetTriples.empty())
4462 const Expr *IfCond =
nullptr;
4464 IfCond = C->getCondition();
4467 const Expr *Device =
nullptr;
4469 Device = C->getDevice();
4471 OMPLexicalScope
Scope(*
this, S,
true);
4472 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond, Device);
4479 if (CGM.getLangOpts().OMPTargetTriples.empty())
4483 const Expr *IfCond =
nullptr;
4485 IfCond = C->getCondition();
4488 const Expr *Device =
nullptr;
4490 Device = C->getDevice();
4492 OMPLexicalScope
Scope(*
this, S,
true);
4493 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond, Device);
4504 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4505 CGF.EmitOMPPrivateClause(S, PrivateScope);
4506 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4509 CGF.EmitStmt(CS->getCapturedStmt());
4510 CGF.EmitOMPReductionClauseFinal(S, OMPD_parallel);
4525 llvm::Constant *Addr;
4528 S, ParentName, Fn, Addr,
true, CodeGen);
4529 assert(Fn && Addr &&
"Target device function emission failed.");
4548 CGF, OMPD_target_parallel_for, S.
hasCancel());
4564 llvm::Constant *Addr;
4567 S, ParentName, Fn, Addr,
true, CodeGen);
4568 assert(Fn && Addr &&
"Target device function emission failed.");
4602 llvm::Constant *Addr;
4605 S, ParentName, Fn, Addr,
true, CodeGen);
4606 assert(Fn && Addr &&
"Target device function emission failed.");
4621 auto *VDecl = cast<VarDecl>(Helper->
getDecl());
4630 auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
4631 auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4632 const Expr *IfCond =
nullptr;
4635 C->getNameModifier() == OMPD_taskloop) {
4636 IfCond = C->getCondition();
4650 Data.
Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4654 Data.
Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4668 llvm::BasicBlock *ContBlock =
nullptr;
4669 OMPLoopScope PreInitScope(CGF, S);
4670 if (CGF.ConstantFoldsToSimpleInteger(S.
getPreCond(), CondConstant)) {
4674 auto *ThenBlock = CGF.createBasicBlock(
"taskloop.if.then");
4675 ContBlock = CGF.createBasicBlock(
"taskloop.if.end");
4677 CGF.getProfileCount(&S));
4678 CGF.EmitBlock(ThenBlock);
4679 CGF.incrementProfileCounter(&S);
4683 CGF.EmitOMPSimdInit(S);
4687 enum { LowerBound = 5, UpperBound, Stride, LastIter };
4688 auto *I = CS->getCapturedDecl()->param_begin();
4689 auto *LBP = std::next(I, LowerBound);
4690 auto *UBP = std::next(I, UpperBound);
4691 auto *STP = std::next(I, Stride);
4692 auto *LIP = std::next(I, LastIter);
4700 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4701 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4705 const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4706 CGF.EmitVarDecl(*IVDecl);
4707 CGF.EmitIgnoredExpr(S.
getInit());
4713 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4721 CGF.EmitOMPLoopBody(S,
JumpDest());
4722 CGF.EmitStopPoint(&S);
4727 CGF.EmitBranch(ContBlock);
4728 CGF.EmitBlock(ContBlock,
true);
4731 if (HasLastprivateClause) {
4732 CGF.EmitOMPLastprivateClauseFinal(
4734 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4735 CGF.GetAddrOfLocalVar(*LIP),
false,
4739 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4743 OMPLoopScope PreInitScope(CGF, S);
4745 OutlinedFn, SharedsTy,
4746 CapturedStruct, IfCond, Data);
4752 EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
4754 CGM.getOpenMPRuntime().emitTaskgroupRegion(
4766 EmitOMPTaskLoopBasedDirective(S);
4771 EmitOMPTaskLoopBasedDirective(S);
4779 if (CGM.getLangOpts().OMPTargetTriples.empty())
4783 const Expr *IfCond =
nullptr;
4785 IfCond = C->getCondition();
4788 const Expr *Device =
nullptr;
4790 Device = C->getDevice();
4792 OMPLexicalScope
Scope(*
this, S,
true);
4793 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*
this, S, IfCond, Device);
4804 if (
const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
4805 for (
const auto *E : LD->counters()) {
4806 if (
const auto *VD = dyn_cast<OMPCapturedExprDecl>(
4807 cast<DeclRefExpr>(E)->getDecl())) {
4809 if (!CGF.LocalDeclMap.count(VD))
4815 while (
const auto *CCS = dyn_cast<CapturedStmt>(CS->getCapturedStmt()))
4817 CGF.
EmitStmt(CS->getCapturedStmt());
4820 OMPSimdLexicalScope
Scope(*
this, D);
4821 CGM.getOpenMPRuntime().emitInlinedDirective(
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
This represents '#pragma omp distribute simd' composite directive.
Expr * getNextUpperBound() const
This represents '#pragma omp master' directive.
static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType, StringRef Name, LValue AddrLV, bool isReferenceType=false)
This represents '#pragma omp task' directive.
static const Decl * getCanonicalDecl(const Decl *D)
An instance of this class is created to represent a function declaration or definition.
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
Expr * getUpperBoundVariable() const
Other implicit parameter.
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
This represents clause 'copyin' in the '#pragma omp ...' directives.
PointerType - C99 6.7.5.1 - Pointer Declarators.
Scheduling data for loop-based OpenMP directives.
A (possibly-)qualified type.
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
CodeGenTypes & getTypes()
ArrayRef< OMPClause * > clauses()
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> &CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
This represents clause 'in_reduction' in the '#pragma omp task' directives.
Expr * getLowerBoundVariable() const
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents '#pragma omp for simd' directive.
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
This represents 'grainsize' clause in the '#pragma omp ...' directive.
This represents '#pragma omp teams distribute parallel for' composite directive.
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents 'if' clause in the '#pragma omp ...' directive.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
This represents 'priority' clause in the '#pragma omp ...' directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
This represents '#pragma omp target teams distribute' combined directive.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Floating point control options.
This represents '#pragma omp parallel for' directive.
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
This represents '#pragma omp target teams distribute parallel for' combined directive.
Expr * getCombinedEnsureUpperBound() const
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Represents a point when we exit a loop.
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
SmallVector< const Expr *, 4 > ReductionCopies
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
This represents '#pragma omp target exit data' directive.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
CanQualType getCanonicalParamType(QualType T) const
Return the canonical parameter type corresponding to the specific potentially non-canonical one...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
VarDecl - An instance of this class is created to represent a variable declaration or definition...
This represents clause 'private' in the '#pragma omp ...' directives.
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
const Stmt * getBody() const
Extra information about a function prototype.
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
Expr * getCombinedUpperBoundVariable() const
Expr * getCalcLastIteration() const
llvm::Value * getPointer() const
unsigned getContextParamPosition() const
This represents implicit clause 'flush' for the '#pragma omp flush' directive.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
bool isXLHSInRHSPart() const
Return true if helper update expression has form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' and...
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
This represents 'nogroup' clause in the '#pragma omp ...' directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
This represents 'safelen' clause in the '#pragma omp ...' directive.
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
IdentifierInfo * getIdentifier() const
getIdentifier - Get the identifier that names this declaration, if there is one.
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
SmallVector< const Expr *, 4 > PrivateVars
RecordDecl - Represents a struct/union/class.
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
One of these records is kept for each identifier that is lexed.
void EmitOMPSimdDirective(const OMPSimdDirective &S)
This represents '#pragma omp parallel' directive.
Address getAddress() const
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
A C++ nested-name-specifier augmented with source location information.
This represents 'simd' clause in the '#pragma omp ...' directive.
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
field_range fields() const
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
Expr * getEnsureUpperBound() const
SmallVector< const Expr *, 4 > LastprivateVars
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
CharUnits getAlignment() const
llvm::IntegerType * SizeTy
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, bool IsMonotonic)
ArrayRef< Expr * > updates()
bool isReferenceType() const
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr)
This represents '#pragma omp target simd' directive.
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself...
OpenMPDirectiveKind getDirectiveKind() const
This represents '#pragma omp barrier' directive.
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc...
Expr * getNumTeams()
Return NumTeams number.
This represents '#pragma omp critical' directive.
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
This represents clause 'copyprivate' in the '#pragma omp ...' directives.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
This represents '#pragma omp distribute parallel for' composite directive.
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
This represents '#pragma omp teams distribute parallel for simd' composite directive.
static bool hasScalarEvaluationKind(QualType T)
SourceLocation getLocEnd() const
Returns ending location of directive.
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
ArrayRef< Expr * > finals()
Expr * getIsLastIterVariable() const
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
OpenMPScheduleClauseKind Schedule
CharUnits - This is an opaque type for sizes expressed in character units.
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value *> &CapturedVars)
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
llvm::function_ref< std::pair< LValue, LValue > CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
Expr * getX()
Get 'x' part of the associated expression/statement.
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
CharUnits getAlignment() const
Return the alignment of this pointer.
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
Expr * getIterationVariable() const
A builtin binary operation expression such as "x + y" or "x <= y".
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
OpenMPScheduleClauseModifier M2
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
SmallVector< const Expr *, 4 > PrivateCopies
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S)
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
This represents '#pragma omp cancellation point' directive.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations...
Scope - A scope is a transient data structure that is used while parsing the program.
field_iterator field_begin() const
This represents 'final' clause in the '#pragma omp ...' directive.
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
void EmitOMPParallelDirective(const OMPParallelDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
This represents '#pragma omp teams' directive.
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
SmallVector< const Expr *, 4 > FirstprivateCopies
This represents clause 'reduction' in the '#pragma omp ...' directives.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
This represents '#pragma omp teams distribute simd' combined directive.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CharUnits getPointerAlign() const
An ordinary object is located at an address in memory.
SmallVector< const Expr *, 4 > ReductionOps
Controls insertion of cancellation exit blocks in worksharing constructs.
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler's counter for the given statement by StepV.
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * > CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
This represents the body of a CapturedStmt, and serves as its DeclContext.
SmallVector< const Expr *, 4 > ReductionVars
static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, bool isConstexprSpecified=false)
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address >> &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value *>> &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
SourceLocation getLocStart() const
Returns starting location of directive kind.
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
This represents '#pragma omp target parallel for simd' directive.
ArrayRef< Expr * > private_counters()
CompoundStmt - This represents a group of statements like { stmt stmt }.
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
ASTContext & getContext() const
bool hasCancel() const
Return true if current directive has inner cancel directive.
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
This represents '#pragma omp taskgroup' directive.
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
This represents clause 'aligned' in the '#pragma omp ...' directives.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat)
SourceLocation getLocEnd() const LLVM_READONLY
bool addPrivate(const VarDecl *LocalVD, llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
Class intended to support codegen of all kind of the reduction clauses.
bool isGlobalVarCaptured(const VarDecl *VD) const
Checks if the global variable is captured in current function.
Expr * getCombinedLowerBoundVariable() const
This represents clause 'task_reduction' in the '#pragma omp taskgroup' directives.
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
This represents '#pragma omp distribute' directive.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
unsigned NumberOfTargetItems
const Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
bool hasCancel() const
Return true if current directive has inner cancel directive.
This represents 'proc_bind' clause in the '#pragma omp ...' directive.
ValueDecl - Represent the declaration of a variable (in which case it is an lvalue) a function (in wh...
Expr - This represents one expression.
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
Emit only debug info necessary for generating line number tables (-gline-tables-only).
void EmitAutoVarInit(const AutoVarEmission &emission)
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
const FunctionProtoType * T
This represents 'simdlen' clause in the '#pragma omp ...' directive.
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top, if IgnoreCaptured is true.
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type, where the destination type is an LLVM scalar type.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
const T * castAs() const
Member-template castAs<specific type>.
void SetInternalFunctionAttributes(const Decl *D, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
bool hasCancel() const
Return true if current directive has inner cancel directive.
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
This represents '#pragma omp target teams distribute parallel for simd' combined directive.
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
OpenMPClauseKind
OpenMP clauses.
llvm::PointerType * getType() const
Return the type of the pointer value.
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause *> Clauses)
This represents '#pragma omp target teams distribute simd' combined directive.
void EmitOMPSimdInit(const OMPLoopDirective &D, bool IsMonotonic=false)
Helpers for the OpenMP loop directives.
This represents 'ordered' clause in the '#pragma omp ...' directive.
Expr * getDistInc() const
Expr * getNextLowerBound() const
Expr * getPrevEnsureUpperBound() const
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> &CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
This represents '#pragma omp for' directive.
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
SmallVector< const Expr *, 4 > FirstprivateVars
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
void EmitOMPMasterDirective(const OMPMasterDirective &S)
static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the 'for' loop has a dispatch schedule (e.g.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
This represents '#pragma omp target teams' directive.
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
SourceLocation getEnd() const
UnaryOperator - This represents the unary-expression's (except sizeof and alignof), the postinc/postdec operators from postfix-expression, and various extensions.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
This represents '#pragma omp cancel' directive.
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const LangOptions & getLangOpts() const
ASTContext & getContext() const
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
GlobalDecl - represents a global declaration.
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
This represents '#pragma omp flush' directive.
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind. ...
This represents '#pragma omp parallel for simd' directive.
This represents 'seq_cst' clause in the '#pragma omp atomic' directive.
This represents 'untied' clause in the '#pragma omp ...' directive.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
The l-value was considered opaque, so the alignment was determined from a type.
void EmitAlignmentAssumption(llvm::Value *PtrValue, unsigned Alignment, llvm::Value *OffsetValue=nullptr)
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
Expr * getLastIteration() const
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
bool isPostfixUpdate() const
Return true if 'v' expression must be updated to original value of 'x', false if 'v' must be updated ...
This represents '#pragma omp target enter data' directive.
void EmitOMPFlushDirective(const OMPFlushDirective &S)
Expr * getStrideVariable() const
This represents 'num_teams' clause in the '#pragma omp ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
This captures a statement into a function.
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
This represents '#pragma omp single' directive.
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
This represents 'hint' clause in the '#pragma omp ...' directive.
void EmitOMPForDirective(const OMPForDirective &S)
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
Pseudo declaration for capturing expressions.
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
This is a basic class for representing single OpenMP executable directive.
This represents 'schedule' clause in the '#pragma omp ...' directive.
Expr * getExpr()
Get 'expr' part of the associated expression/statement.
OpenMPDirectiveKind
OpenMP directives.
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> &CondGen)
Emit final code for linear clauses.
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
DeclarationNameInfo getDirectiveName() const
Return name of the directive.
void EmitStmt(const Stmt *S, ArrayRef< const Attr *> Attrs=None)
EmitStmt - Emit the code for the statement.
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
C-style initialization with assignment.
This represents '#pragma omp taskwait' directive.
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
This file defines OpenMP nodes for declarative directives.
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
This is a basic class for representing single OpenMP clause.
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
bool isAnyPointerType() const
virtual void Enter(CodeGenFunction &CGF)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
bool isLValue() const
isLValue - True if this expression is an "l-value" according to the rules of the current language...
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> &CondGen)
This represents '#pragma omp target' directive.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
Expr * getV()
Get 'v' part of the associated expression/statement.
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
This represents '#pragma omp ordered' directive.
This represents '#pragma omp target update' directive.
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
void setAction(PrePostActionTy &Action) const
CGFunctionInfo - Class to encapsulate the information about a function definition.
This class organizes the cross-function state that is used while generating LLVM code.
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
Dataflow Directional Tag Classes.
std::string getAsString() const
getAsString - Retrieve the human-readable string for this name.
Expr * getPrevUpperBoundVariable() const
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
This represents 'device' clause in the '#pragma omp ...' directive.
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value *> &)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
This represents '#pragma omp section' directive.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
OpenMPScheduleClauseModifier M1
This represents '#pragma omp teams distribute' directive.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
const Expr * getInit() const
This represents '#pragma omp simd' directive.
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
DeclarationName - The name of a declaration.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
const Expr * getReductionRef() const
Returns reference to the task_reduction return variable.
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
This represents clause 'linear' in the '#pragma omp ...' directives.
SourceLocation getLocStart() const LLVM_READONLY
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPCancelDirective(const OMPCancelDirective &S)
Expr * getUpdateExpr()
Get helper expression of the form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' or 'OpaqueValueExp...
llvm::Module & getModule() const
This represents '#pragma omp atomic' directive.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
Expr * getCombinedInit() const
Expr * getCombinedNextLowerBound() const
ArrayRef< Expr * > counters()
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD)
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
SwitchStmt - This represents a 'switch' stmt.
Expr * getCombinedNextUpperBound() const
void EmitAutoVarCleanups(const AutoVarEmission &emission)
API for captured statement code generation.
Complex values, per C99 6.2.5p11.
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, LValue LVal, RValue RVal)
This file defines OpenMP AST classes for executable directives and clauses.
ArrayRef< Expr * > inits()
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
bool isStaticLocal() const
isStaticLocal - Returns true if a variable with function scope is a static local variable.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Base for LValueReferenceType and RValueReferenceType.
StringRef getMangledName(GlobalDecl GD)
Address BasePointersArray
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
bool hasAssociatedStmt() const
Returns true if directive has associated statement.
Expr * getPrevLowerBoundVariable() const
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
CanQualType getCanonicalType(QualType T) const
Return the canonical (structural) type corresponding to the specified potentially non-canonical type ...
bool isLValueReferenceType() const
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
TranslationUnitDecl * getTranslationUnitDecl() const
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for reduction variables.
This represents '#pragma omp target parallel' directive.
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
This represents 'nowait' clause in the '#pragma omp ...' directive.
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g., it is an signed integer type or a vector.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> &CondGen)
This represents 'num_tasks' clause in the '#pragma omp ...' directive.
Privates[]
Gets the list of initial values for linear variables.
Expr * getCombinedCond() const
void EmitOMPUseDevicePtrClause(const OMPClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, Address > &CaptureDeviceAddrMap)
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
This represents '#pragma omp taskloop simd' directive.
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
Expr * getPreCond() const
This represents 'dist_schedule' clause in the '#pragma omp ...' directive.
CGCapturedStmtInfo * CapturedStmtInfo
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
const llvm::function_ref< void(CodeGenFunction &, llvm::Value *, const OMPTaskDataTy &)> TaskGenTy
This represents '#pragma omp sections' directive.
This represents '#pragma omp target data' directive.
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> &BodyGen, const llvm::function_ref< void(CodeGenFunction &)> &PostIncGen)
Emit inner loop of the worksharing/simd construct.
A reference to a declared variable, function, enum, etc.
static RValue get(llvm::Value *V)
bool isPointerType() const
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
An l-value expression is a reference to an object with independent storage.
A trivial tuple used to represent a source range.
LValue - This represents an lvalue references.
This represents '#pragma omp taskyield' directive.
This represents '#pragma omp distribute parallel for simd' composite directive.
This represents '#pragma omp parallel sections' directive.
SourceLocation getLocStart() const LLVM_READONLY
SourceLocation getBegin() const
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
Address CreateMemTemp(QualType T, const Twine &Name="tmp", bool CastToDefaultAddrSpace=true)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignment...
llvm::Value * getPointer() const
void EmitOMPTaskDirective(const OMPTaskDirective &S)
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
This represents '#pragma omp target parallel for' directive.
SmallVector< const Expr *, 4 > FirstprivateInits
SourceLocation getLocation() const
This represents clause 'use_device_ptr' in the '#pragma omp ...' directives.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool Privatize()
Privatizes local variables previously registered as private.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
This represents '#pragma omp taskloop' directive.