25 #include "llvm/ADT/ArrayRef.h" 26 #include "llvm/ADT/SetOperations.h" 27 #include "llvm/Bitcode/BitcodeReader.h" 28 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" 29 #include "llvm/IR/DerivedTypes.h" 30 #include "llvm/IR/GlobalValue.h" 31 #include "llvm/IR/Value.h" 32 #include "llvm/Support/Format.h" 33 #include "llvm/Support/raw_ostream.h" 36 using namespace clang;
37 using namespace CodeGen;
45 enum CGOpenMPRegionKind {
48 ParallelOutlinedRegion,
59 const CGOpenMPRegionKind RegionKind,
62 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
63 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
65 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
68 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
69 Kind(Kind), HasCancel(HasCancel) {}
73 virtual const VarDecl *getThreadIDVariable()
const = 0;
84 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
88 bool hasCancel()
const {
return HasCancel; }
90 static bool classof(
const CGCapturedStmtInfo *Info) {
94 ~CGOpenMPRegionInfo()
override =
default;
97 CGOpenMPRegionKind RegionKind;
104 class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
109 StringRef HelperName)
110 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
112 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
113 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
118 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
121 StringRef getHelperName()
const override {
return HelperName; }
123 static bool classof(
const CGCapturedStmtInfo *Info) {
125 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
126 ParallelOutlinedRegion;
133 StringRef HelperName;
137 class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
143 llvm::SwitchInst *UntiedSwitch =
nullptr;
146 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
148 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
158 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
162 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
164 emitUntiedSwitch(CGF);
179 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
185 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
191 const UntiedTaskActionTy &Action)
192 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
193 ThreadIDVar(ThreadIDVar), Action(Action) {
194 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
199 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
205 StringRef getHelperName()
const override {
return ".omp_outlined."; }
208 Action.emitUntiedSwitch(CGF);
211 static bool classof(
const CGCapturedStmtInfo *Info) {
213 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
222 const UntiedTaskActionTy &Action;
227 class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
232 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
234 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
239 return OuterRegionInfo->getContextValue();
240 llvm_unreachable(
"No context value for inlined OpenMP region");
244 if (OuterRegionInfo) {
245 OuterRegionInfo->setContextValue(V);
248 llvm_unreachable(
"No context value for inlined OpenMP region");
254 return OuterRegionInfo->lookup(VD);
260 FieldDecl *getThisFieldDecl()
const override {
262 return OuterRegionInfo->getThisFieldDecl();
268 const VarDecl *getThreadIDVariable()
const override {
270 return OuterRegionInfo->getThreadIDVariable();
277 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
278 llvm_unreachable(
"No LValue for inlined OpenMP construct");
282 StringRef getHelperName()
const override {
283 if (
auto *OuterRegionInfo = getOldCSI())
284 return OuterRegionInfo->getHelperName();
285 llvm_unreachable(
"No helper name for inlined OpenMP construct");
290 OuterRegionInfo->emitUntiedSwitch(CGF);
295 static bool classof(
const CGCapturedStmtInfo *Info) {
297 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
300 ~CGOpenMPInlinedRegionInfo()
override =
default;
305 CGOpenMPRegionInfo *OuterRegionInfo;
313 class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
317 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
319 HelperName(HelperName) {}
323 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
326 StringRef getHelperName()
const override {
return HelperName; }
328 static bool classof(
const CGCapturedStmtInfo *Info) {
330 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
334 StringRef HelperName;
338 llvm_unreachable(
"No codegen for expressions");
342 class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
345 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
352 for (
const auto &C : CS.
captures()) {
353 if (!
C.capturesVariable() && !
C.capturesVariableByCopy())
356 const VarDecl *VD =
C.getCapturedVar();
364 PrivScope.addPrivate(
367 (void)PrivScope.Privatize();
372 if (
const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
379 llvm_unreachable(
"No body for expressions");
384 const VarDecl *getThreadIDVariable()
const override {
385 llvm_unreachable(
"No thread id for expressions");
389 StringRef getHelperName()
const override {
390 llvm_unreachable(
"No helper name for expressions");
393 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
401 class InlinedOpenMPRegionRAII {
403 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
404 FieldDecl *LambdaThisCaptureField =
nullptr;
425 ~InlinedOpenMPRegionRAII() {
442 OMP_IDENT_IMD = 0x01,
444 OMP_IDENT_KMPC = 0x02,
446 OMP_ATOMIC_REDUCE = 0x10,
448 OMP_IDENT_BARRIER_EXPL = 0x20,
450 OMP_IDENT_BARRIER_IMPL = 0x40,
452 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
454 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
456 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
458 OMP_IDENT_WORK_LOOP = 0x200,
460 OMP_IDENT_WORK_SECTIONS = 0x400,
462 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
463 LLVM_MARK_AS_BITMASK_ENUM(OMP_IDENT_WORK_DISTRIBUTE)
471 OMP_REQ_UNDEFINED = 0x000,
473 OMP_REQ_NONE = 0x001,
475 OMP_REQ_REVERSE_OFFLOAD = 0x002,
477 OMP_REQ_UNIFIED_ADDRESS = 0x004,
479 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
481 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
482 LLVM_MARK_AS_BITMASK_ENUM(OMP_REQ_DYNAMIC_ALLOCATORS)
488 OMP_DEVICEID_UNDEF = -1,
784 Callback(CodeGen, CGF, *PrePostAction);
787 Callback(CodeGen, CGF, Action);
795 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
796 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
797 if (
const auto *DRE =
798 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
799 if (
const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
810 std::pair<llvm::Function *, llvm::Function *> Reduction =
812 const auto *CE = cast<CallExpr>(InitOp);
813 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
817 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
819 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
821 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
822 [=]() {
return Private; });
823 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
824 [=]() {
return Original; });
825 (void)PrivateScope.Privatize();
832 auto *GV =
new llvm::GlobalVariable(
834 llvm::GlobalValue::PrivateLinkage, Init, Name);
880 SrcBegin = SrcAddr.getPointer();
888 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
889 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
892 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
897 llvm::PHINode *SrcElementPHI =
nullptr;
900 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
901 "omp.arraycpy.srcElementPast");
902 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
905 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
907 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
908 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
909 DestElementPHI->addIncoming(DestBegin, EntryBB);
917 if (EmitDeclareReductionInit) {
919 SrcElementCurrent, ElementTy);
928 SrcElementPHI, 1,
"omp.arraycpy.dest.element");
929 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
934 DestElementPHI, 1,
"omp.arraycpy.dest.element");
937 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
938 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
939 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
951 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(E))
956 void ReductionCodeGen::emitAggregateInitialization(
962 const auto *PrivateVD =
963 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
964 bool EmitDeclareReductionInit =
967 EmitDeclareReductionInit,
968 EmitDeclareReductionInit ? ClausesData[N].ReductionOp
969 : PrivateVD->getInit(),
976 ClausesData.reserve(Shareds.size());
977 SharedAddresses.reserve(Shareds.size());
978 Sizes.reserve(Shareds.size());
979 BaseDecls.reserve(Shareds.size());
980 auto IPriv = Privates.begin();
981 auto IRed = ReductionOps.begin();
982 for (
const Expr *Ref : Shareds) {
983 ClausesData.emplace_back(Ref, *IPriv, *IRed);
984 std::advance(IPriv, 1);
985 std::advance(IRed, 1);
990 assert(SharedAddresses.size() == N &&
991 "Number of generated lvalues must be exactly N.");
992 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref);
993 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);
994 SharedAddresses.emplace_back(First, Second);
998 const auto *PrivateVD =
999 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1000 QualType PrivateType = PrivateVD->getType();
1001 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref);
1002 if (!PrivateType->isVariablyModifiedType()) {
1005 SharedAddresses[N].first.getType().getNonReferenceType()),
1011 auto *ElemType = cast<llvm::PointerType>(
1012 SharedAddresses[N].first.getPointer(CGF)->getType())
1014 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);
1015 if (AsArraySection) {
1016 Size = CGF.
Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(CGF),
1017 SharedAddresses[N].first.getPointer(CGF));
1018 Size = CGF.
Builder.CreateNUWAdd(
1019 Size, llvm::ConstantInt::get(Size->getType(), 1));
1020 SizeInChars = CGF.
Builder.CreateNUWMul(Size, ElemSizeOf);
1023 SharedAddresses[N].first.getType().getNonReferenceType());
1024 Size = CGF.
Builder.CreateExactUDiv(SizeInChars, ElemSizeOf);
1026 Sizes.emplace_back(SizeInChars, Size);
1029 cast<OpaqueValueExpr>(
1037 const auto *PrivateVD =
1038 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1039 QualType PrivateType = PrivateVD->getType();
1040 if (!PrivateType->isVariablyModifiedType()) {
1041 assert(!Size && !Sizes[N].second &&
1042 "Size should be nullptr for non-variably modified reduction " 1048 cast<OpaqueValueExpr>(
1057 assert(SharedAddresses.size() > N &&
"No variable was generated");
1058 const auto *PrivateVD =
1059 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1062 QualType PrivateType = PrivateVD->getType();
1065 QualType SharedType = SharedAddresses[N].first.getType();
1069 SharedType, SharedAddresses[N].first.getBaseInfo(),
1072 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD);
1073 }
else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1077 }
else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
1080 PrivateVD->
getType().getQualifiers(),
1086 const auto *PrivateVD =
1087 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1088 QualType PrivateType = PrivateVD->getType();
1095 const auto *PrivateVD =
1096 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl());
1097 QualType PrivateType = PrivateVD->getType();
1099 if (needCleanups(N)) {
1102 CGF.
pushDestroy(DTorKind, PrivateAddr, PrivateType);
1151 return Address(Addr, BaseLVAlignment);
1155 const VarDecl *OrigVD =
nullptr;
1156 if (
const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) {
1157 const Expr *
Base = OASE->getBase()->IgnoreParenImpCasts();
1158 while (
const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
1160 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1162 DE = cast<DeclRefExpr>(Base);
1163 OrigVD = cast<VarDecl>(DE->
getDecl());
1164 }
else if (
const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {
1165 const Expr *
Base = ASE->getBase()->IgnoreParenImpCasts();
1166 while (
const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
1168 DE = cast<DeclRefExpr>(Base);
1169 OrigVD = cast<VarDecl>(DE->
getDecl());
1178 BaseDecls.emplace_back(OrigVD);
1181 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),
1182 OriginalBaseLValue);
1184 BaseLValue.
getPointer(CGF), SharedAddresses[N].first.getPointer(CGF));
1188 SharedAddresses[N].first.getAddress(CGF).getType());
1191 SharedAddresses[N].first.getType(),
1195 BaseDecls.emplace_back(
1196 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));
1225 LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1228 getThreadIDVariable()->
getType(),
1244 StringRef Separator)
1245 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator),
1264 KmpCriticalNameTy = llvm::ArrayType::get(CGM.
Int32Ty, 8);
1271 llvm::GlobalValue *OrigAddr,
1272 bool IsForDefinition) {
1275 if (IsForDefinition || OrigAddr)
1279 if (Addr && !Addr->isDeclaration()) {
1280 const auto *D = cast<FunctionDecl>(OldGD.
getDecl());
1293 assert(OrigAddr->isDeclaration() &&
"Expected declaration");
1295 GA->takeName(OrigAddr);
1296 OrigAddr->replaceAllUsesWith(
1297 llvm::ConstantExpr::getBitCast(GA, OrigAddr->getType()));
1298 OrigAddr->eraseFromParent();
1305 if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
1306 D->isWeakImported())
1307 GA->setLinkage(llvm::Function::WeakAnyLinkage);
1316 InternalVars.clear();
1319 if (!Data.getValue().pointsToAliveValue())
1321 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue());
1324 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1326 GV->eraseFromParent();
1340 llvm::raw_svector_ostream OS(Buffer);
1342 for (StringRef Part : Parts) {
1349 static llvm::Function *
1351 const Expr *CombinerInitializer,
const VarDecl *In,
1352 const VarDecl *Out,
bool IsCombiner) {
1361 Args.push_back(&OmpOutParm);
1362 Args.push_back(&OmpInParm);
1367 {IsCombiner ?
"omp_combiner" :
"omp_initializer",
""});
1372 Fn->removeFnAttr(llvm::Attribute::NoInline);
1373 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
1374 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
1383 Scope.
addPrivate(In, [&CGF, AddrIn, PtrTy]() {
1388 Scope.
addPrivate(Out, [&CGF, AddrOut, PtrTy]() {
1393 if (!IsCombiner && Out->
hasInit() &&
1399 if (CombinerInitializer)
1408 if (UDRMap.count(D) > 0)
1412 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerIn())->getDecl()),
1413 cast<VarDecl>(cast<DeclRefExpr>(D->
getCombinerOut())->getDecl()),
1421 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitOrig())->getDecl()),
1422 cast<VarDecl>(cast<DeclRefExpr>(D->
getInitPriv())->getDecl()),
1425 UDRMap.try_emplace(D, Combiner, Initializer);
1427 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->
CurFn);
1428 Decls.second.push_back(D);
1432 std::pair<llvm::Function *, llvm::Function *>
1434 auto I = UDRMap.find(D);
1435 if (I != UDRMap.end())
1438 return UDRMap.lookup(D);
1444 struct PushAndPopStackRAII {
1445 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,
CodeGenFunction &CGF,
1447 : OMPBuilder(OMPBuilder) {
1463 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1464 assert(IP.getBlock()->end() == IP.getPoint() &&
1465 "Clang CG should cause non-terminated block!");
1466 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1475 llvm::OpenMPIRBuilder::FinalizationInfo FI(
1476 {FiniCB, OMPD_parallel, HasCancel});
1477 OMPBuilder->pushFinalizationCB(std::move(FI));
1479 ~PushAndPopStackRAII() {
1481 OMPBuilder->popFinalizationCB();
1483 llvm::OpenMPIRBuilder *OMPBuilder;
1492 "thread id variable must be of type kmp_int32 *");
1494 bool HasCancel =
false;
1495 if (
const auto *OPD = dyn_cast<OMPParallelDirective>(&D))
1496 HasCancel = OPD->hasCancel();
1497 else if (
const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
1498 HasCancel = OPSD->hasCancel();
1499 else if (
const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
1500 HasCancel = OPFD->hasCancel();
1501 else if (
const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D))
1502 HasCancel = OPFD->hasCancel();
1503 else if (
const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D))
1504 HasCancel = OPFD->hasCancel();
1505 else if (
const auto *OPFD =
1506 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D))
1507 HasCancel = OPFD->hasCancel();
1508 else if (
const auto *OPFD =
1509 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D))
1510 HasCancel = OPFD->hasCancel();
1515 PushAndPopStackRAII PSR(OMPBuilder, CGF, HasCancel);
1516 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1517 HasCancel, OutlinedHelperName);
1519 return CGF.GenerateOpenMPCapturedStmtFunction(*CS);
1542 bool Tied,
unsigned &NumberOfParts) {
1549 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
1554 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1558 "thread id variable must be of type kmp_int32 for tasks");
1565 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1567 TD ? TD->hasCancel() :
false, Action);
1569 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);
1571 NumberOfParts = Action.getNumberOfParts();
1577 ArrayRef<llvm::Constant *> Data) {
1579 unsigned PrevIdx = 0;
1581 auto DI = Data.begin();
1585 for (
unsigned I = PrevIdx; I < Idx; ++I)
1586 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I)));
1593 template <
class... As>
1594 static llvm::GlobalVariable *
1596 ArrayRef<llvm::Constant *> Data,
const Twine &Name,
1605 std::forward<As>(Args)...);
1608 template <
typename T>
1611 ArrayRef<llvm::Constant *> Data,
1620 Address CGOpenMPRuntime::getOrCreateDefaultLocation(
unsigned Flags) {
1623 FlagsTy FlagsKey(Flags, Reserved2Flags);
1624 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey);
1626 if (!DefaultOpenMPPSource) {
1631 DefaultOpenMPPSource =
1633 DefaultOpenMPPSource =
1634 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource,
CGM.
Int8PtrTy);
1637 llvm::Constant *Data[] = {
1638 llvm::ConstantInt::getNullValue(
CGM.
Int32Ty),
1640 llvm::ConstantInt::get(
CGM.
Int32Ty, Reserved2Flags),
1641 llvm::ConstantInt::getNullValue(
CGM.
Int32Ty), DefaultOpenMPPSource};
1642 llvm::GlobalValue *DefaultOpenMPLocation =
1644 llvm::GlobalValue::PrivateLinkage);
1645 DefaultOpenMPLocation->setUnnamedAddr(
1646 llvm::GlobalValue::UnnamedAddr::Global);
1648 OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation;
1654 bool AtCurrentPoint) {
1655 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1656 assert(!Elem.second.ServiceInsertPt &&
"Insert point is set already.");
1659 if (AtCurrentPoint) {
1660 Elem.second.ServiceInsertPt =
new llvm::BitCastInst(
1663 Elem.second.ServiceInsertPt =
1664 new llvm::BitCastInst(Undef, CGF.
Int32Ty,
"svcpt");
1670 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1671 if (Elem.second.ServiceInsertPt) {
1672 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt;
1673 Elem.second.ServiceInsertPt =
nullptr;
1674 Ptr->eraseFromParent();
1681 Flags |= OMP_IDENT_KMPC;
1685 return getOrCreateDefaultLocation(Flags).getPointer();
1687 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1691 auto I = OpenMPLocThreadIDMap.find(CGF.
CurFn);
1692 if (I != OpenMPLocThreadIDMap.end())
1693 LocValue =
Address(I->second.DebugLoc, Align);
1700 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1704 if (!Elem.second.ServiceInsertPt)
1706 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1707 CGF.
Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1714 auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin();
1719 if (OMPDebugLoc ==
nullptr) {
1721 llvm::raw_svector_ostream OS2(Buffer2);
1725 if (
const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.
CurFuncDecl))
1726 OS2 << FD->getQualifiedNameAsString();
1728 OMPDebugLoc = CGF.
Builder.CreateGlobalStringPtr(OS2.str());
1741 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1746 auto I = OpenMPLocThreadIDMap.find(CGF.
CurFn);
1747 if (I != OpenMPLocThreadIDMap.end()) {
1748 ThreadID = I->second.ThreadID;
1749 if (ThreadID !=
nullptr)
1753 if (
auto *OMPRegionInfo =
1755 if (OMPRegionInfo->getThreadIDVariable()) {
1757 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1761 CGF.
Builder.GetInsertBlock() == TopBlock ||
1762 !isa<llvm::Instruction>(LVal.
getPointer(CGF)) ||
1763 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1765 cast<llvm::Instruction>(LVal.
getPointer(CGF))->getParent() ==
1766 CGF.
Builder.GetInsertBlock()) {
1770 if (CGF.
Builder.GetInsertBlock() == TopBlock) {
1771 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1772 Elem.second.ThreadID = ThreadID;
1783 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1784 if (!Elem.second.ServiceInsertPt)
1786 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1787 CGF.
Builder.SetInsertPoint(Elem.second.ServiceInsertPt);
1788 llvm::CallInst *Call = CGF.
Builder.CreateCall(
1792 Elem.second.ThreadID = Call;
1797 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1798 if (OpenMPLocThreadIDMap.count(CGF.
CurFn)) {
1800 OpenMPLocThreadIDMap.erase(CGF.
CurFn);
1802 if (FunctionUDRMap.count(CGF.
CurFn) > 0) {
1803 for(
auto *D : FunctionUDRMap[CGF.
CurFn])
1805 FunctionUDRMap.erase(CGF.
CurFn);
1807 auto I = FunctionUDMMap.find(CGF.
CurFn);
1808 if (I != FunctionUDMMap.end()) {
1809 for(
auto *D : I->second)
1811 FunctionUDMMap.erase(I);
1816 return IdentTy->getPointerTo();
1820 if (!Kmpc_MicroTy) {
1824 Kmpc_MicroTy = llvm::FunctionType::get(
CGM.
VoidTy, MicroParams,
true);
1826 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1830 llvm::FunctionCallee RTLFn =
nullptr;
1831 switch (static_cast<OpenMPRTLFunction>(Function)) {
1838 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
true);
1840 if (
auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
1841 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
1842 llvm::LLVMContext &Ctx = F->getContext();
1843 llvm::MDBuilder MDB(Ctx);
1850 llvm::LLVMContext::MD_callback,
1851 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
1862 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
1873 llvm::FunctionType::get(
CGM.
VoidPtrTy, TypeParams,
false);
1882 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1884 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1892 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1895 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1905 false)->getPointerTo();
1908 auto *KmpcCopyCtorTy =
1909 llvm::FunctionType::get(
CGM.
VoidPtrTy, KmpcCopyCtorTyArgs,
1917 KmpcCopyCtorTy, KmpcDtorTy};
1918 auto *FnTy = llvm::FunctionType::get(
CGM.
VoidTy, FnTyArgs,
1928 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1930 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1939 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
1947 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1955 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1965 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1974 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1983 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1991 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
1999 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2007 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2016 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2024 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2032 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2040 assert(KmpRoutineEntryPtrTy !=
nullptr &&
2041 "Type kmp_routine_entry_t must be created.");
2046 llvm::FunctionType::get(
CGM.
VoidPtrTy, TypeParams,
false);
2054 assert(KmpRoutineEntryPtrTy !=
nullptr &&
2055 "Type kmp_routine_entry_t must be created.");
2061 llvm::FunctionType::get(
CGM.
VoidPtrTy, TypeParams,
false);
2071 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2081 llvm::FunctionType::get(
CGM.
VoidTy, CpyTypeParams,
false);
2086 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2095 auto *ReduceFnTy = llvm::FunctionType::get(
CGM.
VoidTy, ReduceTypeParams,
2100 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2102 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2112 auto *ReduceFnTy = llvm::FunctionType::get(
CGM.
VoidTy, ReduceTypeParams,
2117 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2119 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2128 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2130 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2139 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
2141 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2152 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2163 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2165 "__kmpc_omp_task_complete_if0");
2172 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2180 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2188 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2196 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2204 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2213 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2225 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2238 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2247 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2256 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2266 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2276 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
true);
2278 if (
auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
2279 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
2280 llvm::LLVMContext &Ctx = F->getContext();
2281 llvm::MDBuilder MDB(Ctx);
2288 llvm::LLVMContext::MD_callback,
2289 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2312 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2324 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2332 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2342 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2352 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2361 llvm::FunctionType::get(
CGM.
VoidPtrTy, TypeParams,
false);
2371 llvm::FunctionType::get(
CGM.
VoidPtrTy, TypeParams,
false);
2373 FnTy,
"__kmpc_task_reduction_get_th_data");
2381 llvm::FunctionType::get(
CGM.
VoidPtrTy, TypeParams,
false);
2390 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2398 llvm::FunctionType *FnTy =
2399 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2415 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2431 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2449 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2467 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2475 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2489 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2504 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2518 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2533 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2547 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2562 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2570 llvm::FunctionType::get(
CGM.
Int64Ty, TypeParams,
false);
2580 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2585 assert(RTLFn &&
"Unable to find OpenMP runtime function");
2589 llvm::FunctionCallee
2591 assert((IVSize == 32 || IVSize == 64) &&
2592 "IV size is not compatible with the omp runtime");
2593 StringRef Name = IVSize == 32 ? (IVSigned ?
"__kmpc_for_static_init_4" 2594 :
"__kmpc_for_static_init_4u")
2595 : (IVSigned ?
"__kmpc_for_static_init_8" 2596 :
"__kmpc_for_static_init_8u");
2598 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2611 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2615 llvm::FunctionCallee
2617 assert((IVSize == 32 || IVSize == 64) &&
2618 "IV size is not compatible with the omp runtime");
2621 ? (IVSigned ?
"__kmpc_dispatch_init_4" :
"__kmpc_dispatch_init_4u")
2622 : (IVSigned ?
"__kmpc_dispatch_init_8" :
"__kmpc_dispatch_init_8u");
2633 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2637 llvm::FunctionCallee
2639 assert((IVSize == 32 || IVSize == 64) &&
2640 "IV size is not compatible with the omp runtime");
2643 ? (IVSigned ?
"__kmpc_dispatch_fini_4" :
"__kmpc_dispatch_fini_4u")
2644 : (IVSigned ?
"__kmpc_dispatch_fini_8" :
"__kmpc_dispatch_fini_8u");
2650 llvm::FunctionType::get(
CGM.
VoidTy, TypeParams,
false);
2654 llvm::FunctionCallee
2656 assert((IVSize == 32 || IVSize == 64) &&
2657 "IV size is not compatible with the omp runtime");
2660 ? (IVSigned ?
"__kmpc_dispatch_next_4" :
"__kmpc_dispatch_next_4u")
2661 : (IVSigned ?
"__kmpc_dispatch_next_8" :
"__kmpc_dispatch_next_8u");
2663 auto *PtrTy = llvm::PointerType::getUnqual(ITy);
2673 llvm::FunctionType::get(
CGM.
Int32Ty, TypeParams,
false);
2681 unsigned &DeviceID,
unsigned &
FileID,
2682 unsigned &LineNum) {
2688 assert(Loc.
isValid() &&
"Source location is expected to be always valid.");
2691 assert(PLoc.isValid() &&
"Source location is expected to be always valid.");
2693 llvm::sys::fs::UniqueID
ID;
2694 if (
auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(),
ID))
2696 << PLoc.getFilename() << EC.message();
2698 DeviceID = ID.getDevice();
2699 FileID = ID.getFile();
2700 LineNum = PLoc.getLine();
2707 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2708 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link ||
2709 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2713 llvm::raw_svector_ostream OS(PtrName);
2719 DeviceID, FileID,
Line);
2720 OS << llvm::format(
"_%x", FileID);
2722 OS <<
"_decl_tgt_ref_ptr";
2730 auto *GV = cast<llvm::GlobalVariable>(Ptr);
2731 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
2747 std::string Suffix =
getName({
"cache",
""});
2783 Ctor, CopyCtor, Dtor};
2799 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
2809 Args.push_back(&Dst);
2814 std::string Name =
getName({
"__kmpc_global_ctor_",
""});
2815 llvm::Function *Fn =
2842 Args.push_back(&Dst);
2847 std::string Name =
getName({
"__kmpc_global_dtor_",
""});
2848 llvm::Function *Fn =
2869 auto *CopyCtorTy = llvm::FunctionType::get(
CGM.
VoidPtrTy, CopyCtorTyArgs,
2875 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
2876 if (Ctor ==
nullptr) {
2880 Ctor = llvm::Constant::getNullValue(CtorTy);
2882 if (Dtor ==
nullptr) {
2886 Dtor = llvm::Constant::getNullValue(DtorTy);
2889 auto *InitFunctionTy =
2890 llvm::FunctionType::get(
CGM.
VoidTy,
false);
2891 std::string Name =
getName({
"__omp_threadprivate_init_",
""});
2901 return InitFunction;
2909 llvm::GlobalVariable *Addr,
2915 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2916 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
2917 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
2936 llvm::raw_svector_ostream OS(Buffer);
2937 OS <<
"__omp_offloading_" << llvm::format(
"_%x", DeviceID)
2938 << llvm::format(
"_%x_", FileID) << VD->
getName() <<
"_l" <<
Line;
2943 llvm::Constant *Ctor;
2953 FTy, Twine(Buffer,
"_ctor"), FI, Loc);
2964 ID = llvm::ConstantExpr::getBitCast(Fn,
CGM.
Int8PtrTy);
2967 Ctor =
new llvm::GlobalVariable(
2969 llvm::GlobalValue::PrivateLinkage,
2970 llvm::Constant::getNullValue(
CGM.
Int8Ty), Twine(Buffer,
"_ctor"));
2977 DeviceID, FileID, Twine(Buffer,
"_ctor").toStringRef(Out), Line, Ctor,
2981 llvm::Constant *Dtor;
2991 FTy, Twine(Buffer,
"_dtor"), FI, Loc);
3003 ID = llvm::ConstantExpr::getBitCast(Fn,
CGM.
Int8PtrTy);
3006 Dtor =
new llvm::GlobalVariable(
3008 llvm::GlobalValue::PrivateLinkage,
3009 llvm::Constant::getNullValue(
CGM.
Int8Ty), Twine(Buffer,
"_dtor"));
3015 DeviceID, FileID, Twine(Buffer,
"_dtor").toStringRef(Out), Line, Dtor,
3024 std::string Suffix =
getName({
"artificial",
""});
3030 cast<llvm::GlobalVariable>(GAddr)->setThreadLocal(
true);
3033 std::string CacheSuffix =
getName({
"cache",
""});
3046 VarLVType->getPointerTo(0)),
3090 llvm::Function *OutlinedFn,
3092 const Expr *IfCond) {
3096 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](
CodeGenFunction &CGF,
3102 CGF.Builder.getInt32(CapturedVars.size()),
3103 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
3105 RealArgs.append(std::begin(Args), std::end(Args));
3106 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
3108 llvm::FunctionCallee RTLFn =
3110 CGF.EmitRuntimeCall(RTLFn, RealArgs);
3112 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](
CodeGenFunction &CGF,
3119 CGF.EmitRuntimeCall(
3125 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
3126 ".bound.zero.addr");
3127 CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32( 0));
3130 OutlinedFnArgs.push_back(ThreadIDAddr.
getPointer());
3131 OutlinedFnArgs.push_back(ZeroAddrBound.
getPointer());
3132 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
3133 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
3137 CGF.EmitRuntimeCall(
3157 if (
auto *OMPRegionInfo =
3159 if (OMPRegionInfo->getThreadIDVariable())
3160 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF);
3169 return ThreadIDTemp;
3173 llvm::Type *Ty,
const llvm::Twine &Name,
unsigned AddressSpace) {
3175 llvm::raw_svector_ostream Out(Buffer);
3177 StringRef RuntimeName = Out.str();
3178 auto &Elem = *InternalVars.try_emplace(RuntimeName,
nullptr).first;
3180 assert(Elem.second->getType()->getPointerElementType() == Ty &&
3181 "OMP internal variable has different type than requested");
3182 return &*Elem.second;
3185 return Elem.second =
new llvm::GlobalVariable(
3187 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
3188 Elem.first(),
nullptr,
3189 llvm::GlobalValue::NotThreadLocal, AddressSpace);
3193 std::string Prefix = Twine(
"gomp_critical_user_", CriticalName).str();
3194 std::string Name =
getName({Prefix,
"var"});
3201 llvm::FunctionCallee EnterCallee;
3203 llvm::FunctionCallee ExitCallee;
3206 llvm::BasicBlock *ContBlock =
nullptr;
3209 CommonActionTy(llvm::FunctionCallee EnterCallee,
3211 llvm::FunctionCallee ExitCallee,
3213 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
3222 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
3238 StringRef CriticalName,
3252 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
3255 CommonActionTy Action(
3289 llvm::ConstantInt::get(
CGM.
IntTy, 0,
true)};
3291 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
3292 Region->emitUntiedSwitch(CGF);
3315 unsigned Index,
const VarDecl *Var) {
3328 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
3329 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
3338 Args.push_back(&LHSArg);
3339 Args.push_back(&RHSArg);
3348 Fn->setDoesNotRecurse();
3363 for (
unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
3364 const auto *DestVar =
3365 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());
3368 const auto *SrcVar =
3369 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());
3372 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
3374 CGF.
EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);
3383 ArrayRef<const Expr *> CopyprivateVars,
3384 ArrayRef<const Expr *> SrcExprs,
3385 ArrayRef<const Expr *> DstExprs,
3386 ArrayRef<const Expr *> AssignmentOps) {
3389 assert(CopyprivateVars.size() == SrcExprs.size() &&
3390 CopyprivateVars.size() == DstExprs.size() &&
3391 CopyprivateVars.size() == AssignmentOps.size());
3403 if (!CopyprivateVars.empty()) {
3406 C.getIntTypeForBitwidth(32, 1);
3407 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
3425 llvm::APInt ArraySize(32, CopyprivateVars.size());
3426 QualType CopyprivateArrayTy = C.getConstantArrayType(
3431 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
3432 for (
unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
3444 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc);
3485 if (Kind == OMPD_for)
3486 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
3487 else if (Kind == OMPD_sections)
3488 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
3489 else if (Kind == OMPD_single)
3490 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
3491 else if (Kind == OMPD_barrier)
3492 Flags = OMP_IDENT_BARRIER_EXPL;
3494 Flags = OMP_IDENT_BARRIER_IMPL;
3506 ScheduleKind = OMPC_SCHEDULE_static;
3518 bool ForceSimpleCall) {
3520 auto *OMPRegionInfo =
3524 CGF.
Builder.restoreIP(OMPBuilder->CreateBarrier(
3525 CGF.
Builder, Kind, ForceSimpleCall, EmitChecks));
3538 if (OMPRegionInfo) {
3539 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
3549 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3565 bool Chunked,
bool Ordered) {
3566 switch (ScheduleKind) {
3567 case OMPC_SCHEDULE_static:
3570 case OMPC_SCHEDULE_dynamic:
3572 case OMPC_SCHEDULE_guided:
3574 case OMPC_SCHEDULE_runtime:
3576 case OMPC_SCHEDULE_auto:
3579 assert(!Chunked &&
"chunk was specified but schedule kind not known");
3582 llvm_unreachable(
"Unexpected runtime schedule");
3593 bool Chunked)
const {
3606 bool Chunked)
const {
3630 case OMPC_SCHEDULE_MODIFIER_monotonic:
3633 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3636 case OMPC_SCHEDULE_MODIFIER_simd:
3645 case OMPC_SCHEDULE_MODIFIER_monotonic:
3648 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
3651 case OMPC_SCHEDULE_MODIFIER_simd:
3665 if (CGM.
getLangOpts().OpenMP >= 50 && Modifier == 0) {
3683 ScheduleKind.
Schedule, DispatchValues.
Chunk !=
nullptr, Ordered);
3695 : CGF.
Builder.getIntN(IVSize, 1);
3700 CGM, Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
3703 CGF.
Builder.getIntN(IVSize, 1),
3713 const CGOpenMPRuntime::StaticRTInput &Values) {
3717 assert(!Values.Ordered);
3730 if (Chunk ==
nullptr) {
3733 "expected static non-chunked schedule");
3735 Chunk = CGF.
Builder.getIntN(Values.IVSize, 1);
3741 "expected static chunked schedule");
3748 Values.IL.getPointer(),
3749 Values.LB.getPointer(),
3750 Values.UB.getPointer(),
3751 Values.ST.getPointer(),
3752 CGF.
Builder.getIntN(Values.IVSize, 1),
3766 "Expected loop-based or sections-based directive.");
3769 ? OMP_IDENT_WORK_LOOP
3770 : OMP_IDENT_WORK_SECTIONS);
3772 llvm::FunctionCallee StaticInitFunction =
3775 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, Values);
3781 const CGOpenMPRuntime::StaticRTInput &Values) {
3787 llvm::FunctionCallee StaticInitFunction =
3803 ? OMP_IDENT_WORK_DISTRIBUTE
3805 ? OMP_IDENT_WORK_LOOP
3806 : OMP_IDENT_WORK_SECTIONS),
3861 ProcBindKind ProcBind,
3865 assert(ProcBind != OMP_PROC_BIND_unknown &&
"Unsupported proc_bind value.");
3869 llvm::ConstantInt::get(
CGM.
IntTy,
unsigned(ProcBind),
true)};
3908 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty()
const {
3909 return OffloadEntriesTargetRegion.empty() &&
3910 OffloadEntriesDeviceGlobalVar.empty();
3914 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3915 initializeTargetRegionEntryInfo(
unsigned DeviceID,
unsigned FileID,
3916 StringRef ParentName,
unsigned LineNum,
3918 assert(
CGM.
getLangOpts().OpenMPIsDevice &&
"Initialization of entries is " 3919 "only required for the device " 3920 "code generation.");
3921 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
3922 OffloadEntryInfoTargetRegion(Order,
nullptr,
nullptr,
3923 OMPTargetRegionEntryTargetRegion);
3924 ++OffloadingEntriesNum;
3927 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3928 registerTargetRegionEntryInfo(
unsigned DeviceID,
unsigned FileID,
3929 StringRef ParentName,
unsigned LineNum,
3930 llvm::Constant *Addr, llvm::Constant *
ID,
3931 OMPTargetRegionEntryKind Flags) {
3935 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) {
3938 "Unable to find target region on line '%0' in the device code.");
3943 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
3944 assert(Entry.isValid() &&
"Entry not initialized!");
3945 Entry.setAddress(Addr);
3947 Entry.setFlags(Flags);
3949 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
3950 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
3951 ++OffloadingEntriesNum;
3955 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
3956 unsigned DeviceID,
unsigned FileID, StringRef ParentName,
3957 unsigned LineNum)
const {
3958 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
3959 if (PerDevice == OffloadEntriesTargetRegion.end())
3961 auto PerFile = PerDevice->second.find(FileID);
3962 if (PerFile == PerDevice->second.end())
3964 auto PerParentName = PerFile->second.find(ParentName);
3965 if (PerParentName == PerFile->second.end())
3967 auto PerLine = PerParentName->second.find(LineNum);
3968 if (PerLine == PerParentName->second.end())
3971 if (PerLine->second.getAddress() || PerLine->second.getID())
3976 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
3977 const OffloadTargetRegionEntryInfoActTy &Action) {
3979 for (
const auto &D : OffloadEntriesTargetRegion)
3980 for (
const auto &F : D.second)
3981 for (
const auto &
P : F.second)
3982 for (
const auto &L :
P.second)
3983 Action(D.first, F.first,
P.first(), L.first, L.second);
3986 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3987 initializeDeviceGlobalVarEntryInfo(StringRef Name,
3988 OMPTargetGlobalVarEntryKind Flags,
3990 assert(
CGM.
getLangOpts().OpenMPIsDevice &&
"Initialization of entries is " 3991 "only required for the device " 3992 "code generation.");
3993 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags);
3994 ++OffloadingEntriesNum;
3997 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
3998 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr,
4000 OMPTargetGlobalVarEntryKind Flags,
4001 llvm::GlobalValue::LinkageTypes
Linkage) {
4003 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
4004 assert(Entry.isValid() && Entry.getFlags() == Flags &&
4005 "Entry not initialized!");
4006 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&
4007 "Resetting with the new address.");
4008 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
4009 if (Entry.getVarSize().isZero()) {
4010 Entry.setVarSize(VarSize);
4011 Entry.setLinkage(Linkage);
4015 Entry.setVarSize(VarSize);
4016 Entry.setLinkage(Linkage);
4017 Entry.setAddress(Addr);
4019 if (hasDeviceGlobalVarEntryInfo(VarName)) {
4020 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
4021 assert(Entry.isValid() && Entry.getFlags() == Flags &&
4022 "Entry not initialized!");
4023 assert((!Entry.getAddress() || Entry.getAddress() == Addr) &&
4024 "Resetting with the new address.");
4025 if (Entry.getVarSize().isZero()) {
4026 Entry.setVarSize(VarSize);
4027 Entry.setLinkage(Linkage);
4031 OffloadEntriesDeviceGlobalVar.try_emplace(
4032 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage);
4033 ++OffloadingEntriesNum;
4037 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
4038 actOnDeviceGlobalVarEntriesInfo(
4039 const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
4041 for (
const auto &E : OffloadEntriesDeviceGlobalVar)
4042 Action(E.getKey(), E.getValue());
4046 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags,
4047 llvm::GlobalValue::LinkageTypes Linkage) {
4048 StringRef Name = Addr->getName();
4050 llvm::LLVMContext &C = M.getContext();
4053 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
4055 std::string StringName =
getName({
"omp_offloading",
"entry_name"});
4056 auto *Str =
new llvm::GlobalVariable(
4057 M, StrPtrInit->getType(),
true,
4059 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
4061 llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID,
CGM.
VoidPtrTy),
4063 llvm::ConstantInt::get(
CGM.
SizeTy, Size),
4066 std::string EntryName =
getName({
"omp_offloading",
"entry",
""});
4069 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
4072 Entry->setSection(
"omp_offloading_entries");
4091 llvm::LLVMContext &C = M.getContext();
4100 auto &&GetMDInt = [
this](
unsigned V) {
4101 return llvm::ConstantAsMetadata::get(
4105 auto &&GetMDString = [&C](StringRef
V) {
return llvm::MDString::get(C, V); };
4108 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata(
"omp_offload.info");
4111 auto &&TargetRegionMetadataEmitter =
4112 [
this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt,
4114 unsigned DeviceID,
unsigned FileID, StringRef ParentName,
4127 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID),
4128 GetMDInt(FileID), GetMDString(ParentName),
4129 GetMDInt(Line), GetMDInt(E.getOrder())};
4135 if (I->getFirst()->getUniqueID().getDevice() == DeviceID &&
4136 I->getFirst()->getUniqueID().getFile() == FileID) {
4138 I->getFirst(),
Line, 1);
4143 OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName);
4144 ParentFunctions[E.getOrder()] = ParentName;
4147 MD->addOperand(llvm::MDNode::get(C, Ops));
4151 TargetRegionMetadataEmitter);
4154 auto &&DeviceGlobalVarMetadataEmitter =
4155 [&C, &OrderedEntries, &GetMDInt, &GetMDString,
4156 MD](StringRef MangledName,
4166 llvm::Metadata *Ops[] = {
4167 GetMDInt(E.getKind()), GetMDString(MangledName),
4168 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
4171 OrderedEntries[E.getOrder()] =
4172 std::make_tuple(&E, SourceLocation(), MangledName);
4175 MD->addOperand(llvm::MDNode::get(C, Ops));
4179 DeviceGlobalVarMetadataEmitter);
4181 for (
const auto &E : OrderedEntries) {
4182 assert(std::get<0>(E) &&
"All ordered entries must exist!");
4183 if (
const auto *CE =
4184 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
4186 if (!CE->getID() || !CE->getAddress()) {
4188 StringRef FnName = ParentFunctions[CE->getOrder()];
4193 "Offloading entry for target region in %0 is incorrect: either the " 4194 "address or the ID is invalid.");
4199 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage);
4201 OffloadEntryInfoDeviceGlobalVar>(
4211 if (!CE->getAddress()) {
4214 "variable %0 is incorrect: the " 4215 "address is invalid.");
4220 if (CE->getVarSize().isZero())
4225 assert(((
CGM.
getLangOpts().OpenMPIsDevice && !CE->getAddress()) ||
4227 "Declaret target link address is set.");
4230 if (!CE->getAddress()) {
4233 "Offloading entry for declare target variable is incorrect: the " 4234 "address is invalid.");
4241 CE->getVarSize().getQuantity(), Flags,
4244 llvm_unreachable(
"Unsupported entry kind.");
4262 if (
auto EC = Buf.getError()) {
4268 llvm::LLVMContext C;
4269 auto ME = expectedToErrorOrAndEmitErrors(
4270 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C));
4272 if (
auto EC = ME.getError()) {
4280 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata(
"omp_offload.info");
4284 for (llvm::MDNode *MN : MD->operands()) {
4285 auto &&GetMDInt = [MN](
unsigned Idx) {
4286 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
4287 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
4290 auto &&GetMDString = [MN](
unsigned Idx) {
4291 auto *V = cast<llvm::MDString>(MN->getOperand(Idx));
4292 return V->getString();
4295 switch (GetMDInt(0)) {
4297 llvm_unreachable(
"Unexpected metadata!");
4302 GetMDInt(1), GetMDInt(2),
4303 GetMDString(3), GetMDInt(4),
4310 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>(
4319 if (!KmpRoutineEntryPtrTy) {
4353 RD->
addAttr(PackedAttr::CreateImplicit(C));
4360 struct PrivateHelpersTy {
4361 PrivateHelpersTy(
const VarDecl *Original,
const VarDecl *PrivateCopy,
4362 const VarDecl *PrivateElemInit)
4363 : Original(Original), PrivateCopy(PrivateCopy),
4364 PrivateElemInit(PrivateElemInit) {}
4367 const VarDecl *PrivateElemInit;
4369 typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
4374 if (!Privates.empty()) {
4381 for (
const auto &Pair : Privates) {
4382 const VarDecl *VD = Pair.second.Original;
4401 QualType KmpRoutineEntryPointerQTy) {
4446 ArrayRef<PrivateDataTy>
Privates) {
4472 static llvm::Function *
4475 QualType KmpTaskTWithPrivatesPtrQTy,
4477 QualType SharedsPtrTy, llvm::Function *TaskFunction,
4486 Args.push_back(&GtidArg);
4487 Args.push_back(&TaskTypeArg);
4488 const auto &TaskEntryFnInfo =
4490 llvm::FunctionType *TaskEntryTy =
4496 TaskEntry->setDoesNotRecurse();
4511 const auto *KmpTaskTWithPrivatesQTyRD =
4512 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
4515 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
4516 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
4518 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
4520 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
4526 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
4528 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
4533 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
4536 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
4543 std::end(CommonArgs));
4545 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
4548 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
4551 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
4554 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
4557 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);
4560 CallArgs.push_back(LBParam);
4561 CallArgs.push_back(UBParam);
4562 CallArgs.push_back(StParam);
4563 CallArgs.push_back(LIParam);
4564 CallArgs.push_back(RParam);
4566 CallArgs.push_back(SharedsParam);
4579 QualType KmpTaskTWithPrivatesPtrQTy,
4580 QualType KmpTaskTWithPrivatesQTy) {
4588 Args.push_back(&GtidArg);
4589 Args.push_back(&TaskTypeArg);
4590 const auto &DestructorFnInfo =
4592 llvm::FunctionType *DestructorFnTy =
4596 auto *DestructorFn =
4601 DestructorFn->setDoesNotRecurse();
4609 const auto *KmpTaskTWithPrivatesQTyRD =
4610 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
4611 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
4613 for (
const auto *Field :
4614 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
4616 Field->getType().isDestructedType()) {
4622 return DestructorFn;
4637 ArrayRef<const Expr *> PrivateVars,
4638 ArrayRef<const Expr *> FirstprivateVars,
4639 ArrayRef<const Expr *> LastprivateVars,
4641 ArrayRef<PrivateDataTy>
Privates) {
4645 C,
nullptr, Loc,
nullptr,
4648 Args.push_back(&TaskPrivatesArg);
4649 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
4650 unsigned Counter = 1;
4651 for (
const Expr *E : PrivateVars) {
4653 C,
nullptr, Loc,
nullptr,
4658 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4659 PrivateVarsPos[VD] = Counter;
4662 for (
const Expr *E : FirstprivateVars) {
4664 C,
nullptr, Loc,
nullptr,
4669 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4670 PrivateVarsPos[VD] = Counter;
4673 for (
const Expr *E : LastprivateVars) {
4675 C,
nullptr, Loc,
nullptr,
4680 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4681 PrivateVarsPos[VD] = Counter;
4684 const auto &TaskPrivatesMapFnInfo =
4686 llvm::FunctionType *TaskPrivatesMapTy =
4694 TaskPrivatesMapFnInfo);
4696 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);
4697 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);
4698 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
4702 TaskPrivatesMapFnInfo, Args, Loc, Loc);
4708 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
4710 for (
const FieldDecl *Field : PrivatesQTyRD->fields()) {
4712 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
4721 return TaskPrivatesMap;
4731 ArrayRef<PrivateDataTy>
Privates,
bool ForDup) {
4733 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
4748 (IsTargetTask && KmpTaskSharedsPtr.
isValid())) {
4754 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
4755 for (
const PrivateDataTy &Pair : Privates) {
4756 const VarDecl *VD = Pair.second.PrivateCopy;
4758 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
4761 if (
const VarDecl *Elem = Pair.second.PrivateElemInit) {
4762 const VarDecl *OriginalVD = Pair.second.Original;
4768 if (IsTargetTask && !SharedField) {
4769 assert(isa<ImplicitParamDecl>(OriginalVD) &&
4772 ->getNumParams() == 0 &&
4773 isa<TranslationUnitDecl>(
4775 ->getDeclContext()) &&
4776 "Expected artificial target data variable.");
4798 [&CGF, Elem, Init, &CapturesInfo](
Address DestElement,
4803 Elem, [SrcElement]() ->
Address {
return SrcElement; });
4807 CGF, &CapturesInfo);
4808 CGF.EmitAnyExprToMem(Init, DestElement,
4809 Init->getType().getQualifiers(),
4833 ArrayRef<PrivateDataTy>
Privates) {
4834 bool InitRequired =
false;
4835 for (
const PrivateDataTy &Pair : Privates) {
4836 const VarDecl *VD = Pair.second.PrivateCopy;
4838 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
4843 return InitRequired;
4860 QualType KmpTaskTWithPrivatesPtrQTy,
4864 ArrayRef<PrivateDataTy>
Privates,
bool WithLastIter) {
4868 KmpTaskTWithPrivatesPtrQTy,
4871 KmpTaskTWithPrivatesPtrQTy,
4875 Args.push_back(&DstArg);
4876 Args.push_back(&SrcArg);
4877 Args.push_back(&LastprivArg);
4878 const auto &TaskDupFnInfo =
4885 TaskDup->setDoesNotRecurse();
4895 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
4897 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
4905 assert(!Privates.empty());
4912 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
4920 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
4921 SharedsTy, SharedsPtrTy, Data, Privates,
true);
4930 bool NeedsCleanup =
false;
4931 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin(), 1);
4932 const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
4933 for (
const FieldDecl *FD : PrivateRD->fields()) {
4934 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
4938 return NeedsCleanup;
4941 CGOpenMPRuntime::TaskResultTy
4944 llvm::Function *TaskFunction,
QualType SharedsTy,
4951 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4952 Privates.emplace_back(
4954 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4961 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4962 Privates.emplace_back(
4965 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4966 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));
4972 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
4973 Privates.emplace_back(
4975 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
4979 llvm::stable_sort(Privates, [](PrivateDataTy L, PrivateDataTy R) {
4980 return L.first > R.first;
4996 "Expected taskloop, task or target directive");
5005 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
5008 QualType KmpTaskTWithPrivatesPtrQTy =
5012 KmpTaskTWithPrivatesTy->getPointerTo();
5020 std::next(TaskFunction->arg_begin(), 3)->getType();
5021 if (!Privates.empty()) {
5022 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
5027 TaskPrivatesMap, TaskPrivatesMapTy);
5029 TaskPrivatesMap = llvm::ConstantPointerNull::get(
5030 cast<llvm::PointerType>(TaskPrivatesMapTy));
5036 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
5048 DestructorsFlag = 0x8,
5051 unsigned Flags = Data.
Tied ? TiedFlag : 0;
5052 bool NeedsCleanup =
false;
5053 if (!Privates.empty()) {
5056 Flags = Flags | DestructorsFlag;
5059 Flags = Flags | PriorityFlag;
5061 Data.
Final.getPointer()
5063 CGF.
Builder.getInt32(FinalFlag),
5065 : CGF.
Builder.getInt32(Data.
Final.getInt() ? FinalFlag : 0);
5066 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
5069 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
5071 TaskEntry, KmpRoutineEntryPtrTy)};
5075 const Expr *Device =
nullptr;
5077 Device = C->getDevice();
5084 DeviceID = CGF.
Builder.getInt64(OMP_DEVICEID_UNDEF);
5085 AllocArgs.push_back(DeviceID);
5094 NewTask, KmpTaskTWithPrivatesPtrTy);
5096 KmpTaskTWithPrivatesQTy);
5106 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
5116 if (!Privates.empty()) {
5117 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
5118 SharedsTy, SharedsPtrTy, Data, Privates,
5123 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
5124 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
5129 enum {
Priority = 0, Destructors = 1 };
5131 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
5133 (*FI)->getType()->getAsUnionType()->getDecl();
5136 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
5137 KmpTaskTWithPrivatesQTy);
5140 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
5142 DestructorFn, KmpRoutineEntryPtrTy),
5148 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
5150 Data2LV, *std::next(KmpCmplrdataUD->field_begin(),
Priority));
5163 llvm::Function *TaskFunction,
5171 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5173 llvm::Function *TaskEntry = Result.
TaskEntry;
5180 unsigned NumDependencies = Data.
Dependences.size();
5181 if (NumDependencies) {
5183 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3, DepMutexInOutSet = 0x4 };
5184 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
5207 for (
unsigned I = 0; I < NumDependencies; ++I) {
5212 if (
const auto *ASE =
5221 Size = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
5230 Base, *std::next(KmpDependInfoRD->
field_begin(), BaseAddr));
5236 Base, *std::next(KmpDependInfoRD->
field_begin(), Len));
5239 RTLDependenceKindTy DepKind;
5241 case OMPC_DEPEND_in:
5245 case OMPC_DEPEND_out:
5246 case OMPC_DEPEND_inout:
5249 case OMPC_DEPEND_mutexinoutset:
5250 DepKind = DepMutexInOutSet;
5252 case OMPC_DEPEND_source:
5253 case OMPC_DEPEND_sink:
5255 llvm_unreachable(
"Unknown task dependence type");
5258 Base, *std::next(KmpDependInfoRD->
field_begin(), Flags));
5274 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
5276 if (NumDependencies) {
5277 DepTaskArgs[0] = UpLoc;
5278 DepTaskArgs[1] = ThreadID;
5279 DepTaskArgs[2] = NewTask;
5280 DepTaskArgs[3] = CGF.
Builder.getInt32(NumDependencies);
5281 DepTaskArgs[4] = DependenciesArray.
getPointer();
5282 DepTaskArgs[5] = CGF.
Builder.getInt32(0);
5283 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
5285 auto &&ThenCodeGen = [
this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies,
5289 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
5290 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
5291 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
5293 if (NumDependencies) {
5294 CGF.EmitRuntimeCall(
5302 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
5303 Region->emitUntiedSwitch(CGF);
5307 if (NumDependencies) {
5308 DepWaitTaskArgs[0] = UpLoc;
5309 DepWaitTaskArgs[1] = ThreadID;
5310 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
5311 DepWaitTaskArgs[3] = DependenciesArray.
getPointer();
5312 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
5313 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
5315 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
5316 NumDependencies, &DepWaitTaskArgs,
5324 if (NumDependencies)
5328 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
5331 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
5332 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
5341 CommonActionTy Action(
5358 llvm::Function *TaskFunction,
5365 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
5378 IfVal = llvm::ConstantInt::getSigned(CGF.
IntTy, 1);
5415 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
5424 llvm::ConstantInt::getSigned(
5426 llvm::ConstantInt::getSigned(
5428 ? Data.
Schedule.getInt() ? NumTasks : Grainsize
5433 : llvm::ConstantInt::get(CGF.
Int64Ty, 0),
5436 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy)};
5453 const Expr *,
const Expr *)> &RedOpGen,
5454 const Expr *XExpr =
nullptr,
const Expr *EExpr =
nullptr,
5455 const Expr *UpExpr =
nullptr) {
5473 CGF.
Builder.CreateICmpEQ(LHSBegin, LHSEnd,
"omp.arraycpy.isempty");
5474 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
5477 llvm::BasicBlock *EntryBB = CGF.
Builder.GetInsertBlock();
5482 llvm::PHINode *RHSElementPHI = CGF.
Builder.CreatePHI(
5483 RHSBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
5484 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
5489 llvm::PHINode *LHSElementPHI = CGF.
Builder.CreatePHI(
5490 LHSBegin->getType(), 2,
"omp.arraycpy.destElementPast");
5491 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
5498 Scope.addPrivate(LHSVar, [=]() {
return LHSElementCurrent; });
5499 Scope.addPrivate(RHSVar, [=]() {
return RHSElementCurrent; });
5501 RedOpGen(CGF, XExpr, EExpr, UpExpr);
5502 Scope.ForceCleanup();
5506 LHSElementPHI, 1,
"omp.arraycpy.dest.element");
5508 RHSElementPHI, 1,
"omp.arraycpy.src.element");
5511 CGF.
Builder.CreateICmpEQ(LHSElementNext, LHSEnd,
"omp.arraycpy.done");
5512 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
5513 LHSElementPHI->addIncoming(LHSElementNext, CGF.
Builder.GetInsertBlock());
5514 RHSElementPHI->addIncoming(RHSElementNext, CGF.
Builder.GetInsertBlock());
5524 const Expr *ReductionOp) {
5525 if (
const auto *CE = dyn_cast<CallExpr>(ReductionOp))
5526 if (
const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
5527 if (
const auto *DRE =
5528 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
5529 if (
const auto *DRD =
5530 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
5531 std::pair<llvm::Function *, llvm::Function *> Reduction =
5543 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
5544 ArrayRef<const Expr *> ReductionOps) {
5553 Args.push_back(&LHSArg);
5554 Args.push_back(&RHSArg);
5557 std::string Name =
getName({
"omp",
"reduction",
"reduction_func"});
5562 Fn->setDoesNotRecurse();
5579 auto IPriv = Privates.begin();
5581 for (
unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5582 const auto *RHSVar =
5583 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());
5584 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() {
5587 const auto *LHSVar =
5588 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());
5589 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() {
5592 QualType PrivTy = (*IPriv)->getType();
5600 const auto *OVE = cast<OpaqueValueExpr>(VLA->
getSizeExpr());
5607 IPriv = Privates.begin();
5608 auto ILHS = LHSExprs.begin();
5609 auto IRHS = RHSExprs.begin();
5610 for (
const Expr *E : ReductionOps) {
5611 if ((*IPriv)->getType()->isArrayType()) {
5613 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5614 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5616 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
5628 Scope.ForceCleanup();
5634 const Expr *ReductionOp,
5635 const Expr *PrivateRef,
5640 const auto *LHSVar = cast<VarDecl>(LHS->
getDecl());
5641 const auto *RHSVar = cast<VarDecl>(RHS->
getDecl());
5643 CGF, PrivateRef->
getType(), LHSVar, RHSVar,
5702 if (SimpleReduction) {
5704 auto IPriv = Privates.begin();
5705 auto ILHS = LHSExprs.begin();
5706 auto IRHS = RHSExprs.begin();
5707 for (
const Expr *E : ReductionOps) {
5709 cast<DeclRefExpr>(*IRHS));
5719 auto Size = RHSExprs.size();
5720 for (
const Expr *E : Privates) {
5730 CGF.
CreateMemTemp(ReductionArrayTy,
".omp.reduction.red_list");
5731 auto IPriv = Privates.begin();
5733 for (
unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5739 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5756 LHSExprs, RHSExprs, ReductionOps);
5759 std::string Name =
getName({
"reduction"});
5772 CGF.
Builder.getInt32(RHSExprs.size()),
5773 ReductionArrayTySize,
5784 llvm::BasicBlock *DefaultBB = CGF.
createBasicBlock(
".omp.reduction.default");
5785 llvm::SwitchInst *SwInst =
5786 CGF.
Builder.CreateSwitch(Res, DefaultBB, 2);
5795 SwInst->addCase(CGF.
Builder.getInt32(1), Case1BB);
5804 auto &&CodeGen = [
Privates, LHSExprs, RHSExprs, ReductionOps](
5807 auto IPriv = Privates.begin();
5808 auto ILHS = LHSExprs.begin();
5809 auto IRHS = RHSExprs.begin();
5810 for (
const Expr *E : ReductionOps) {
5811 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS),
5812 cast<DeclRefExpr>(*IRHS));
5819 CommonActionTy Action(
5827 CGF.EmitBranch(DefaultBB);
5834 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(
".omp.reduction.case2");
5835 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
5836 CGF.EmitBlock(Case2BB);
5838 auto &&AtomicCodeGen = [Loc,
Privates, LHSExprs, RHSExprs, ReductionOps](
5840 auto ILHS = LHSExprs.begin();
5841 auto IRHS = RHSExprs.begin();
5842 auto IPriv = Privates.begin();
5843 for (
const Expr *E : ReductionOps) {
5844 const Expr *XExpr =
nullptr;
5845 const Expr *EExpr =
nullptr;
5846 const Expr *UpExpr =
nullptr;
5848 if (
const auto *BO = dyn_cast<BinaryOperator>(E)) {
5849 if (BO->getOpcode() == BO_Assign) {
5850 XExpr = BO->getLHS();
5851 UpExpr = BO->getRHS();
5855 const Expr *RHSExpr = UpExpr;
5858 if (
const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5862 RHSExpr = ACO->getCond();
5864 if (
const auto *BORHS =
5866 EExpr = BORHS->getRHS();
5867 BO = BORHS->getOpcode();
5871 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5872 auto &&AtomicRedGen = [BO, VD,
5874 const Expr *EExpr,
const Expr *UpExpr) {
5875 LValue X = CGF.EmitLValue(XExpr);
5878 E = CGF.EmitAnyExpr(EExpr);
5879 CGF.EmitOMPAtomicSimpleUpdateExpr(
5881 llvm::AtomicOrdering::Monotonic, Loc,
5882 [&CGF, UpExpr, VD, Loc](
RValue XRValue) {
5885 VD, [&CGF, VD, XRValue, Loc]() {
5886 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
5887 CGF.emitOMPSimpleStore(
5888 CGF.MakeAddrLValue(LHSTemp, VD->
getType()), XRValue,
5889 VD->getType().getNonReferenceType(), Loc);
5893 return CGF.EmitAnyExpr(UpExpr);
5896 if ((*IPriv)->getType()->isArrayType()) {
5898 const auto *RHSVar =
5899 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5901 AtomicRedGen, XExpr, EExpr, UpExpr);
5904 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5909 const Expr *,
const Expr *) {
5911 std::string Name = RT.getName({
"atomic_reduction"});
5912 RT.emitCriticalRegion(
5920 if ((*IPriv)->getType()->isArrayType()) {
5921 const auto *LHSVar =
5922 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
5923 const auto *RHSVar =
5924 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
5928 CritRedGen(CGF,
nullptr,
nullptr,
nullptr);
5953 CGF.EmitBranch(DefaultBB);
5954 CGF.EmitBlock(DefaultBB,
true);
5962 llvm::raw_svector_ostream Out(Buffer);
5966 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());
5969 {D->isLocalVarDeclOrParm() ? D->getName() : CGM.
getMangledName(D)});
5970 Out << Prefix << Name <<
"_" 5971 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
5990 Args.emplace_back(&Param);
5991 const auto &FnInfo =
5998 Fn->setDoesNotRecurse();
6030 llvm::ConstantPointerNull::get(CGM.
VoidPtrTy),
6055 const Expr *ReductionOp,
6057 const Expr *PrivateRef) {
6059 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());
6060 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());
6066 Args.emplace_back(&ParamInOut);
6067 Args.emplace_back(&ParamIn);
6068 const auto &FnInfo =
6075 Fn->setDoesNotRecurse();
6093 PrivateScope.
addPrivate(LHSVD, [&C, &CGF, &ParamInOut, LHSVD]() {
6101 PrivateScope.
addPrivate(RHSVD, [&C, &CGF, &ParamIn, RHSVD]() {
6114 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),
6115 cast<DeclRefExpr>(RHS));
6137 Args.emplace_back(&Param);
6138 const auto &FnInfo =
6145 Fn->setDoesNotRecurse();
6171 ArrayRef<const Expr *> RHSExprs,
const OMPTaskDataTy &Data) {
6204 for (
unsigned Cnt = 0; Cnt < Size; ++Cnt) {
6207 llvm::ConstantInt::get(
CGM.
SizeTy, Cnt)};
6215 RCG.emitSharedLValue(CGF, Cnt);
6222 std::tie(SizeValInChars, SizeVal) = RCG.
getSizes(Cnt);
6229 bool DelayedCreation = !!SizeVal;
6255 if (DelayedCreation) {
6257 llvm::ConstantInt::get(
CGM.
Int32Ty, 1,
true),
6268 llvm::ConstantInt::get(
CGM.
IntTy, Size,
true),
6329 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
6330 Region->emitUntiedSwitch(CGF);
6339 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
6355 if (CancelRegion == OMPD_parallel)
6356 CancelKind = CancelParallel;
6357 else if (CancelRegion == OMPD_for)
6358 CancelKind = CancelLoop;
6359 else if (CancelRegion == OMPD_sections)
6360 CancelKind = CancelSections;
6362 assert(CancelRegion == OMPD_taskgroup);
6363 CancelKind = CancelTaskgroup;
6375 if (
auto *OMPRegionInfo =
6379 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6392 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6410 if (
auto *OMPRegionInfo =
6412 auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](
CodeGenFunction &CGF,
6424 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
".cancel.exit");
6425 llvm::BasicBlock *ContBB = CGF.createBasicBlock(
".cancel.continue");
6426 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result);
6427 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
6428 CGF.EmitBlock(ExitBB);
6431 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
6432 CGF.EmitBranchThroughCleanup(CancelDest);
6433 CGF.EmitBlock(ContBB,
true);
6447 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6449 assert(!ParentName.empty() &&
"Invalid target region parent name!");
6452 IsOffloadEntry, CodeGen);
6457 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6475 llvm::raw_svector_ostream OS(EntryFnName);
6476 OS <<
"__omp_offloading" << llvm::format(
"_%x", DeviceID)
6477 << llvm::format(
"_%x_", FileID) << ParentName <<
"_l" <<
Line;
6483 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
6490 if (!IsOffloadEntry)
6505 OutlinedFnID = llvm::ConstantExpr::getBitCast(OutlinedFn,
CGM.
Int8PtrTy);
6506 OutlinedFn->setLinkage(llvm::GlobalValue::WeakAnyLinkage);
6507 OutlinedFn->setDSOLocal(
false);
6509 std::string Name =
getName({EntryFnName,
"region_id"});
6510 OutlinedFnID =
new llvm::GlobalVariable(
6512 llvm::GlobalValue::WeakAnyLinkage,
6513 llvm::Constant::getNullValue(
CGM.
Int8Ty), Name);
6518 DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID,
6535 while (
const auto *C = dyn_cast_or_null<CompoundStmt>(Child)) {
6537 for (
const Stmt *S : C->body()) {
6538 if (
const auto *E = dyn_cast<Expr>(S)) {
6543 if (isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) ||
6544 isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S))
6547 if (
const auto *DS = dyn_cast<DeclStmt>(S)) {
6548 if (llvm::all_of(DS->decls(), [&Ctx](
const Decl *D) {
6549 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||
6550 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||
6551 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||
6552 isa<UsingDirectiveDecl>(D) ||
6553 isa<OMPDeclareReductionDecl>(D) ||
6554 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))
6556 const auto *VD = dyn_cast<
VarDecl>(D);
6589 "Clauses associated with the teams directive expected to be emitted " 6590 "only for the host!");
6593 "Expected target-based executable directive.");
6595 switch (DirectiveKind) {
6600 const Stmt *ChildStmt =
6602 if (
const auto *NestedDir =
6603 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
6606 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6608 const Expr *NumTeams =
6613 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6616 return Bld.getInt32(0);
6620 return Bld.getInt32(1);
6621 return Bld.getInt32(0);
6625 case OMPD_target_teams:
6626 case OMPD_target_teams_distribute:
6627 case OMPD_target_teams_distribute_simd:
6628 case OMPD_target_teams_distribute_parallel_for:
6629 case OMPD_target_teams_distribute_parallel_for_simd: {
6632 const Expr *NumTeams =
6637 return Bld.CreateIntCast(NumTeamsVal, CGF.
Int32Ty,
6640 return Bld.getInt32(0);
6642 case OMPD_target_parallel:
6643 case OMPD_target_parallel_for:
6644 case OMPD_target_parallel_for_simd:
6645 case OMPD_target_simd:
6646 return Bld.getInt32(1);
6649 case OMPD_parallel_for:
6650 case OMPD_parallel_master:
6651 case OMPD_parallel_sections:
6653 case OMPD_parallel_for_simd:
6655 case OMPD_cancellation_point:
6657 case OMPD_threadprivate:
6666 case OMPD_taskyield:
6669 case OMPD_taskgroup:
6673 case OMPD_target_data:
6674 case OMPD_target_exit_data:
6675 case OMPD_target_enter_data:
6676 case OMPD_distribute:
6677 case OMPD_distribute_simd:
6678 case OMPD_distribute_parallel_for:
6679 case OMPD_distribute_parallel_for_simd:
6680 case OMPD_teams_distribute:
6681 case OMPD_teams_distribute_simd:
6682 case OMPD_teams_distribute_parallel_for:
6683 case OMPD_teams_distribute_parallel_for_simd:
6684 case OMPD_target_update:
6685 case OMPD_declare_simd:
6686 case OMPD_declare_variant:
6687 case OMPD_declare_target:
6688 case OMPD_end_declare_target:
6689 case OMPD_declare_reduction:
6690 case OMPD_declare_mapper:
6692 case OMPD_taskloop_simd:
6693 case OMPD_master_taskloop:
6694 case OMPD_master_taskloop_simd:
6695 case OMPD_parallel_master_taskloop:
6696 case OMPD_parallel_master_taskloop_simd:
6701 llvm_unreachable(
"Unexpected directive kind.");
6708 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6715 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6718 for (
const auto *C : Dir->getClausesOfKind<
OMPIfClause>()) {
6719 if (C->getNameModifier() == OMPD_unknown ||
6720 C->getNameModifier() == OMPD_parallel) {
6730 return CGF.
Builder.getInt32(1);
6733 if (
const auto *PreInit =
6735 for (
const auto *I : PreInit->decls()) {
6736 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6752 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6754 const auto *NumThreadsClause =
6757 CGF, NumThreadsClause->getNumThreads()->getSourceRange());
6758 if (
const auto *PreInit =
6759 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {
6760 for (
const auto *I : PreInit->decls()) {
6761 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6770 NumThreads = CGF.
EmitScalarExpr(NumThreadsClause->getNumThreads());
6773 if (DefaultThreadLimitVal)
6774 NumThreads = CGF.
Builder.CreateSelect(
6775 CGF.
Builder.CreateICmpULT(DefaultThreadLimitVal, NumThreads),
6776 DefaultThreadLimitVal, NumThreads);
6778 NumThreads = DefaultThreadLimitVal ? DefaultThreadLimitVal
6783 NumThreads = CGF.
Builder.CreateSelect(CondVal, NumThreads,
6789 return CGF.
Builder.getInt32(1);
6790 return DefaultThreadLimitVal;
6792 return DefaultThreadLimitVal ? DefaultThreadLimitVal
6808 "Clauses associated with the teams directive expected to be emitted " 6809 "only for the host!");
6812 "Expected target-based executable directive.");
6816 switch (DirectiveKind) {
6823 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6825 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6827 const auto *ThreadLimitClause =
6830 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange());
6831 if (
const auto *PreInit =
6832 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {
6833 for (
const auto *I : PreInit->decls()) {
6834 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6844 ThreadLimitClause->getThreadLimit(),
true);
6846 Bld.CreateIntCast(ThreadLimit, CGF.
Int32Ty,
false);
6850 CS = Dir->getInnermostCapturedStmt();
6853 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);
6857 CS = Dir->getInnermostCapturedStmt();
6862 return Bld.getInt32(1);
6864 return ThreadLimitVal ? ThreadLimitVal : Bld.getInt32(0);
6866 case OMPD_target_teams: {
6871 ThreadLimitClause->getThreadLimit(),
true);
6873 Bld.CreateIntCast(ThreadLimit, CGF.
Int32Ty,
false);
6880 if (
const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {
6881 if (Dir->getDirectiveKind() == OMPD_distribute) {
6882 CS = Dir->getInnermostCapturedStmt();
6887 return ThreadLimitVal ? ThreadLimitVal : Bld.getInt32(0);
6889 case OMPD_target_teams_distribute:
6894 ThreadLimitClause->getThreadLimit(),
true);
6896 Bld.CreateIntCast(ThreadLimit, CGF.
Int32Ty,
false);
6899 case OMPD_target_parallel:
6900 case OMPD_target_parallel_for:
6901 case OMPD_target_parallel_for_simd:
6902 case OMPD_target_teams_distribute_parallel_for:
6903 case OMPD_target_teams_distribute_parallel_for_simd: {
6910 if (C->getNameModifier() == OMPD_unknown ||
6911 C->getNameModifier() == OMPD_parallel) {
6921 return Bld.getInt32(1);
6932 ThreadLimitClause->getThreadLimit(),
true);
6934 Bld.CreateIntCast(ThreadLimit, CGF.
Int32Ty,
false);
6940 NumThreadsClause->getNumThreads(),
true);
6942 Bld.CreateIntCast(NumThreads, CGF.
Int32Ty,
false);
6943 ThreadLimitVal = ThreadLimitVal
6944 ? Bld.CreateSelect(Bld.CreateICmpULT(NumThreadsVal,
6946 NumThreadsVal, ThreadLimitVal)
6949 if (!ThreadLimitVal)
6950 ThreadLimitVal = Bld.getInt32(0);
6952 return Bld.CreateSelect(CondVal, ThreadLimitVal, Bld.getInt32(1));
6953 return ThreadLimitVal;
6955 case OMPD_target_teams_distribute_simd:
6956 case OMPD_target_simd:
6957 return Bld.getInt32(1);
6960 case OMPD_parallel_for:
6961 case OMPD_parallel_master:
6962 case OMPD_parallel_sections:
6964 case OMPD_parallel_for_simd:
6966 case OMPD_cancellation_point:
6968 case OMPD_threadprivate:
6977 case OMPD_taskyield:
6980 case OMPD_taskgroup:
6984 case OMPD_target_data:
6985 case OMPD_target_exit_data:
6986 case OMPD_target_enter_data:
6987 case OMPD_distribute:
6988 case OMPD_distribute_simd:
6989 case OMPD_distribute_parallel_for:
6990 case OMPD_distribute_parallel_for_simd:
6991 case OMPD_teams_distribute:
6992 case OMPD_teams_distribute_simd:
6993 case OMPD_teams_distribute_parallel_for:
6994 case OMPD_teams_distribute_parallel_for_simd:
6995 case OMPD_target_update:
6996 case OMPD_declare_simd:
6997 case OMPD_declare_variant:
6998 case OMPD_declare_target:
6999 case OMPD_end_declare_target:
7000 case OMPD_declare_reduction:
7001 case OMPD_declare_mapper:
7003 case OMPD_taskloop_simd:
7004 case OMPD_master_taskloop:
7005 case OMPD_master_taskloop_simd:
7006 case OMPD_parallel_master_taskloop:
7007 case OMPD_parallel_master_taskloop_simd:
7012 llvm_unreachable(
"Unsupported directive kind.");
7022 class MappableExprsHandler {
7026 enum OpenMPOffloadMappingFlags : uint64_t {
7032 OMP_MAP_FROM = 0x02,
7035 OMP_MAP_ALWAYS = 0x04,
7038 OMP_MAP_DELETE = 0x08,
7041 OMP_MAP_PTR_AND_OBJ = 0x10,
7044 OMP_MAP_TARGET_PARAM = 0x20,
7048 OMP_MAP_RETURN_PARAM = 0x40,
7051 OMP_MAP_PRIVATE = 0x80,
7053 OMP_MAP_LITERAL = 0x100,
7055 OMP_MAP_IMPLICIT = 0x200,
7058 OMP_MAP_CLOSE = 0x400,
7061 OMP_MAP_MEMBER_OF = 0xffff000000000000,
7062 LLVM_MARK_AS_BITMASK_ENUM( OMP_MAP_MEMBER_OF),
7066 static unsigned getFlagMemberOffset() {
7068 for (uint64_t Remain = OMP_MAP_MEMBER_OF; !(Remain & 1);
7069 Remain = Remain >> 1)
7076 class BasePointerInfo {
7085 : Ptr(Ptr), DevPtrDecl(DevPtrDecl) {}
7087 const ValueDecl *getDevicePtrDecl()
const {
return DevPtrDecl; }
7088 void setDevicePtrDecl(
const ValueDecl *D) { DevPtrDecl = D; }
7099 struct StructRangeInfoTy {
7100 std::pair<
unsigned ,
Address > LowestElem = {
7102 std::pair<
unsigned ,
Address > HighestElem = {
7112 ArrayRef<OpenMPMapModifierKind> MapModifiers;
7113 bool ReturnDevicePointer =
false;
7114 bool IsImplicit =
false;
7116 MapInfo() =
default;
7120 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7121 bool ReturnDevicePointer,
bool IsImplicit)
7122 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
7123 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit) {}
7129 struct DeferredDevicePtrEntryTy {
7130 const Expr *IE =
nullptr;
7133 DeferredDevicePtrEntryTy(
const Expr *IE,
const ValueDecl *VD)
7149 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,
bool> FirstPrivateDecls;
7168 if (
const auto *OAE = dyn_cast<OMPArraySectionExpr>(E)) {
7170 OAE->getBase()->IgnoreParenImpCasts())
7171 .getCanonicalType();
7176 if (!OAE->getLength() && OAE->getColonLoc().isValid() &&
7177 !OAE->getLowerBound())
7182 ElemSize = CGF.
getTypeSize(PTy->getPointeeType().getCanonicalType());
7184 const auto *ATy = cast<ArrayType>(BaseTy.
getTypePtr());
7185 assert(ATy &&
"Expecting array type if not a pointer type.");
7186 ElemSize = CGF.
getTypeSize(ATy->getElementType().getCanonicalType());
7191 if (!OAE->getLength() && OAE->getColonLoc().isInvalid())
7194 if (
const Expr *LenExpr = OAE->getLength()) {
7198 LenExpr->getExprLoc());
7199 return CGF.
Builder.CreateNUWMul(LengthVal, ElemSize);
7201 assert(!OAE->getLength() && OAE->getColonLoc().isValid() &&
7202 OAE->getLowerBound() &&
"expected array_section[lb:].");
7208 OAE->getLowerBound()->getExprLoc());
7209 LBVal = CGF.
Builder.CreateNUWMul(LBVal, ElemSize);
7212 LengthVal = CGF.
Builder.CreateSelect(
7213 Cmp, TrueVal, llvm::ConstantInt::get(CGF.
SizeTy, 0));
7223 OpenMPOffloadMappingFlags getMapTypeBits(
7225 bool IsImplicit,
bool AddPtrFlag,
bool AddIsTargetParamFlag)
const {
7226 OpenMPOffloadMappingFlags Bits =
7227 IsImplicit ? OMP_MAP_IMPLICIT : OMP_MAP_NONE;
7229 case OMPC_MAP_alloc:
7230 case OMPC_MAP_release:
7240 Bits |= OMP_MAP_FROM;
7242 case OMPC_MAP_tofrom:
7243 Bits |= OMP_MAP_TO | OMP_MAP_FROM;
7245 case OMPC_MAP_delete:
7246 Bits |= OMP_MAP_DELETE;
7249 llvm_unreachable(
"Unexpected map type!");
7252 Bits |= OMP_MAP_PTR_AND_OBJ;
7253 if (AddIsTargetParamFlag)
7254 Bits |= OMP_MAP_TARGET_PARAM;
7255 if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_always)
7256 != MapModifiers.end())
7257 Bits |= OMP_MAP_ALWAYS;
7258 if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_close)
7259 != MapModifiers.end())
7260 Bits |= OMP_MAP_CLOSE;
7266 bool isFinalArraySectionExpression(
const Expr *E)
const {
7274 if (OASE->getColonLoc().isInvalid())
7277 const Expr *Length = OASE->getLength();
7284 OASE->getBase()->IgnoreParenImpCasts())
7285 .getCanonicalType();
7286 if (
const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.
getTypePtr()))
7287 return ATy->getSize().getSExtValue() != 1;
7300 return ConstLength.getSExtValue() != 1;
7307 void generateInfoForComponentList(
7309 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7311 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
7312 MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
7313 StructRangeInfoTy &PartialStruct,
bool IsFirstComponentList,
7315 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7474 bool IsCaptureFirstInfo = IsFirstComponentList;
7478 bool RequiresReference =
false;
7481 auto CI = Components.rbegin();
7482 auto CE = Components.rend();
7487 bool IsExpressionFirstInfo =
true;
7489 const Expr *AssocExpr = I->getAssociatedExpression();
7493 if (isa<MemberExpr>(AssocExpr)) {
7497 }
else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||
7499 isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {
7505 if (
const auto *VD =
7506 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {
7508 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
7509 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
7510 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
7512 RequiresReference =
true;
7522 I->getAssociatedDeclaration()->getType().getNonReferenceType();
7546 bool ShouldBeMemberOf =
false;
7557 for (; I != CE; ++I) {
7559 if (!EncounteredME) {
7560 EncounteredME = dyn_cast<
MemberExpr>(I->getAssociatedExpression());
7564 ShouldBeMemberOf =
true;
7567 auto Next = std::next(I);
7575 bool IsFinalArraySection =
7576 isFinalArraySectionExpression(I->getAssociatedExpression());
7589 if (Next == CE || IsPointer || IsFinalArraySection) {
7592 assert((Next == CE ||
7593 isa<MemberExpr>(Next->getAssociatedExpression()) ||
7594 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
7595 isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) &&
7596 "Unexpected expression");
7604 bool IsMemberPointer =
7605 IsPointer && EncounteredME &&
7606 (dyn_cast<
MemberExpr>(I->getAssociatedExpression()) ==
7608 if (!OverlappedElements.empty()) {
7610 assert(!PartialStruct.Base.isValid() &&
"The base element is set.");
7611 assert(Next == CE &&
7612 "Expected last element for the overlapped elements.");
7613 assert(!IsPointer &&
7614 "Unexpected base element with the pointer type.");
7617 PartialStruct.LowestElem = {0, LB};
7619 I->getAssociatedExpression()->getType());
7624 PartialStruct.HighestElem = {
7625 std::numeric_limits<decltype(
7626 PartialStruct.HighestElem.first)>
::max(),
7628 PartialStruct.Base = BP;
7630 OpenMPOffloadMappingFlags Flags =
7632 getMapTypeBits(MapType, MapModifiers, IsImplicit,
7639 Component : OverlappedElements) {
7643 if (MC.getAssociatedDeclaration()) {
7647 Size = CGF.
Builder.CreatePtrDiff(
7654 Pointers.push_back(LB.getPointer());
7657 Types.push_back(Flags);
7661 Pointers.push_back(LB.getPointer());
7662 Size = CGF.
Builder.CreatePtrDiff(
7668 Types.push_back(Flags);
7671 llvm::Value *Size = getExprTypeSize(I->getAssociatedExpression());
7672 if (!IsMemberPointer) {
7674 Pointers.push_back(LB.getPointer());
7682 OpenMPOffloadMappingFlags Flags = getMapTypeBits(
7683 MapType, MapModifiers, IsImplicit,
7684 !IsExpressionFirstInfo || RequiresReference,
7685 IsCaptureFirstInfo && !RequiresReference);
7687 if (!IsExpressionFirstInfo) {
7691 Flags &= ~(OMP_MAP_TO | OMP_MAP_FROM | OMP_MAP_ALWAYS |
7692 OMP_MAP_DELETE | OMP_MAP_CLOSE);
7694 if (ShouldBeMemberOf) {
7697 Flags |= OMP_MAP_MEMBER_OF;
7700 ShouldBeMemberOf =
false;
7704 Types.push_back(Flags);
7710 if (EncounteredME) {
7711 const auto *FD = dyn_cast<
FieldDecl>(EncounteredME->getMemberDecl());
7715 if (!PartialStruct.Base.isValid()) {
7716 PartialStruct.LowestElem = {FieldIndex, LB};
7717 PartialStruct.HighestElem = {FieldIndex, LB};
7718 PartialStruct.Base = BP;
7719 }
else if (FieldIndex < PartialStruct.LowestElem.first) {
7720 PartialStruct.LowestElem = {FieldIndex, LB};
7721 }
else if (FieldIndex > PartialStruct.HighestElem.first) {
7722 PartialStruct.HighestElem = {FieldIndex, LB};
7727 if (IsFinalArraySection)
7734 IsExpressionFirstInfo =
false;
7735 IsCaptureFirstInfo =
false;
7743 MappableExprsHandler::OpenMPOffloadMappingFlags
7753 return MappableExprsHandler::OMP_MAP_ALWAYS |
7754 MappableExprsHandler::OMP_MAP_TO;
7756 return MappableExprsHandler::OMP_MAP_TO |
7757 MappableExprsHandler::OMP_MAP_PTR_AND_OBJ;
7758 return MappableExprsHandler::OMP_MAP_PRIVATE |
7759 MappableExprsHandler::OMP_MAP_TO;
7761 return MappableExprsHandler::OMP_MAP_TO |
7762 MappableExprsHandler::OMP_MAP_FROM;
7765 static OpenMPOffloadMappingFlags getMemberOfFlag(
unsigned Position) {
7767 return static_cast<OpenMPOffloadMappingFlags
>(((uint64_t)Position + 1)
7768 << getFlagMemberOffset());
7771 static void setCorrectMemberOfFlag(OpenMPOffloadMappingFlags &Flags,
7772 OpenMPOffloadMappingFlags MemberOfFlag) {
7776 if ((Flags & OMP_MAP_PTR_AND_OBJ) &&
7777 ((Flags & OMP_MAP_MEMBER_OF) != OMP_MAP_MEMBER_OF))
7782 Flags &= ~OMP_MAP_MEMBER_OF;
7783 Flags |= MemberOfFlag;
7788 bool AsBase)
const {
7791 llvm::StructType *St =
7794 unsigned NumElements = St->getNumElements();
7796 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
7797 RecordLayout(NumElements);
7800 for (
const auto &I : RD->
bases()) {
7803 const auto *
Base = I.getType()->getAsCXXRecordDecl();
7812 RecordLayout[FieldIndex] =
Base;
7815 for (
const auto &I : RD->
vbases()) {
7816 const auto *
Base = I.getType()->getAsCXXRecordDecl();
7818 if (
Base->isEmpty())
7821 if (RecordLayout[FieldIndex])
7823 RecordLayout[FieldIndex] =
Base;
7826 assert(!RD->
isUnion() &&
"Unexpected union.");
7827 for (
const auto *Field : RD->
fields()) {
7830 if (!Field->isBitField() && !Field->isZeroSize(CGF.
getContext())) {
7832 RecordLayout[FieldIndex] = Field;
7835 for (
const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
7836 &Data : RecordLayout) {
7840 getPlainLayout(
Base, Layout,
true);
7842 Layout.push_back(Data.get<
const FieldDecl *>());
7847 MappableExprsHandler(
const OMPExecutableDirective &Dir,
CodeGenFunction &CGF)
7848 : CurDir(&Dir), CGF(CGF) {
7851 for (
const auto *D : C->varlists())
7852 FirstPrivateDecls.try_emplace(
7853 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl()), C->isImplicit());
7856 for (
auto L : C->component_lists())
7857 DevPointersMap[L.first].push_back(L.second);
7862 : CurDir(&Dir), CGF(CGF) {}
7867 void emitCombinedEntry(MapBaseValuesArrayTy &BasePointers,
7868 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
7869 MapFlagsArrayTy &Types, MapFlagsArrayTy &CurTypes,
7870 const StructRangeInfoTy &PartialStruct)
const {
7872 BasePointers.push_back(PartialStruct.Base.getPointer());
7874 llvm::Value *LB = PartialStruct.LowestElem.second.getPointer();
7875 Pointers.push_back(LB);
7877 llvm::Value *HB = PartialStruct.HighestElem.second.getPointer();
7884 Sizes.push_back(Size);
7886 Types.push_back(OMP_MAP_TARGET_PARAM);
7888 (*CurTypes.begin()) &= ~OMP_MAP_TARGET_PARAM;
7893 OpenMPOffloadMappingFlags MemberOfFlag =
7894 getMemberOfFlag(BasePointers.size() - 1);
7895 for (
auto &M : CurTypes)
7896 setCorrectMemberOfFlag(M, MemberOfFlag);
7903 void generateAllInfo(MapBaseValuesArrayTy &BasePointers,
7904 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
7905 MapFlagsArrayTy &Types)
const {
7909 llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info;
7913 auto &&InfoGen = [&Info](
7917 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7918 bool ReturnDevicePointer,
bool IsImplicit) {
7919 const ValueDecl *VD =
7921 Info[VD].emplace_back(L, MapType, MapModifiers, ReturnDevicePointer,
7925 assert(CurDir.is<
const OMPExecutableDirective *>() &&
7926 "Expect a executable directive");
7927 const auto *CurExecDir = CurDir.get<
const OMPExecutableDirective *>();
7928 for (
const auto *C : CurExecDir->getClausesOfKind<
OMPMapClause>())
7929 for (
const auto L : C->component_lists()) {
7930 InfoGen(L.first, L.second, C->getMapType(), C->getMapTypeModifiers(),
7931 false, C->isImplicit());
7933 for (
const auto *C : CurExecDir->getClausesOfKind<
OMPToClause>())
7934 for (
const auto L : C->component_lists()) {
7935 InfoGen(L.first, L.second, OMPC_MAP_to,
llvm::None,
7936 false, C->isImplicit());
7938 for (
const auto *C : CurExecDir->getClausesOfKind<
OMPFromClause>())
7939 for (
const auto L : C->component_lists()) {
7940 InfoGen(L.first, L.second, OMPC_MAP_from,
llvm::None,
7941 false, C->isImplicit());
7950 llvm::MapVector<const ValueDecl *, SmallVector<DeferredDevicePtrEntryTy, 4>>
7953 for (
const auto *C :
7955 for (
const auto L : C->component_lists()) {
7956 assert(!L.second.empty() &&
"Not expecting empty list of components!");
7957 const ValueDecl *VD = L.second.back().getAssociatedDeclaration();
7958 VD = cast<ValueDecl>(VD->getCanonicalDecl());
7959 const Expr *IE = L.second.back().getAssociatedExpression();
7963 auto It = Info.find(isa<MemberExpr>(IE) ?
nullptr : VD);
7967 if (It != Info.end()) {
7968 auto CI = std::find_if(
7969 It->second.begin(), It->second.end(), [VD](
const MapInfo &MI) {
7970 return MI.Components.back().getAssociatedDeclaration() == VD;
7974 if (CI != It->second.end()) {
7975 CI->ReturnDevicePointer =
true;
7983 if (isa<MemberExpr>(IE)) {
7992 false, C->isImplicit());
7993 DeferredInfo[
nullptr].emplace_back(IE, VD);
7997 BasePointers.emplace_back(Ptr, VD);
7998 Pointers.push_back(Ptr);
7999 Sizes.push_back(llvm::Constant::getNullValue(CGF.
Int64Ty));
8000 Types.push_back(OMP_MAP_RETURN_PARAM | OMP_MAP_TARGET_PARAM);
8005 for (
const auto &M : Info) {
8008 bool IsFirstComponentList =
true;
8011 MapBaseValuesArrayTy CurBasePointers;
8012 MapValuesArrayTy CurPointers;
8013 MapValuesArrayTy CurSizes;
8014 MapFlagsArrayTy CurTypes;
8015 StructRangeInfoTy PartialStruct;
8017 for (
const MapInfo &L : M.second) {
8018 assert(!L.Components.empty() &&
8019 "Not expecting declaration with no component lists.");
8022 unsigned CurrentBasePointersIdx = CurBasePointers.size();
8023 generateInfoForComponentList(L.MapType, L.MapModifiers, L.Components,
8024 CurBasePointers, CurPointers, CurSizes,
8025 CurTypes, PartialStruct,
8026 IsFirstComponentList, L.IsImplicit);
8030 if (L.ReturnDevicePointer) {
8031 assert(CurBasePointers.size() > CurrentBasePointersIdx &&
8032 "Unexpected number of mapped base pointers.");
8034 const ValueDecl *RelevantVD =
8035 L.Components.back().getAssociatedDeclaration();
8036 assert(RelevantVD &&
8037 "No relevant declaration related with device pointer??");
8039 CurBasePointers[CurrentBasePointersIdx].setDevicePtrDecl(RelevantVD);
8040 CurTypes[CurrentBasePointersIdx] |= OMP_MAP_RETURN_PARAM;
8042 IsFirstComponentList =
false;
8047 auto CI = DeferredInfo.find(M.first);
8048 if (CI != DeferredInfo.end()) {
8049 for (
const DeferredDevicePtrEntryTy &L : CI->second) {
8052 this->CGF.
EmitLValue(L.IE), L.IE->getExprLoc());
8053 CurBasePointers.emplace_back(BasePtr, L.VD);
8054 CurPointers.push_back(Ptr);
8055 CurSizes.push_back(llvm::Constant::getNullValue(this->CGF.
Int64Ty));
8059 CurTypes.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_RETURN_PARAM |
8066 if (PartialStruct.Base.isValid())
8067 emitCombinedEntry(BasePointers, Pointers, Sizes, Types, CurTypes,
8071 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
8072 Pointers.append(CurPointers.begin(), CurPointers.end());
8073 Sizes.append(CurSizes.begin(), CurSizes.end());
8074 Types.append(CurTypes.begin(), CurTypes.end());
8080 void generateAllInfoForMapper(MapBaseValuesArrayTy &BasePointers,
8081 MapValuesArrayTy &Pointers,
8082 MapValuesArrayTy &Sizes,
8083 MapFlagsArrayTy &Types)
const {
8085 "Expect a declare mapper directive");
8090 llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info;
8094 auto &&InfoGen = [&Info](
8098 ArrayRef<OpenMPMapModifierKind> MapModifiers,
8099 bool ReturnDevicePointer,
bool IsImplicit) {
8100 const ValueDecl *VD =
8102 Info[VD].emplace_back(L, MapType, MapModifiers, ReturnDevicePointer,
8106 for (
const auto *C : CurMapperDir->clauselists()) {
8107 const auto *MC = cast<OMPMapClause>(C);
8108 for (
const auto L : MC->component_lists()) {
8109 InfoGen(L.first, L.second, MC->getMapType(), MC->getMapTypeModifiers(),
8110 false, MC->isImplicit());
8114 for (
const auto &M : Info) {
8117 bool IsFirstComponentList =
true;
8120 MapBaseValuesArrayTy CurBasePointers;
8121 MapValuesArrayTy CurPointers;
8122 MapValuesArrayTy CurSizes;
8123 MapFlagsArrayTy CurTypes;
8124 StructRangeInfoTy PartialStruct;
8126 for (
const MapInfo &L : M.second) {
8127 assert(!L.Components.empty() &&
8128 "Not expecting declaration with no component lists.");
8129 generateInfoForComponentList(L.MapType, L.MapModifiers, L.Components,
8130 CurBasePointers, CurPointers, CurSizes,
8131 CurTypes, PartialStruct,
8132 IsFirstComponentList, L.IsImplicit);
8133 IsFirstComponentList =
false;
8138 if (PartialStruct.Base.isValid())
8139 emitCombinedEntry(BasePointers, Pointers, Sizes, Types, CurTypes,
8143 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
8144 Pointers.append(CurPointers.begin(), CurPointers.end());
8145 Sizes.append(CurSizes.begin(), CurSizes.end());
8146 Types.append(CurTypes.begin(), CurTypes.end());
8151 void generateInfoForLambdaCaptures(
8152 const ValueDecl *VD,
llvm::Value *Arg, MapBaseValuesArrayTy &BasePointers,
8153 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
8154 MapFlagsArrayTy &Types,
8155 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers)
const {
8156 const auto *RD = VD->
getType()
8165 llvm::DenseMap<const VarDecl *, FieldDecl *> Captures;
8172 LambdaPointers.try_emplace(ThisLVal.
getPointer(CGF),
8173 VDLVal.getPointer(CGF));
8174 BasePointers.push_back(ThisLVal.
getPointer(CGF));
8175 Pointers.push_back(ThisLValVal.
getPointer(CGF));
8179 Types.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
8180 OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT);
8183 if (!LC.capturesVariable())
8185 const VarDecl *VD = LC.getCapturedVar();
8188 auto It = Captures.find(VD);
8189 assert(It != Captures.end() &&
"Found lambda capture without field.");
8193 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
8194 VDLVal.getPointer(CGF));
8195 BasePointers.push_back(VarLVal.getPointer(CGF));
8196 Pointers.push_back(VarLValVal.
getPointer(CGF));
8197 Sizes.push_back(CGF.
Builder.CreateIntCast(
8203 LambdaPointers.try_emplace(VarLVal.getPointer(CGF),
8204 VDLVal.getPointer(CGF));
8205 BasePointers.push_back(VarLVal.getPointer(CGF));
8207 Sizes.push_back(llvm::ConstantInt::get(CGF.
Int64Ty, 0));
8209 Types.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
8210 OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT);
8215 void adjustMemberOfForLambdaCaptures(
8216 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
8217 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
8218 MapFlagsArrayTy &Types)
const {
8219 for (
unsigned I = 0, E = Types.size(); I < E; ++I) {
8221 if (Types[I] != (OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL |
8222 OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT))
8224 llvm::Value *BasePtr = LambdaPointers.lookup(*BasePointers[I]);
8225 assert(BasePtr &&
"Unable to find base lambda address.");
8227 for (
unsigned J = I; J > 0; --J) {
8228 unsigned Idx = J - 1;
8229 if (Pointers[Idx] != BasePtr)
8234 assert(TgtIdx != -1 &&
"Unable to find parent lambda.");
8238 OpenMPOffloadMappingFlags MemberOfFlag = getMemberOfFlag(TgtIdx);
8239 setCorrectMemberOfFlag(Types[I], MemberOfFlag);
8247 MapBaseValuesArrayTy &BasePointers,
8248 MapValuesArrayTy &Pointers,
8249 MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
8250 StructRangeInfoTy &PartialStruct)
const {
8252 "Not expecting to generate map info for a variable array type!");
8262 if (DevPointersMap.count(VD)) {
8263 BasePointers.emplace_back(Arg, VD);
8264 Pointers.push_back(Arg);
8268 Types.push_back(OMP_MAP_LITERAL | OMP_MAP_TARGET_PARAM);
8276 assert(CurDir.is<
const OMPExecutableDirective *>() &&
8277 "Expect a executable directive");
8278 const auto *CurExecDir = CurDir.get<
const OMPExecutableDirective *>();
8279 for (
const auto *C : CurExecDir->getClausesOfKind<
OMPMapClause>()) {
8280 for (
const auto L : C->decl_component_lists(VD)) {
8281 assert(L.first == VD &&
8282 "We got information for the wrong declaration??");
8283 assert(!L.second.empty() &&
8284 "Not expecting declaration with no component lists.");
8285 DeclComponentLists.emplace_back(L.second, C->getMapType(),
8286 C->getMapTypeModifiers(),
8292 llvm::SmallDenseMap<
8299 for (
const MapData &L : DeclComponentLists) {
8300 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
8301 OpenMPMapClauseKind MapType;
8302 ArrayRef<OpenMPMapModifierKind> MapModifiers;
8304 std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
8306 for (
const MapData &L1 : makeArrayRef(DeclComponentLists).slice(Count)) {
8307 OMPClauseMappableExprCommon::MappableExprComponentListRef Components1;
8308 std::tie(Components1, MapType, MapModifiers, IsImplicit) = L1;
8309 auto CI = Components.rbegin();
8310 auto CE = Components.rend();
8311 auto SI = Components1.rbegin();
8312 auto SE = Components1.rend();
8313 for (; CI != CE && SI != SE; ++CI, ++SI) {
8314 if (CI->getAssociatedExpression()->getStmtClass() !=
8315 SI->getAssociatedExpression()->getStmtClass())
8318 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
8323 if (CI == CE || SI == SE) {
8324 assert((CI != CE || SI != SE) &&
8325 "Unexpected full match of the mapping components.");
8326 const MapData &BaseData = CI == CE ? L : L1;
8327 OMPClauseMappableExprCommon::MappableExprComponentListRef SubData =
8328 SI == SE ? Components : Components1;
8329 auto &OverlappedElements = OverlappedData.FindAndConstruct(&BaseData);
8330 OverlappedElements.getSecond().push_back(SubData);
8336 if (!OverlappedData.empty()) {
8337 if (
const auto *CRD =
8338 VD->getType().getCanonicalType()->getAsCXXRecordDecl())
8339 getPlainLayout(CRD, Layout,
false);
8341 const auto *RD = VD->getType().getCanonicalType()->getAsRecordDecl();
8345 for (
auto &Pair : OverlappedData) {
8349 OMPClauseMappableExprCommon::MappableExprComponentListRef
First,
8350 OMPClauseMappableExprCommon::MappableExprComponentListRef
8352 auto CI = First.rbegin();
8353 auto CE = First.rend();
8354 auto SI = Second.rbegin();
8355 auto SE = Second.rend();
8356 for (; CI != CE && SI != SE; ++CI, ++SI) {
8357 if (CI->getAssociatedExpression()->getStmtClass() !=
8358 SI->getAssociatedExpression()->getStmtClass())
8361 if (CI->getAssociatedDeclaration() !=
8362 SI->getAssociatedDeclaration())
8367 if (CI == CE && SI == SE)
8371 if (CI == CE || SI == SE)
8374 const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration());
8375 const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration());
8376 if (FD1->getParent() == FD2->getParent())
8377 return FD1->getFieldIndex() < FD2->getFieldIndex();
8379 llvm::find_if(Layout, [FD1, FD2](
const FieldDecl *FD) {
8380 return FD == FD1 || FD == FD2;
8388 for (
const auto &Pair : OverlappedData) {
8389 const MapData &L = *Pair.getFirst();
8390 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
8391 OpenMPMapClauseKind MapType;
8392 ArrayRef<OpenMPMapModifierKind> MapModifiers;
8394 std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
8395 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
8396 OverlappedComponents = Pair.getSecond();
8397 bool IsFirstComponentList =
true;
8398 generateInfoForComponentList(MapType, MapModifiers, Components,
8399 BasePointers, Pointers, Sizes, Types,
8400 PartialStruct, IsFirstComponentList,
8401 IsImplicit, OverlappedComponents);
8404 bool IsFirstComponentList = OverlappedData.empty();
8405 for (
const MapData &L : DeclComponentLists) {
8406 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
8407 OpenMPMapClauseKind MapType;
8408 ArrayRef<OpenMPMapModifierKind> MapModifiers;
8410 std::tie(Components, MapType, MapModifiers, IsImplicit) = L;
8411 auto It = OverlappedData.find(&L);
8412 if (It == OverlappedData.end())
8413 generateInfoForComponentList(MapType, MapModifiers, Components,
8414 BasePointers, Pointers, Sizes, Types,
8415 PartialStruct, IsFirstComponentList,
8417 IsFirstComponentList =
false;
8423 void generateInfoForDeclareTargetLink(MapBaseValuesArrayTy &BasePointers,
8424 MapValuesArrayTy &Pointers,
8425 MapValuesArrayTy &Sizes,
8426 MapFlagsArrayTy &Types)
const {
8427 assert(CurDir.is<
const OMPExecutableDirective *>() &&
8428 "Expect a executable directive");
8429 const auto *CurExecDir = CurDir.get<
const OMPExecutableDirective *>();
8432 for (
const auto *C : CurExecDir->getClausesOfKind<
OMPMapClause>()) {
8433 for (
const auto L : C->component_lists()) {
8436 const auto *VD = dyn_cast<
VarDecl>(L.first);
8440 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
8442 !Res || *Res != OMPDeclareTargetDeclAttr::MT_Link)
8444 StructRangeInfoTy PartialStruct;
8445 generateInfoForComponentList(
8446 C->getMapType(), C->getMapTypeModifiers(), L.second, BasePointers,
8447 Pointers, Sizes, Types, PartialStruct,
8448 true, C->isImplicit());
8449 assert(!PartialStruct.Base.isValid() &&
8450 "No partial structs for declare target link expected.");
8459 MapBaseValuesArrayTy &CurBasePointers,
8460 MapValuesArrayTy &CurPointers,
8461 MapValuesArrayTy &CurSizes,
8462 MapFlagsArrayTy &CurMapTypes)
const {
8463 bool IsImplicit =
true;
8466 CurBasePointers.push_back(CV);
8467 CurPointers.push_back(CV);
8473 CurMapTypes.push_back(OMP_MAP_TO | OMP_MAP_FROM);
8475 CurBasePointers.push_back(CV);
8476 CurPointers.push_back(CV);
8480 CurMapTypes.push_back(OMP_MAP_LITERAL);
8481 CurSizes.push_back(CGF.
Builder.CreateIntCast(
8486 CurMapTypes.push_back(OMP_MAP_NONE);
8487 CurSizes.push_back(llvm::Constant::getNullValue(CGF.
Int64Ty));
8490 auto I = FirstPrivateDecls.find(VD);
8491 if (I != FirstPrivateDecls.end())
8492 IsImplicit = I->getSecond();
8497 CurSizes.push_back(CGF.
Builder.CreateIntCast(
8502 CurMapTypes.push_back(getMapModifiersForPrivateClauses(CI));
8504 auto I = FirstPrivateDecls.find(VD);
8505 if (I != FirstPrivateDecls.end() &&
8507 llvm::Constant *Addr =
8513 CurSizes.back(),
false);
8515 CurBasePointers.push_back(Addr);
8516 CurPointers.push_back(Addr);
8518 CurBasePointers.push_back(CV);
8519 if (I != FirstPrivateDecls.end() && ElementType->isAnyPointerType()) {
8525 CurPointers.push_back(CV);
8528 if (I != FirstPrivateDecls.end())
8529 IsImplicit = I->getSecond();
8532 CurMapTypes.back() |= OMP_MAP_TARGET_PARAM;
8536 CurMapTypes.back() |= OMP_MAP_IMPLICIT;
8550 CGOpenMPRuntime::TargetDataInfo &Info) {
8555 Info.clearArrayInfo();
8556 Info.NumberOfPtrs = BasePointers.size();
8558 if (Info.NumberOfPtrs) {
8561 bool hasRuntimeEvaluationCaptureSize =
false;
8563 if (!isa<llvm::Constant>(S)) {
8564 hasRuntimeEvaluationCaptureSize =
true;
8568 llvm::APInt PointerNumAP(32, Info.NumberOfPtrs,
true);
8573 Info.BasePointersArray =
8575 Info.PointersArray =
8583 if (hasRuntimeEvaluationCaptureSize) {
8594 ConstSizes.push_back(cast<llvm::Constant>(S));
8596 auto *SizesArrayInit = llvm::ConstantArray::get(
8597 llvm::ArrayType::get(CGM.
Int64Ty, ConstSizes.size()), ConstSizes);
8599 auto *SizesArrayGbl =
new llvm::GlobalVariable(
8600 CGM.
getModule(), SizesArrayInit->getType(),
8601 true, llvm::GlobalValue::PrivateLinkage,
8602 SizesArrayInit, Name);
8603 SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
8604 Info.SizesArray = SizesArrayGbl;
8610 llvm::copy(MapTypes, Mapping.begin());
8611 llvm::Constant *MapTypesArrayInit =
8612 llvm::ConstantDataArray::get(CGF.
Builder.getContext(), Mapping);
8613 std::string MaptypesName =
8615 auto *MapTypesArrayGbl =
new llvm::GlobalVariable(
8616 CGM.
getModule(), MapTypesArrayInit->getType(),
8617 true, llvm::GlobalValue::PrivateLinkage,
8618 MapTypesArrayInit, MaptypesName);
8619 MapTypesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
8620 Info.MapTypesArray = MapTypesArrayGbl;
8622 for (
unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
8625 llvm::ArrayType::get(CGM.
VoidPtrTy, Info.NumberOfPtrs),
8626 Info.BasePointersArray, 0, I);
8628 BP, BPVal->getType()->getPointerTo(0));
8632 if (Info.requiresDevicePointerInfo())
8633 if (
const ValueDecl *DevVD = BasePointers[I].getDevicePtrDecl())
8634 Info.CaptureDeviceAddrMap.try_emplace(DevVD, BPAddr);
8638 llvm::ArrayType::get(CGM.
VoidPtrTy, Info.NumberOfPtrs),
8639 Info.PointersArray, 0, I);
8641 P, PVal->getType()->getPointerTo(0));
8645 if (hasRuntimeEvaluationCaptureSize) {
8647 llvm::ArrayType::get(CGM.
Int64Ty, Info.NumberOfPtrs),
8665 llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info) {
8667 if (Info.NumberOfPtrs) {
8669 llvm::ArrayType::get(CGM.
VoidPtrTy, Info.NumberOfPtrs),
8670 Info.BasePointersArray,
8673 llvm::ArrayType::get(CGM.
VoidPtrTy, Info.NumberOfPtrs),
8678 llvm::ArrayType::get(CGM.
Int64Ty, Info.NumberOfPtrs), Info.SizesArray,
8681 llvm::ArrayType::get(CGM.
Int64Ty, Info.NumberOfPtrs),
8686 BasePointersArrayArg = llvm::ConstantPointerNull::get(CGM.
VoidPtrPtrTy);
8687 PointersArrayArg = llvm::ConstantPointerNull::get(CGM.
VoidPtrPtrTy);
8688 SizesArrayArg = llvm::ConstantPointerNull::get(CGM.
Int64Ty->getPointerTo());
8690 llvm::ConstantPointerNull::get(CGM.
Int64Ty->getPointerTo());
8700 const Stmt *ChildStmt =
8703 if (
const auto *NestedDir =
8704 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8710 if (DKind == OMPD_teams) {
8711 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
8716 if (
const auto *NND =
8717 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
8718 DKind = NND->getDirectiveKind();
8724 case OMPD_target_teams:
8728 case OMPD_target_parallel:
8729 case OMPD_target_simd:
8730 case OMPD_target_parallel_for:
8731 case OMPD_target_parallel_for_simd:
8733 case OMPD_target_teams_distribute:
8734 case OMPD_target_teams_distribute_simd:
8735 case OMPD_target_teams_distribute_parallel_for:
8736 case OMPD_target_teams_distribute_parallel_for_simd:
8739 case OMPD_parallel_for:
8740 case OMPD_parallel_master:
8741 case OMPD_parallel_sections:
8743 case OMPD_parallel_for_simd:
8745 case OMPD_cancellation_point:
8747 case OMPD_threadprivate:
8756 case OMPD_taskyield:
8759 case OMPD_taskgroup:
8763 case OMPD_target_data:
8764 case OMPD_target_exit_data:
8765 case OMPD_target_enter_data:
8766 case OMPD_distribute:
8767 case OMPD_distribute_simd:
8768 case OMPD_distribute_parallel_for:
8769 case OMPD_distribute_parallel_for_simd:
8770 case OMPD_teams_distribute:
8771 case OMPD_teams_distribute_simd:
8772 case OMPD_teams_distribute_parallel_for:
8773 case OMPD_teams_distribute_parallel_for_simd:
8774 case OMPD_target_update:
8775 case OMPD_declare_simd:
8776 case OMPD_declare_variant:
8777 case OMPD_declare_target:
8778 case OMPD_end_declare_target:
8779 case OMPD_declare_reduction:
8780 case OMPD_declare_mapper:
8782 case OMPD_taskloop_simd:
8783 case OMPD_master_taskloop:
8784 case OMPD_master_taskloop_simd:
8785 case OMPD_parallel_master_taskloop:
8786 case OMPD_parallel_master_taskloop_simd:
8789 llvm_unreachable(
"Unexpected directive.");
8826 if (UDMMap.count(D) > 0)
8832 auto *MapperVarDecl =
8849 Args.push_back(&HandleArg);
8850 Args.push_back(&BaseArg);
8851 Args.push_back(&BeginArg);
8852 Args.push_back(&SizeArg);
8853 Args.push_back(&TypeArg);
8858 llvm::raw_svector_ostream Out(TyStr);
8864 Fn->removeFnAttr(llvm::Attribute::OptimizeNone);
8892 llvm::BasicBlock *HeadBB = MapperCGF.
createBasicBlock(
"omp.arraymap.head");
8894 ElementSize, HeadBB,
true);
8900 llvm::BasicBlock *BodyBB = MapperCGF.
createBasicBlock(
"omp.arraymap.body");
8904 MapperCGF.
Builder.CreateICmpEQ(PtrBegin, PtrEnd,
"omp.arraymap.isempty");
8905 MapperCGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
8906 llvm::BasicBlock *EntryBB = MapperCGF.
Builder.GetInsertBlock();
8910 llvm::PHINode *PtrPHI = MapperCGF.
Builder.CreatePHI(
8911 PtrBegin->getType(), 2,
"omp.arraymap.ptrcurrent");
8912 PtrPHI->addIncoming(PtrBegin, EntryBB);
8919 Scope.addPrivate(MapperVarDecl, [&MapperCGF, PtrCurrent, PtrTy]() {
8924 (void)Scope.Privatize();
8931 MappableExprsHandler MEHandler(*D, MapperCGF);
8932 MEHandler.generateAllInfoForMapper(BasePointers, Pointers, Sizes, MapTypes);
8941 MapperCGF.
Builder.getInt64(MappableExprsHandler::getFlagMemberOffset()));
8944 for (
unsigned I = 0; I < BasePointers.size(); ++I) {
8957 MapperCGF.
Builder.getInt64(MappableExprsHandler::OMP_MAP_MEMBER_OF));
8958 llvm::BasicBlock *MemberCombineBB =
8962 MapperCGF.
Builder.CreateCondBr(IsMember, TypeBB, MemberCombineBB);
8967 MapperCGF.
Builder.CreateNUWAdd(OriMapType, ShiftedPreviousSize);
8970 llvm::PHINode *MemberMapType =
8972 MemberMapType->addIncoming(OriMapType, MemberBB);
8973 MemberMapType->addIncoming(CombinedMember, MemberCombineBB);
8989 MapperCGF.
Builder.getInt64(MappableExprsHandler::OMP_MAP_TO |
8990 MappableExprsHandler::OMP_MAP_FROM));
8992 llvm::BasicBlock *AllocElseBB =
8995 llvm::BasicBlock *ToElseBB = MapperCGF.
createBasicBlock(
"omp.type.to.else");
8999 MapperCGF.
Builder.CreateCondBr(IsAlloc, AllocBB, AllocElseBB);
9004 MapperCGF.
Builder.getInt64(~(MappableExprsHandler::OMP_MAP_TO |
9005 MappableExprsHandler::OMP_MAP_FROM)));
9006 MapperCGF.
Builder.CreateBr(EndBB);
9010 MapperCGF.
Builder.getInt64(MappableExprsHandler::OMP_MAP_TO));
9011 MapperCGF.
Builder.CreateCondBr(IsTo, ToBB, ToElseBB);
9016 MapperCGF.
Builder.getInt64(~MappableExprsHandler::OMP_MAP_FROM));
9017 MapperCGF.
Builder.CreateBr(EndBB);
9021 MapperCGF.
Builder.getInt64(MappableExprsHandler::OMP_MAP_FROM));
9022 MapperCGF.
Builder.CreateCondBr(IsFrom, FromBB, EndBB);
9027 MapperCGF.
Builder.getInt64(~MappableExprsHandler::OMP_MAP_TO));
9030 llvm::PHINode *CurMapType =
9032 CurMapType->addIncoming(AllocMapType, AllocBB);
9033 CurMapType->addIncoming(ToMapType, ToBB);
9034 CurMapType->addIncoming(FromMapType, FromBB);
9035 CurMapType->addIncoming(MemberMapType, ToElseBB);
9041 llvm::Value *OffloadingArgs[] = {Handle, CurBaseArg, CurBeginArg,
9042 CurSizeArg, CurMapType};
9051 PtrPHI, 1,
"omp.arraymap.next");
9052 PtrPHI->addIncoming(PtrNext, BodyBB);
9054 MapperCGF.
Builder.CreateICmpEQ(PtrNext, PtrEnd,
"omp.arraymap.isdone");
9055 llvm::BasicBlock *ExitBB = MapperCGF.
createBasicBlock(
"omp.arraymap.exit");
9056 MapperCGF.
Builder.CreateCondBr(IsDone, ExitBB, BodyBB);
9062 ElementSize, DoneBB,
false);
9067 UDMMap.try_emplace(D, Fn);
9069 auto &Decls = FunctionUDMMap.FindAndConstruct(CGF->
CurFn);
9070 Decls.second.push_back(D);
9083 CharUnits ElementSize, llvm::BasicBlock *ExitBB,
bool IsInit) {
9084 StringRef Prefix = IsInit ?
".init" :
".del";
9087 llvm::BasicBlock *IsDeleteBB =
9089 llvm::BasicBlock *BodyBB = MapperCGF.
createBasicBlock(
"omp.array" + Prefix);
9091 Size, MapperCGF.
Builder.getInt64(1),
"omp.arrayinit.isarray");
9092 MapperCGF.
Builder.CreateCondBr(IsArray, IsDeleteBB, ExitBB);
9098 MapperCGF.
Builder.getInt64(MappableExprsHandler::OMP_MAP_DELETE));
9101 DeleteCond = MapperCGF.
Builder.CreateIsNull(
9102 DeleteBit,
"omp.array" + Prefix +
".delete");
9104 DeleteCond = MapperCGF.
Builder.CreateIsNotNull(
9105 DeleteBit,
"omp.array" + Prefix +
".delete");
9107 MapperCGF.
Builder.CreateCondBr(DeleteCond, BodyBB, ExitBB);
9118 MapperCGF.
Builder.getInt64(~(MappableExprsHandler::OMP_MAP_TO |
9119 MappableExprsHandler::OMP_MAP_FROM)));
9122 llvm::Value *OffloadingArgs[] = {Handle, Base,
Begin, ArraySize, MapTypeArg};
9140 const auto *LD = cast<OMPLoopDirective>(TD);
9141 auto &&CodeGen = [LD, DeviceID, SizeEmitter,
this](
CodeGenFunction &CGF,
9143 if (
llvm::Value *NumIterations = SizeEmitter(CGF, *LD)) {
9145 CGF.EmitRuntimeCall(
9154 llvm::Function *OutlinedFn,
llvm::Value *OutlinedFnID,
const Expr *IfCond,
9162 assert(OutlinedFn &&
"Invalid outlined function!");
9169 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9176 auto &&ThenGen = [
this, Device, OutlinedFn, OutlinedFnID, &D, &InputInfo,
9177 &MapTypesArray, &CS, RequiresOuterTask, &CapturedVars,
9189 assert(OutlinedFnID &&
"Invalid outlined function ID!");
9194 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
9197 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
9202 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
9247 assert(NumThreads &&
"Thread limit expression should be available along " 9248 "with number of teams.");
9252 InputInfo.BasePointersArray.getPointer(),
9253 InputInfo.PointersArray.getPointer(),
9254 InputInfo.SizesArray.getPointer(),
9258 Return = CGF.EmitRuntimeCall(
9266 InputInfo.BasePointersArray.getPointer(),
9267 InputInfo.PointersArray.getPointer(),
9268 InputInfo.SizesArray.getPointer(),
9270 Return = CGF.EmitRuntimeCall(
9277 llvm::BasicBlock *OffloadFailedBlock =
9278 CGF.createBasicBlock(
"omp_offload.failed");
9279 llvm::BasicBlock *OffloadContBlock =
9280 CGF.createBasicBlock(
"omp_offload.cont");
9281 llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return);
9282 CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
9284 CGF.EmitBlock(OffloadFailedBlock);
9285 if (RequiresOuterTask) {
9286 CapturedVars.clear();
9287 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9290 CGF.EmitBranch(OffloadContBlock);
9292 CGF.EmitBlock(OffloadContBlock,
true);
9296 auto &&ElseGen = [
this, &D, OutlinedFn, &CS, &CapturedVars,
9299 if (RequiresOuterTask) {
9300 CapturedVars.clear();
9301 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
9306 auto &&TargetThenGen = [
this, &ThenGen, &D, &InputInfo, &MapTypesArray,
9307 &CapturedVars, RequiresOuterTask,
9316 MappableExprsHandler MEHandler(D, CGF);
9317 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
9319 auto RI = CS.getCapturedRecordDecl()->field_begin();
9320 auto CV = CapturedVars.begin();
9322 CE = CS.capture_end();
9323 CI != CE; ++CI, ++RI, ++CV) {
9328 MappableExprsHandler::StructRangeInfoTy PartialStruct;
9332 if (CI->capturesVariableArrayType()) {
9333 CurBasePointers.push_back(*CV);
9334 CurPointers.push_back(*CV);
9335 CurSizes.push_back(CGF.Builder.CreateIntCast(
9336 CGF.getTypeSize(RI->getType()), CGF.Int64Ty,
true));
9338 CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_LITERAL |
9339 MappableExprsHandler::OMP_MAP_TARGET_PARAM |
9340 MappableExprsHandler::OMP_MAP_IMPLICIT);
9344 MEHandler.generateInfoForCapture(CI, *CV, CurBasePointers, CurPointers,
9345 CurSizes, CurMapTypes, PartialStruct);
9346 if (CurBasePointers.empty())
9347 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurBasePointers,
9348 CurPointers, CurSizes, CurMapTypes);
9351 if (CI->capturesVariable())
9352 MEHandler.generateInfoForLambdaCaptures(
9353 CI->getCapturedVar(), *CV, CurBasePointers, CurPointers, CurSizes,
9354 CurMapTypes, LambdaPointers);
9357 assert(!CurBasePointers.empty() &&
9358 "Non-existing map pointer for capture!");
9359 assert(CurBasePointers.size() == CurPointers.size() &&
9360 CurBasePointers.size() == CurSizes.size() &&
9361 CurBasePointers.size() == CurMapTypes.size() &&
9362 "Inconsistent map information sizes!");
9366 if (PartialStruct.Base.isValid())
9367 MEHandler.emitCombinedEntry(BasePointers, Pointers, Sizes, MapTypes,
9368 CurMapTypes, PartialStruct);
9371 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
9372 Pointers.append(CurPointers.begin(), CurPointers.end());
9373 Sizes.append(CurSizes.begin(), CurSizes.end());
9374 MapTypes.append(CurMapTypes.begin(), CurMapTypes.end());
9377 MEHandler.adjustMemberOfForLambdaCaptures(LambdaPointers, BasePointers,
9378 Pointers, MapTypes);
9381 MEHandler.generateInfoForDeclareTargetLink(BasePointers, Pointers, Sizes,
9391 InputInfo.BasePointersArray =
9393 InputInfo.PointersArray =
9397 if (RequiresOuterTask)
9398 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
9403 auto &&TargetElseGen = [
this, &ElseGen, &D, RequiresOuterTask](
9405 if (RequiresOuterTask) {
9407 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo);
9419 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen);
9431 StringRef ParentName) {
9436 bool RequiresDeviceCodegen =
9437 isa<OMPExecutableDirective>(S) &&
9439 cast<OMPExecutableDirective>(S)->getDirectiveKind());
9441 if (RequiresDeviceCodegen) {
9442 const auto &E = *cast<OMPExecutableDirective>(S);
9455 switch (E.getDirectiveKind()) {
9458 cast<OMPTargetDirective>(E));
9460 case OMPD_target_parallel:
9462 CGM, ParentName, cast<OMPTargetParallelDirective>(E));
9464 case OMPD_target_teams:
9466 CGM, ParentName, cast<OMPTargetTeamsDirective>(E));
9468 case OMPD_target_teams_distribute:
9470 CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E));
9472 case OMPD_target_teams_distribute_simd:
9474 CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E));
9476 case OMPD_target_parallel_for:
9478 CGM, ParentName, cast<OMPTargetParallelForDirective>(E));
9480 case OMPD_target_parallel_for_simd:
9482 CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E));
9484 case OMPD_target_simd:
9486 CGM, ParentName, cast<OMPTargetSimdDirective>(E));
9488 case OMPD_target_teams_distribute_parallel_for:
9491 cast<OMPTargetTeamsDistributeParallelForDirective>(E));
9493 case OMPD_target_teams_distribute_parallel_for_simd:
9497 cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E));
9501 case OMPD_parallel_for:
9502 case OMPD_parallel_master:
9503 case OMPD_parallel_sections:
9505 case OMPD_parallel_for_simd:
9507 case OMPD_cancellation_point:
9509 case OMPD_threadprivate:
9518 case OMPD_taskyield:
9521 case OMPD_taskgroup:
9525 case OMPD_target_data:
9526 case OMPD_target_exit_data:
9527 case OMPD_target_enter_data:
9528 case OMPD_distribute:
9529 case OMPD_distribute_simd:
9530 case OMPD_distribute_parallel_for:
9531 case OMPD_distribute_parallel_for_simd:
9532 case OMPD_teams_distribute:
9533 case OMPD_teams_distribute_simd:
9534 case OMPD_teams_distribute_parallel_for:
9535 case OMPD_teams_distribute_parallel_for_simd:
9536 case OMPD_target_update:
9537 case OMPD_declare_simd:
9538 case OMPD_declare_variant:
9539 case OMPD_declare_target:
9540 case OMPD_end_declare_target:
9541 case OMPD_declare_reduction:
9542 case OMPD_declare_mapper:
9544 case OMPD_taskloop_simd:
9545 case OMPD_master_taskloop:
9546 case OMPD_master_taskloop_simd:
9547 case OMPD_parallel_master_taskloop:
9548 case OMPD_parallel_master_taskloop_simd:
9551 llvm_unreachable(
"Unknown target directive for OpenMP device codegen.");
9556 if (
const auto *E = dyn_cast<OMPExecutableDirective>(S)) {
9557 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
9561 E->getInnermostCapturedStmt()->getCapturedStmt(), ParentName);
9566 if (
const auto *L = dyn_cast<LambdaExpr>(S))
9578 if (
const auto *FD = dyn_cast<FunctionDecl>(GD.
getDecl())) {
9580 OMPDeclareTargetDeclAttr::getDeviceType(FD);
9582 if (DevTy && *DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
9590 if (
const auto *FD = dyn_cast<FunctionDecl>(VD)) {
9594 OMPDeclareTargetDeclAttr::getDeviceType(FD);
9596 if (DevTy && *DevTy == OMPDeclareTargetDeclAttr::DT_Host)
9601 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
9615 StringRef ParentName =
9620 StringRef ParentName =
9628 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
9630 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
9631 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9643 "Expected constant variable.");
9645 llvm::Constant *Addr;
9646 llvm::GlobalValue::LinkageTypes
Linkage;
9655 llvm::raw_svector_ostream OS(Buffer);
9656 OS <<
"__omp_offloading_firstprivate_" << llvm::format(
"_%x", DeviceID)
9657 << llvm::format(
"_%x_", FileID) << VD->
getName() <<
"_l" <<
Line;
9664 cast<llvm::GlobalValue>(Addr)->setLinkage(Linkage);
9668 VarName, Addr, VarSize,
9674 llvm::Constant *Addr) {
9679 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9693 llvm::GlobalValue::LinkageTypes
Linkage;
9695 if (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9701 assert(!VarSize.
isZero() &&
"Expected non-zero size of the variable");
9708 std::string RefName =
getName({VarName,
"ref"});
9710 llvm::Constant *AddrRef =
9712 auto *GVAddrRef = cast<llvm::GlobalVariable>(AddrRef);
9713 GVAddrRef->setConstant(
true);
9715 GVAddrRef->setInitializer(Addr);
9720 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
9721 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9723 "Declare target attribute must link or to with unified memory.");
9724 if (*Res == OMPDeclareTargetDeclAttr::MT_Link)
9730 VarName = Addr->getName();
9737 Linkage = llvm::GlobalValue::WeakAnyLinkage;
9741 VarName, Addr, VarSize, Flags, Linkage);
9745 if (isa<FunctionDecl>(GD.
getDecl()) ||
9746 isa<OMPDeclareReductionDecl>(GD.
getDecl()))
9755 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
9758 if (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9762 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
9763 (*Res == OMPDeclareTargetDeclAttr::MT_To &&
9765 "Expected link clause or to clause with unified memory.");
9774 " Expected target-based directive.");
9780 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
9789 if (!VD || !VD->
hasAttr<OMPAllocateDeclAttr>())
9791 const auto *A = VD->
getAttr<OMPAllocateDeclAttr>();
9792 switch(A->getAllocatorType()) {
9793 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
9795 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
9796 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
9797 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
9798 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
9799 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
9800 case OMPAllocateDeclAttr::OMPConstMemAlloc:
9801 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
9804 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
9805 llvm_unreachable(
"Expected predefined allocator for the variables with the " 9833 const auto *D = cast<FunctionDecl>(GD.
getDecl());
9836 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) {
9838 if (
auto *F = dyn_cast_or_null<llvm::Function>(
9840 return !F->isDeclaration();
9862 llvm::Function *RequiresRegFn;
9867 std::string ReqName =
getName({
"omp_offloading",
"requires_reg"});
9880 "Target or declare target region expected.");
9882 Flags = OMP_REQ_UNIFIED_SHARED_MEMORY;
9884 llvm::ConstantInt::get(CGM.
Int64Ty, Flags));
9887 return RequiresRegFn;
9893 llvm::Function *OutlinedFn,
9904 CGF.
Builder.getInt32(CapturedVars.size()),
9907 RealArgs.append(std::begin(Args), std::end(Args));
9908 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
9915 const Expr *NumTeams,
9916 const Expr *ThreadLimit,
9955 auto &&BeginThenGen = [
this, &D, Device, &Info,
9964 MappableExprsHandler MCHandler(D, CGF);
9965 MCHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
9975 SizesArrayArg, MapTypesArrayArg, Info);
9980 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
9983 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
9987 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
9990 DeviceID, PointerNum, BasePointersArrayArg,
9991 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
9997 if (!Info.CaptureDeviceAddrMap.empty())
10004 assert(Info.isValid() &&
"Invalid data environment closing arguments.");
10011 SizesArrayArg, MapTypesArrayArg, Info);
10016 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
10017 CGF.Int64Ty,
true);
10019 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
10023 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs);
10026 DeviceID, PointerNum, BasePointersArrayArg,
10027 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
10035 auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](
CodeGenFunction &CGF,
10037 if (!Info.CaptureDeviceAddrMap.empty()) {
10038 CodeGen.setAction(NoPrivAction);
10048 emitIfClause(CGF, IfCond, BeginThenGen, BeginElseGen);
10056 if (Info.CaptureDeviceAddrMap.empty()) {
10057 CodeGen.setAction(NoPrivAction);
10071 const Expr *Device) {
10075 assert((isa<OMPTargetEnterDataDirective>(D) ||
10076 isa<OMPTargetExitDataDirective>(D) ||
10077 isa<OMPTargetUpdateDirective>(D)) &&
10078 "Expecting either target enter, exit data, or update directives.");
10083 auto &&ThenGen = [
this, &D, Device, &InputInfo,
10088 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
10089 CGF.Int64Ty,
true);
10091 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
10095 llvm::Constant *PointerNum =
10096 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
10100 InputInfo.BasePointersArray.getPointer(),
10101 InputInfo.PointersArray.getPointer(),
10102 InputInfo.SizesArray.getPointer(),
10110 case OMPD_target_enter_data:
10114 case OMPD_target_exit_data:
10118 case OMPD_target_update:
10122 case OMPD_parallel:
10124 case OMPD_parallel_for:
10125 case OMPD_parallel_master:
10126 case OMPD_parallel_sections:
10127 case OMPD_for_simd:
10128 case OMPD_parallel_for_simd:
10130 case OMPD_cancellation_point:
10132 case OMPD_threadprivate:
10133 case OMPD_allocate:
10136 case OMPD_sections:
10140 case OMPD_critical:
10141 case OMPD_taskyield:
10143 case OMPD_taskwait:
10144 case OMPD_taskgroup:
10148 case OMPD_target_data:
10149 case OMPD_distribute:
10150 case OMPD_distribute_simd:
10151 case OMPD_distribute_parallel_for:
10152 case OMPD_distribute_parallel_for_simd:
10153 case OMPD_teams_distribute:
10154 case OMPD_teams_distribute_simd:
10155 case OMPD_teams_distribute_parallel_for:
10156 case OMPD_teams_distribute_parallel_for_simd:
10157 case OMPD_declare_simd:
10158 case OMPD_declare_variant:
10159 case OMPD_declare_target:
10160 case OMPD_end_declare_target:
10161 case OMPD_declare_reduction:
10162 case OMPD_declare_mapper:
10163 case OMPD_taskloop:
10164 case OMPD_taskloop_simd:
10165 case OMPD_master_taskloop:
10166 case OMPD_master_taskloop_simd:
10167 case OMPD_parallel_master_taskloop:
10168 case OMPD_parallel_master_taskloop_simd:
10170 case OMPD_target_simd:
10171 case OMPD_target_teams_distribute:
10172 case OMPD_target_teams_distribute_simd:
10173 case OMPD_target_teams_distribute_parallel_for:
10174 case OMPD_target_teams_distribute_parallel_for_simd:
10175 case OMPD_target_teams:
10176 case OMPD_target_parallel:
10177 case OMPD_target_parallel_for:
10178 case OMPD_target_parallel_for_simd:
10179 case OMPD_requires:
10181 llvm_unreachable(
"Unexpected standalone target data directive.");
10187 auto &&TargetThenGen = [
this, &ThenGen, &D, &InputInfo, &MapTypesArray](
10196 MappableExprsHandler MEHandler(D, CGF);
10197 MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
10206 InputInfo.BasePointersArray =
10208 InputInfo.PointersArray =
10210 InputInfo.SizesArray =
10214 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
10232 struct ParamAttrTy {
10240 ArrayRef<ParamAttrTy> ParamAttrs) {
10269 if (
const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
10270 if (ParamAttrs[Offset].Kind == Vector)
10275 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
10276 if (ParamAttrs[I + Offset].Kind == Vector) {
10294 ArrayRef<ParamAttrTy> ParamAttrs,
10295 OMPDeclareSimdDeclAttr::BranchStateTy
State) {
10298 unsigned VecRegSize;
10300 ISADataTy ISAData[] = {
10316 case OMPDeclareSimdDeclAttr::BS_Undefined:
10317 Masked.push_back(
'N');
10318 Masked.push_back(
'M');
10320 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10321 Masked.push_back(
'N');
10323 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10324 Masked.push_back(
'M');
10327 for (
char Mask : Masked) {
10328 for (
const ISADataTy &Data : ISAData) {
10330 llvm::raw_svector_ostream Out(Buffer);
10331 Out <<
"_ZGV" << Data.ISA << Mask;
10334 assert(NumElts &&
"Non-zero simdlen/cdtsize expected");
10335 Out << llvm::APSInt::getUnsigned(Data.VecRegSize / NumElts);
10339 for (
const ParamAttrTy &ParamAttr : ParamAttrs) {
10340 switch (ParamAttr.Kind){
10341 case LinearWithVarStride:
10342 Out <<
's' << ParamAttr.StrideOrArg;
10346 if (!!ParamAttr.StrideOrArg)
10347 Out << ParamAttr.StrideOrArg;
10356 if (!!ParamAttr.Alignment)
10357 Out <<
'a' << ParamAttr.Alignment;
10359 Out <<
'_' << Fn->getName();
10360 Fn->addFnAttr(Out.str());
10382 if (Kind == ParamKindTy::Uniform)
10385 if (Kind == ParamKindTy::Linear)
10390 if (Kind == ParamKindTy::LinearWithVarStride)
10402 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
10437 static std::tuple<unsigned, unsigned, bool>
10443 bool OutputBecomesInput =
false;
10447 Sizes.push_back(
getAArch64LS(RetType, ParamKindTy::Vector, C));
10449 OutputBecomesInput =
true;
10451 for (
unsigned I = 0, E = FD->
getNumParams(); I < E; ++I) {
10453 Sizes.push_back(
getAArch64LS(QT, ParamAttrs[I].Kind, C));
10456 assert(!Sizes.empty() &&
"Unable to determine NDS and WDS.");
10459 assert(std::all_of(Sizes.begin(), Sizes.end(),
10460 [](
unsigned Size) {
10461 return Size == 8 || Size == 16 || Size == 32 ||
10462 Size == 64 || Size == 128;
10466 return std::make_tuple(*std::min_element(std::begin(Sizes), std::end(Sizes)),
10467 *std::max_element(std::begin(Sizes), std::end(Sizes)),
10468 OutputBecomesInput);
10476 llvm::raw_svector_ostream Out(Buffer);
10477 for (
const auto &ParamAttr : ParamAttrs) {
10478 switch (ParamAttr.Kind) {
10479 case LinearWithVarStride:
10480 Out <<
"ls" << ParamAttr.StrideOrArg;
10486 if (!!ParamAttr.StrideOrArg && ParamAttr.StrideOrArg != 1)
10487 Out << ParamAttr.StrideOrArg;
10497 if (!!ParamAttr.Alignment)
10498 Out <<
'a' << ParamAttr.Alignment;
10507 template <
typename T>
10509 char ISA, StringRef ParSeq,
10510 StringRef MangledName,
bool OutputBecomesInput,
10511 llvm::Function *Fn) {
10513 llvm::raw_svector_ostream Out(Buffer);
10514 Out << Prefix << ISA << LMask << VLEN;
10515 if (OutputBecomesInput)
10517 Out << ParSeq <<
"_" << MangledName;
10518 Fn->addFnAttr(Out.str());
10524 StringRef Prefix,
char ISA,
10525 StringRef ParSeq, StringRef MangledName,
10526 bool OutputBecomesInput,
10527 llvm::Function *Fn) {
10531 OutputBecomesInput, Fn);
10533 OutputBecomesInput, Fn);
10537 OutputBecomesInput, Fn);
10539 OutputBecomesInput, Fn);
10543 OutputBecomesInput, Fn);
10545 OutputBecomesInput, Fn);
10550 OutputBecomesInput, Fn);
10553 llvm_unreachable(
"Scalar type is too wide.");
10560 ArrayRef<ParamAttrTy> ParamAttrs,
10561 OMPDeclareSimdDeclAttr::BranchStateTy
State, StringRef MangledName,
10562 char ISA,
unsigned VecRegSize, llvm::Function *Fn,
SourceLocation SLoc) {
10565 const auto Data =
getNDSWDS(FD, ParamAttrs);
10566 const unsigned NDS = std::get<0>(Data);
10567 const unsigned WDS = std::get<1>(Data);
10568 const bool OutputBecomesInput = std::get<2>(Data);
10572 if (UserVLEN == 1) {
10575 "The clause simdlen(1) has no effect when targeting aarch64.");
10582 if (ISA ==
'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {
10585 "power of 2 when targeting Advanced SIMD.");
10592 if (ISA ==
's' && UserVLEN != 0) {
10593 if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {
10596 "lanes in the architectural constraints " 10597 "for SVE (min is 128-bit, max is " 10598 "2048-bit, by steps of 128-bit)");
10606 StringRef Prefix =
"_ZGV";
10612 OutputBecomesInput, Fn);
10614 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
10618 case OMPDeclareSimdDeclAttr::BS_Undefined:
10620 OutputBecomesInput, Fn);
10622 OutputBecomesInput, Fn);
10624 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10626 OutputBecomesInput, Fn);
10628 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10630 OutputBecomesInput, Fn);
10640 OutputBecomesInput, Fn);
10642 assert(ISA ==
'n' &&
"Expected ISA either 's' or 'n'.");
10647 case OMPDeclareSimdDeclAttr::BS_Undefined:
10649 OutputBecomesInput, Fn);
10651 OutputBecomesInput, Fn);
10653 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
10655 OutputBecomesInput, Fn);
10657 case OMPDeclareSimdDeclAttr::BS_Inbranch:
10659 OutputBecomesInput, Fn);
10667 llvm::Function *Fn) {
10671 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
10672 if (isa<CXXMethodDecl>(FD))
10673 ParamPositions.try_emplace(FD, 0);
10674 unsigned ParamPos = ParamPositions.size();
10676 ParamPositions.try_emplace(
P->getCanonicalDecl(), ParamPos);
10683 for (
const Expr *E :
Attr->uniforms()) {
10686 if (isa<CXXThisExpr>(E)) {
10687 Pos = ParamPositions[FD];
10689 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10691 Pos = ParamPositions[PVD];
10693 ParamAttrs[Pos].Kind = Uniform;
10696 auto NI =
Attr->alignments_begin();
10697 for (
const Expr *E :
Attr->aligneds()) {
10701 if (isa<CXXThisExpr>(E)) {
10702 Pos = ParamPositions[FD];
10705 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10707 Pos = ParamPositions[PVD];
10708 ParmTy = PVD->getType();
10710 ParamAttrs[Pos].Alignment =
10712 ? (*NI)->EvaluateKnownConstInt(C)
10713 : llvm::APSInt::getUnsigned(
10719 auto SI =
Attr->steps_begin();
10720 auto MI =
Attr->modifiers_begin();
10721 for (
const Expr *E :
Attr->linears()) {
10724 if (isa<CXXThisExpr>(E)) {
10725 Pos = ParamPositions[FD];
10727 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl())
10729 Pos = ParamPositions[PVD];
10731 ParamAttrTy &ParamAttr = ParamAttrs[Pos];
10732 ParamAttr.Kind = Linear;
10736 if (
const auto *DRE =
10737 cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
10738 if (
const auto *StridePVD = cast<ParmVarDecl>(DRE->getDecl())) {
10739 ParamAttr.Kind = LinearWithVarStride;
10740 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(
10741 ParamPositions[StridePVD->getCanonicalDecl()]);
10745 ParamAttr.StrideOrArg = Result.
Val.
getInt();
10753 const Expr *VLENExpr =
Attr->getSimdlen();
10758 OMPDeclareSimdDeclAttr::BranchStateTy
State =
Attr->getBranchState();
10759 if (CGM.getTriple().isX86()) {
10761 }
else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) {
10762 unsigned VLEN = VLENVal.getExtValue();
10763 StringRef MangledName = Fn->getName();
10764 if (CGM.getTarget().hasFeature(
"sve"))
10766 MangledName,
's', 128, Fn, ExprLoc);
10767 if (CGM.getTarget().hasFeature(
"neon"))
10769 MangledName,
'n', 128, Fn, ExprLoc);
10780 static const int DoacrossFinArgs = 2;
10783 llvm::FunctionCallee RTLFn;
10787 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
10790 assert(CallArgs.size() == DoacrossFinArgs);
10791 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
10832 enum { LowerFD = 0, UpperFD, StrideFD };
10834 for (
unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
10839 DimsLVal, *std::next(RD->
field_begin(), UpperFD));
10847 DimsLVal, *std::next(RD->
field_begin(), StrideFD));
10857 llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()),
10862 llvm::FunctionCallee RTLFn =
10865 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
10867 llvm::FunctionCallee FiniRTLFn =
10870 llvm::makeArrayRef(FiniArgs));
10876 CGM.getContext().getIntTypeForBitwidth(64, 1);
10878 QualType ArrayTy = CGM.getContext().getConstantArrayType(
10881 for (
unsigned I = 0, E = C->
getNumLoops(); I < E; ++I) {
10883 assert(CounterVal);
10894 llvm::FunctionCallee RTLFn;
10905 llvm::FunctionCallee Callee,
10907 assert(Loc.
isValid() &&
"Outlined function call location must be valid.");
10910 if (
auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {
10911 if (Fn->doesNotThrow()) {
10922 emitCall(CGF, Loc, OutlinedFn, Args);
10926 if (
const auto *FD = dyn_cast<FunctionDecl>(D))
10927 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))
10933 const VarDecl *TargetParam)
const {
10941 static const int CleanupArgs = 3;
10944 llvm::FunctionCallee RTLFn;
10948 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
10951 assert(CallArgs.size() == CleanupArgs &&
10952 "Size of arguments does not match.");
10953 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));
10968 if (!CVD->
hasAttr<OMPAllocateDeclAttr>())
10970 const auto *AA = CVD->
getAttr<OMPAllocateDeclAttr>();
10972 if (AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
10973 !AA->getAllocator())
10976 CharUnits Align = CGM.getContext().getDeclAlign(CVD);
10980 Size = CGF.
Builder.CreateNUWAdd(
10982 Size = CGF.
Builder.CreateUDiv(Size, CGM.getSize(Align));
10983 Size = CGF.
Builder.CreateNUWMul(Size, CGM.getSize(Align));
10986 Size = CGM.getSize(Sz.
alignTo(Align));
10989 assert(AA->getAllocator() &&
10990 "Expected allocator expression for non-default allocator.");
10994 if (Allocator->getType()->isIntegerTy())
10995 Allocator = CGF.
Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
10996 else if (Allocator->getType()->isPointerTy())
10999 llvm::Value *Args[] = {ThreadID, Size, Allocator};
11003 CVD->
getName() +
".void.addr");
11004 llvm::Value *FiniArgs[OMPAllocateCleanupTy::CleanupArgs] = {ThreadID, Addr,
11009 llvm::makeArrayRef(FiniArgs));
11018 using OMPContextSelectorData =
11025 typename... Arguments>
11027 Arguments... Params) {
11029 "Unknown context selector or context selector set.");
11037 const OMPContextSelectorData &Data) {
11038 return llvm::all_of(Data.Names,
11039 [](StringRef S) { return !S.compare_lower(
"llvm"); });
11049 bool checkContext<OMP_CTX_SET_device, OMP_CTX_kind, CodeGenModule &>(
11051 for (StringRef Name : Data.Names) {
11052 if (!Name.compare_lower(
"host")) {
11053 if (CGM.getLangOpts().OpenMPIsDevice)
11057 if (!Name.compare_lower(
"nohost")) {
11058 if (!CGM.getLangOpts().OpenMPIsDevice)
11062 switch (CGM.getTriple().getArch()) {
11063 case llvm::Triple::arm:
11064 case llvm::Triple::armeb:
11065 case llvm::Triple::aarch64:
11066 case llvm::Triple::aarch64_be:
11067 case llvm::Triple::aarch64_32:
11068 case llvm::Triple::ppc:
11069 case llvm::Triple::ppc64:
11070 case llvm::Triple::ppc64le:
11071 case llvm::Triple::x86:
11072 case llvm::Triple::x86_64:
11073 if (Name.compare_lower(
"cpu"))
11076 case llvm::Triple::amdgcn:
11077 case llvm::Triple::nvptx:
11078 case llvm::Triple::nvptx64:
11079 if (Name.compare_lower(
"gpu"))
11082 case llvm::Triple::UnknownArch:
11083 case llvm::Triple::arc:
11084 case llvm::Triple::avr:
11085 case llvm::Triple::bpfel:
11086 case llvm::Triple::bpfeb:
11087 case llvm::Triple::hexagon:
11088 case llvm::Triple::mips:
11089 case llvm::Triple::mipsel:
11090 case llvm::Triple::mips64:
11091 case llvm::Triple::mips64el:
11092 case llvm::Triple::msp430:
11093 case llvm::Triple::r600:
11094 case llvm::Triple::riscv32:
11095 case llvm::Triple::riscv64:
11096 case llvm::Triple::sparc:
11097 case llvm::Triple::sparcv9:
11098 case llvm::Triple::sparcel:
11099 case llvm::Triple::systemz:
11100 case llvm::Triple::tce:
11101 case llvm::Triple::tcele:
11102 case llvm::Triple::thumb:
11103 case llvm::Triple::thumbeb:
11104 case llvm::Triple::xcore:
11105 case llvm::Triple::le32:
11106 case llvm::Triple::le64:
11107 case llvm::Triple::amdil:
11108 case llvm::Triple::amdil64:
11109 case llvm::Triple::hsail:
11110 case llvm::Triple::hsail64:
11111 case llvm::Triple::spir:
11112 case llvm::Triple::spir64:
11113 case llvm::Triple::kalimba:
11114 case llvm::Triple::shave:
11115 case llvm::Triple::lanai:
11116 case llvm::Triple::wasm32:
11117 case llvm::Triple::wasm64:
11118 case llvm::Triple::renderscript32:
11119 case llvm::Triple::renderscript64:
11120 case llvm::Triple::ve:
11128 const CompleteOMPContextSelectorData &ContextData) {
11129 for (
const OMPContextSelectorData &Data : ContextData) {
11130 switch (Data.Ctx) {
11131 case OMP_CTX_vendor:
11132 assert(Data.CtxSet == OMP_CTX_SET_implementation &&
11133 "Expected implementation context selector set.");
11138 assert(Data.CtxSet == OMP_CTX_SET_device &&
11139 "Expected device context selector set.");
11140 if (!checkContext<OMP_CTX_SET_device, OMP_CTX_kind, CodeGenModule &>(Data,
11145 llvm_unreachable(
"Unknown context selector kind.");
11151 static CompleteOMPContextSelectorData
11153 const OMPDeclareVariantAttr *A) {
11154 CompleteOMPContextSelectorData Data;
11155 for (
unsigned I = 0, E = A->scores_size(); I < E; ++I) {
11156 Data.emplace_back();
11158 *std::next(A->ctxSelectorSets_begin(), I));
11160 *std::next(A->ctxSelectors_begin(), I));
11161 Data.back().CtxSet = CtxSet;
11162 Data.back().Ctx = Ctx;
11163 const Expr *Score = *std::next(A->scores_begin(), I);
11166 case OMP_CTX_vendor:
11167 assert(CtxSet == OMP_CTX_SET_implementation &&
11168 "Expected implementation context selector set.");
11169 Data.back().Names =
11170 llvm::makeArrayRef(A->implVendors_begin(), A->implVendors_end());
11173 assert(CtxSet == OMP_CTX_SET_device &&
11174 "Expected device context selector set.");
11175 Data.back().Names =
11176 llvm::makeArrayRef(A->deviceKinds_begin(), A->deviceKinds_end());
11179 llvm_unreachable(
"Unknown context selector kind.");
11186 const CompleteOMPContextSelectorData &RHS) {
11187 llvm::SmallDenseMap<std::pair<int, int>, llvm::StringSet<>, 4> RHSData;
11188 for (
const OMPContextSelectorData &D : RHS) {
11189 auto &Pair = RHSData.FindAndConstruct(std::make_pair(D.CtxSet, D.Ctx));
11190 Pair.getSecond().insert(D.Names.begin(), D.Names.end());
11192 bool AllSetsAreEqual =
true;
11193 for (
const OMPContextSelectorData &D : LHS) {
11194 auto It = RHSData.find(std::make_pair(D.CtxSet, D.Ctx));
11195 if (It == RHSData.end())
11197 if (D.Names.size() > It->getSecond().size())
11199 if (llvm::set_union(It->getSecond(), D.Names))
11202 AllSetsAreEqual && (D.Names.size() == It->getSecond().size());
11205 return LHS.size() != RHS.size() || !AllSetsAreEqual;
11209 const CompleteOMPContextSelectorData &RHS) {
11213 if (RHSIsSubsetOfLHS) {
11214 LHSScore = llvm::APSInt::get(0);
11216 for (
const OMPContextSelectorData &Data : LHS) {
11217 if (Data.Score.getBitWidth() > LHSScore.getBitWidth()) {
11218 LHSScore = LHSScore.extend(Data.Score.getBitWidth()) + Data.Score;
11219 }
else if (Data.Score.getBitWidth() < LHSScore.getBitWidth()) {
11220 LHSScore += Data.Score.extend(LHSScore.getBitWidth());
11222 LHSScore += Data.Score;
11228 RHSScore = llvm::APSInt::get(0);
11230 for (
const OMPContextSelectorData &Data : RHS) {
11231 if (Data.Score.getBitWidth() > RHSScore.getBitWidth()) {
11232 RHSScore = RHSScore.extend(Data.Score.getBitWidth()) + Data.Score;
11233 }
else if (Data.Score.getBitWidth() < RHSScore.getBitWidth()) {
11234 RHSScore += Data.Score.extend(RHSScore.getBitWidth());
11236 RHSScore += Data.Score;
11240 return llvm::APSInt::compareValues(LHSScore, RHSScore) >= 0;
11250 const OMPDeclareVariantAttr *TopMostAttr =
nullptr;
11251 CompleteOMPContextSelectorData TopMostData;
11252 for (
const auto *A : FD->
specific_attrs<OMPDeclareVariantAttr>()) {
11253 CompleteOMPContextSelectorData Data =
11261 TopMostData.swap(Data);
11266 return cast<FunctionDecl>(
11267 cast<DeclRefExpr>(TopMostAttr->getVariantFuncRef()->IgnoreParenImpCasts())
11272 const auto *D = cast<FunctionDecl>(GD.
getDecl());
11274 StringRef MangledName = CGM.getMangledName(GD);
11275 llvm::GlobalValue *Orig = CGM.GetGlobalValue(MangledName);
11276 if (Orig && !Orig->isDeclaration())
11295 assert(CGM.
getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11301 for (
const Stmt *Ref : C->private_refs()) {
11302 const auto *SimpleRefExpr = cast<Expr>(Ref)->IgnoreParenImpCasts();
11304 if (
const auto *DRE = dyn_cast<DeclRefExpr>(SimpleRefExpr)) {
11305 VD = DRE->getDecl();
11307 const auto *ME = cast<MemberExpr>(SimpleRefExpr);
11308 assert((ME->isImplicitCXXThis() ||
11309 isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&
11310 "Expected member of current class.");
11311 VD = ME->getMemberDecl();
11325 assert(CGM.
getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11327 return llvm::any_of(
11337 return C->getKind() ==
11338 OMPC_LASTPRIVATE_conditional;
11340 assert(CGM.
getLangOpts().OpenMP &&
"Not in OpenMP mode.");
11346 if (C->getKind() != OMPC_LASTPRIVATE_conditional)
11349 for (
const Expr *Ref : C->varlists()) {
11351 cast<DeclRefExpr>(Ref->IgnoreParenImpCasts())->getDecl(),
11365 llvm::raw_svector_ostream OS(Buffer);
11368 assert(PLoc.
isValid() &&
"Source location is expected to be always valid.");
11370 llvm::sys::fs::UniqueID
ID;
11371 if (
auto EC = llvm::sys::fs::getUniqueID(PLoc.
getFilename(),
ID))
11374 OS <<
"$pl_cond_" << ID.getDevice() <<
"_" << ID.getFile() <<
"_" 11390 return C->getKind() == OMPC_LASTPRIVATE_conditional;
11408 class LastprivateConditionalRefChecker final
11412 const Expr *FoundE =
nullptr;
11413 const Decl *FoundD =
nullptr;
11414 StringRef UniqueDeclName;
11418 bool UseOriginalIV =
false;
11423 llvm::reverse(LPM)) {
11424 auto It = D.DeclToUniqeName.find(E->
getDecl());
11425 if (It == D.DeclToUniqeName.end())
11429 UniqueDeclName = It->getSecond();
11432 UseOriginalIV = D.UseOriginalIV;
11435 return FoundE == E;
11438 if (!CGF.IsWrappedCXXThis(E->
getBase()))
11441 llvm::reverse(LPM)) {
11443 if (It == D.DeclToUniqeName.end())
11447 UniqueDeclName = It->getSecond();
11450 UseOriginalIV = D.UseOriginalIV;
11453 return FoundE == E;
11455 bool VisitStmt(
const Stmt *S) {
11459 if (
const auto *E = dyn_cast<Expr>(Child))
11467 explicit LastprivateConditionalRefChecker(
11470 : CGF(CGF), LPM(LPM) {}
11471 std::tuple<const Expr *, const Decl *, StringRef, LValue, StringRef, bool>
11472 getFoundData()
const {
11473 return std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, IVName,
11484 if (!Checker.Visit(LHS))
11486 const Expr *FoundE;
11487 const Decl *FoundD;
11488 StringRef UniqueDeclName;
11491 bool UseOriginalIV;
11492 std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, IVName, UseOriginalIV) =
11493 Checker.getFoundData();
11498 llvm::Constant *LastIV =
11500 cast<llvm::GlobalVariable>(LastIV)->setAlignment(
11511 cast<llvm::GlobalVariable>(Last)->setAlignment(
11518 if (!UseOriginalIV) {
11530 auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
11534 CGF.EmitLoadOfScalar(LastIVLVal, FoundE->
getExprLoc());
11538 if (IVLVal.getType()->isSignedIntegerType()) {
11539 CmpRes = CGF.Builder.CreateICmpSLE(LastIVVal, IVVal);
11541 assert(IVLVal.getType()->isUnsignedIntegerType() &&
11542 "Loop iteration variable must be integer.");
11543 CmpRes = CGF.Builder.CreateICmpULE(LastIVVal, IVVal);
11545 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(
"lp_cond_then");
11546 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(
"lp_cond_exit");
11547 CGF.Builder.CreateCondBr(CmpRes, ThenBB, ExitBB);
11549 CGF.EmitBlock(ThenBB);
11552 CGF.EmitStoreOfScalar(IVVal, LastIVLVal);
11555 switch (CGF.getEvaluationKind(LVal.getType())) {
11558 CGF.EmitStoreOfScalar(PrivVal, LastLVal);
11563 CGF.EmitLoadOfComplex(LVal, FoundE->
getExprLoc());
11564 CGF.EmitStoreOfComplex(PrivVal, LastLVal,
false);
11569 "Aggregates are not supported in lastprivate conditional.");
11572 CGF.EmitBranch(ExitBB);
11575 CGF.EmitBlock(ExitBB,
true);
11578 if (CGM.getLangOpts().OpenMPSimd) {
11594 "Unknown lastprivate conditional variable.");
11595 StringRef UniqueName = It->getSecond();
11596 llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(UniqueName);
11609 llvm_unreachable(
"Not supported in SIMD-only mode");
11615 llvm_unreachable(
"Not supported in SIMD-only mode");
11622 bool Tied,
unsigned &NumberOfParts) {
11623 llvm_unreachable(
"Not supported in SIMD-only mode");
11628 llvm::Function *OutlinedFn,
11630 const Expr *IfCond) {
11631 llvm_unreachable(
"Not supported in SIMD-only mode");
11637 const Expr *Hint) {
11638 llvm_unreachable(
"Not supported in SIMD-only mode");
11644 llvm_unreachable(
"Not supported in SIMD-only mode");
11649 llvm_unreachable(
"Not supported in SIMD-only mode");
11655 llvm_unreachable(
"Not supported in SIMD-only mode");
11663 llvm_unreachable(
"Not supported in SIMD-only mode");
11670 llvm_unreachable(
"Not supported in SIMD-only mode");
11677 bool ForceSimpleCall) {
11678 llvm_unreachable(
"Not supported in SIMD-only mode");
11685 llvm_unreachable(
"Not supported in SIMD-only mode");
11691 llvm_unreachable(
"Not supported in SIMD-only mode");
11697 llvm_unreachable(
"Not supported in SIMD-only mode");
11704 llvm_unreachable(
"Not supported in SIMD-only mode");
11710 llvm_unreachable(
"Not supported in SIMD-only mode");
11715 unsigned IVSize,
bool IVSigned,
11718 llvm_unreachable(
"Not supported in SIMD-only mode");
11724 llvm_unreachable(
"Not supported in SIMD-only mode");
11728 ProcBindKind ProcBind,
11730 llvm_unreachable(
"Not supported in SIMD-only mode");
11737 llvm_unreachable(
"Not supported in SIMD-only mode");
11743 llvm_unreachable(
"Not supported in SIMD-only mode");
11748 llvm_unreachable(
"Not supported in SIMD-only mode");
11754 llvm_unreachable(
"Not supported in SIMD-only mode");
11759 llvm::Function *TaskFunction,
11761 const Expr *IfCond,
11763 llvm_unreachable(
"Not supported in SIMD-only mode");
11770 llvm_unreachable(
"Not supported in SIMD-only mode");
11777 assert(Options.
SimpleReduction &&
"Only simple reduction is expected.");
11779 ReductionOps, Options);
11785 llvm_unreachable(
"Not supported in SIMD-only mode");
11792 llvm_unreachable(
"Not supported in SIMD-only mode");
11799 llvm_unreachable(
"Not supported in SIMD-only mode");
11804 llvm_unreachable(
"Not supported in SIMD-only mode");
11810 llvm_unreachable(
"Not supported in SIMD-only mode");
11816 llvm_unreachable(
"Not supported in SIMD-only mode");
11821 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
11823 llvm_unreachable(
"Not supported in SIMD-only mode");
11828 llvm::Function *OutlinedFn,
llvm::Value *OutlinedFnID,
const Expr *IfCond,
11829 const Expr *Device,
11833 llvm_unreachable(
"Not supported in SIMD-only mode");
11837 llvm_unreachable(
"Not supported in SIMD-only mode");
11841 llvm_unreachable(
"Not supported in SIMD-only mode");
11851 llvm::Function *OutlinedFn,
11853 llvm_unreachable(
"Not supported in SIMD-only mode");
11857 const Expr *NumTeams,
11858 const Expr *ThreadLimit,
11860 llvm_unreachable(
"Not supported in SIMD-only mode");
11866 llvm_unreachable(
"Not supported in SIMD-only mode");
11871 const Expr *Device) {
11872 llvm_unreachable(
"Not supported in SIMD-only mode");
11878 llvm_unreachable(
"Not supported in SIMD-only mode");
11883 llvm_unreachable(
"Not supported in SIMD-only mode");
11888 const VarDecl *NativeParam)
const {
11889 llvm_unreachable(
"Not supported in SIMD-only mode");
11895 const VarDecl *TargetParam)
const {
11896 llvm_unreachable(
"Not supported in SIMD-only mode");
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
llvm::PointerType * Int8PtrPtrTy
RecordDecl * buildImplicitRecord(StringRef Name, RecordDecl::TagKind TK=TTK_Struct) const
Create a new implicit TU-level CXXRecordDecl or RecordDecl declaration.
TBAAAccessInfo getTBAAInfoForSubobject(LValue Base, QualType AccessType)
getTBAAInfoForSubobject - Get TBAA information for an access with a given base lvalue.
void pushTerminate()
Push a terminate handler on the stack.
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, llvm::Type *BaseLVType, CharUnits BaseLVAlignment, llvm::Value *Addr)
CharUnits alignTo(const CharUnits &Align) const
alignTo - Returns the next integer (mod 2**64) that is greater than or equal to this quantity and is ...
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
DisableAutoDeclareTargetRAII(CodeGenModule &CGM)
This represents '#pragma omp task' directive.
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
llvm::IntegerType * IntTy
int
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
bool hasRequiresUnifiedSharedMemory() const
Return whether the unified_shared_memory has been specified.
virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD)
Gets the OpenMP-specific address of the local variable.
llvm::SmallVector< LastprivateConditionalData, 4 > LastprivateConditionalStack
Stack for list of addresses of declarations in current context marked as lastprivate conditional...
bool checkContext< OMP_CTX_SET_implementation, OMP_CTX_vendor >(const OMPContextSelectorData &Data)
Checks for implementation={vendor(<vendor>)} context selector.
Expr * getUpperBoundVariable() const
Other implicit parameter.
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param Data Additional data for task generation like final list of privates etc *TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data)
unsigned getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it...
PointerType - C99 6.7.5.1 - Pointer Declarators.
unsigned llvm::PointerUnion< const Decl *, const Expr * > DeclTy
Scheduling data for loop-based OpenMP directives.
Struct to store the context selectors info.
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
A (possibly-)qualified type.
CodeGenTypes & getTypes()
llvm::Function * emitReductionFunction(SourceLocation Loc, llvm::Type *ArgsType, ArrayRef< const Expr *> Privates, ArrayRef< const Expr *> LHSExprs, ArrayRef< const Expr *> RHSExprs, ArrayRef< const Expr *> ReductionOps)
Emits reduction function.
ValueDecl * getMemberDecl() const
Retrieve the member declaration to which this expression refers.
llvm::Type * ConvertTypeForMem(QualType T)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
const CodeGenOptions & getCodeGenOpts() const
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
GlobalDecl getWithDecl(const Decl *D)
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
CGRecordLayout - This class handles struct and union layout info while lowering AST types to LLVM typ...
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
const RecordDecl * KmpTaskTQTyRD
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
void emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS)
Emits single reduction combiner.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
void actOnDeviceGlobalVarEntriesInfo(const OffloadDeviceGlobalVarEntryInfoActTy &Action)
~LastprivateConditionalRAII()
The standard implementation of ConstantInitBuilder used in Clang.
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D...
Stmt - This represents one statement.
Expr * getLowerBoundVariable() const
OpenMPOffloadingRequiresDirFlags
Values for bit flags for marking which requires clauses have been used.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D)
Emit code for the specified user defined reduction construct.
Expr * getLoopData(unsigned NumLoop)
Get the loop data.
bool capturesThis() const
Determine whether this capture handles the C++ 'this' pointer.
VarDecl * getCapturedVar() const
Retrieve the declaration of the variable being captured.
bool emitTargetGlobal(GlobalDecl GD) override
Emit the global GD if it is meaningful for the target.
CharUnits getAlignOfGlobalVarInChars(QualType T) const
Return the alignment in characters that should be given to a global variable with type T...
bool hasNonTrivialCall(const ASTContext &Ctx) const
Determine whether this expression involves a call to any function that is not trivial.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator, StringRef Separator)
Constructor allowing to redefine the name separator for the variables.
bool isRecordType() const
virtual void initLastprivateConditionalCounter(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Initializes global counter for lastprivate conditional.
fileinfo_iterator fileinfo_end() const
void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) override
This is used for non static scheduled types and when the ordered clause is present on the loop constr...
void clearLocThreadIdInsertPt(CodeGenFunction &CGF)
Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const override
Gets the address of the native argument basing on the address of the target-specific parameter...
static CompleteOMPContextSelectorData translateAttrToContextSelectorData(ASTContext &C, const OMPDeclareVariantAttr *A)
SmallVector< const Expr *, 4 > LastprivateCopies
Decl - This represents one declaration (or definition), e.g.
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
static llvm::Value * getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, llvm::Value *DefaultThreadLimitVal)
static llvm::Value * emitNumTeamsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit the number of teams for a target directive.
Entry is a target region.
CharUnits getPointerSize() const
virtual llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr *> LHSExprs, ArrayRef< const Expr *> RHSExprs, const OMPTaskDataTy &Data)
Emit a code for initialization of task reduction clause.
const RecordType * getAsStructureType() const
llvm::SmallDenseSet< const VarDecl * > DeferredGlobalVariables
List of variables that can become declare target implicitly and, thus, must be emitted.
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Function * emitTeamsOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP teams directive D.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
This represents 'if' clause in the '#pragma omp ...' directive.
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, ArrayRef< const Expr *> PrivateVars, ArrayRef< const Expr *> FirstprivateVars, ArrayRef< const Expr *> LastprivateVars, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables...
virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name)
Creates artificial threadprivate variable with name Name and type VarType.
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
unsigned getFieldIndex() const
Returns the index of this field within its record, as appropriate for passing to ASTRecordLayout::get...
Call to void __kmpc_threadprivate_register( ident_t *, void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
ReductionCodeGen(ArrayRef< const Expr *> Shareds, ArrayRef< const Expr *> Privates, ArrayRef< const Expr *> ReductionOps)
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
bool isOpenMPTargetDataManagementDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target data offload directive.
The base class of the type hierarchy.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr *> Privates, ArrayRef< const Expr *> LHSExprs, ArrayRef< const Expr *> RHSExprs, ArrayRef< const Expr *> ReductionOps, ReductionOptionsTy Options) override
Emit a code for reduction clause.
void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device) override
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Represents an array type, per C99 6.7.5.2 - Array Declarators.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
llvm::Value * PointersArray
The array of section pointers passed to the runtime library.
virtual void completeDefinition()
Note that the definition of this type is now complete.
bool isZero() const
isZero - Test whether the quantity equals zero.
QualType withConst() const
bool markAsGlobalTarget(GlobalDecl GD)
Marks the declaration as already emitted for the device code and returns true, if it was marked alrea...
static const FunctionDecl * getDeclareVariantFunction(CodeGenModule &CGM, const FunctionDecl *FD)
Finds the variant function that matches current context with its context selector.
static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2)
const TargetInfo & getTargetInfo() const
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
capture_const_range captures() const
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
void emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Expr * getCondition() const
Returns condition.
LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field)
EmitLValueForFieldInitialization - Like EmitLValueForField, except that if the Field is a reference...
const Expr * getAnyInitializer() const
Get the initializer for this variable, no matter which declaration it is attached to...
virtual void checkArchForUnifiedAddressing(const OMPRequiresDecl *D)
Perform check on requires decl to ensure that target architecture supports unified addressing...
Describes the capture of a variable or of this, or of a C++1y init-capture.
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and and emit all target regions found along the way.
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
Represents a C++ constructor within a class.
static llvm::Value * emitNumThreadsForTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit the number of threads for a target directive.
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS...
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
OpenMPSchedType
Schedule types for 'omp for' loops (these enumerators are taken from the enum sched_type in kmp...
bool isTrivialType(const ASTContext &Context) const
Return true if this is a trivial type per (C++0x [basic.types]p9)
SmallVector< const Expr *, 4 > ReductionCopies
SourceLocation getEndLoc() const
Returns ending location of directive.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
static const Stmt * getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body)
Checks if the Body is the CompoundStmt and returns its child statement iff there is only one that is ...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
Represents a variable declaration or definition.
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
QualType getReturnType() const
This represents 'num_threads' clause in the '#pragma omp ...' directive.
const T * getAs() const
Member-template getAs<specific type>'.
LastprivateConditionalRAII(CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal)
Extra information about a function prototype.
LangAS
Defines the address space values used by the address space qualifier of QualType. ...
llvm::GlobalVariable * finishAndCreateGlobal(As &&...args)
Given that this builder was created by beginning an array or struct directly on a ConstantInitBuilder...
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
DiagnosticsEngine & getDiags() const
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
llvm::Value * getPointer() const
llvm::Type * ConvertTypeForMem(QualType T)
ConvertTypeForMem - Convert type T into a llvm::Type.
Don't generate debug info.
static llvm::GlobalVariable * createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant, ArrayRef< llvm::Constant *> Data, const Twine &Name, As &&... Args)
Represents a parameter to a function.
Linkage
Describes the different kinds of linkage (C++ [basic.link], C99 6.2.2) that an entity may have...
static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix, const Expr *Ref)
Generates unique name for artificial threadprivate variables.
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata...
llvm::FunctionCallee createForStaticInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_for_static_init_* runtime function for the specified size IVSize and sign IVSigned...
static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
std::string getName(ArrayRef< StringRef > Parts) const
Get the platform-specific name separator.
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
Struct that keeps all the relevant information that should be kept throughout a 'target data' region...
QualType getTgtOffloadEntryQTy()
Returns __tgt_offload_entry type.
llvm::Function * TaskEntry
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
SmallVector< const Expr *, 4 > PrivateVars
Represents a struct/union/class.
clauselist_range clauselists()
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
Source[4] in Fortran, do not use for C++.
llvm::StringMap< llvm::WeakTrackingVH > EmittedNonTargetVariables
List of the global variables with their addresses that should not be emitted for the target...
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
emitDestroy - Immediately perform the destruction of the given object.
bool HasEmittedTargetRegion
Flag for keeping track of weather a target region has been emitted.
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV)
virtual llvm::Function * emitParallelOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param Data Additional data for task generation like final list of privates etc *virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, const Expr *Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit the target offloading code associated with D.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
Call to void *__kmpc_threadprivate_cached(ident_t *loc, kmp_int32 global_tid, void *data...
void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc) override
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
llvm::IntegerType * Int64Ty
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
field_range fields() const
SmallVector< const Expr *, 4 > LastprivateVars
SourceLocation getBeginLoc() const LLVM_READONLY
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
Represents a member of a struct/union/class.
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
CharUnits getAlignment() const
SourceLocation translateFileLineCol(const FileEntry *SourceFile, unsigned Line, unsigned Col) const
Get the source location for the given file:line:col triplet.
llvm::IntegerType * SizeTy
llvm::SmallDenseMap< CanonicalDeclPtr< const Decl >, SmallString< 16 > > DeclToUniqeName
static std::string mangleVectorParameters(ArrayRef< ParamAttrTy > ParamAttrs)
Mangle the parameter part of the vector function name according to their OpenMP classification.
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive...
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
bool isNontemporalDecl(const ValueDecl *VD) const
Checks if the VD variable is marked as nontemporal declaration in current context.
static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C)
Computes the lane size (LS) of a return type or of an input parameter, as defined by LS(P) in 3...
OpenMPContextSelectorKind
OpenMP context selectors.
void setLocThreadIdInsertPt(CodeGenFunction &CGF, bool AtCurrentPoint=false)
std::pair< llvm::Value *, llvm::Value * > getSizes(unsigned N) const
Returns the size of the reduction item (in chars and total number of elements in the item)...
LValue EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, bool IsLowerBound=true)
unsigned getNonVirtualBaseLLVMFieldNo(const CXXRecordDecl *RD) const
static void emitOffloadingArrays(CodeGenFunction &CGF, MappableExprsHandler::MapBaseValuesArrayTy &BasePointers, MappableExprsHandler::MapValuesArrayTy &Pointers, MappableExprsHandler::MapValuesArrayTy &Sizes, MappableExprsHandler::MapFlagsArrayTy &MapTypes, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the arrays used to pass the captures and map information to the offloading runtime library...
void startDefinition()
Starts the definition of this tag declaration.
bool isReferenceType() const
This represents clause 'map' in the '#pragma omp ...' directives.
InitKind getInitializerKind() const
Get initializer kind.
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
OMPTargetGlobalVarEntryKind
Kind of the global variable entry..
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable. ...
This represents clause 'to' in the '#pragma omp ...' directives.
static CharUnits Zero()
Zero - Construct a CharUnits quantity of zero.
clang::CharUnits operator*(clang::CharUnits::QuantityType Scale, const clang::CharUnits &CU)
static void emitAArch64DeclareSimdFunction(CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName, char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc)
Emit vector function attributes for AArch64, as defined in the AAVFABI.
llvm::Type * getKmpc_MicroPointerTy()
Returns pointer to kmpc_micro type.
OpenMPDirectiveKind getDirectiveKind() const
__DEVICE__ int max(int __a, int __b)
SourceLocation getBeginLoc() const LLVM_READONLY
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc...
void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc) override
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const
Return the store size, in character units, of the given LLVM type.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
ArrayRef< ParmVarDecl * > parameters() const
bool EvaluateAsBooleanCondition(bool &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsBooleanCondition - Return true if this is a constant which we can fold and convert to a boo...
Mark the entry as a to declare target.
virtual bool tryEmitDeclareVariant(const GlobalDecl &NewGD, const GlobalDecl &OldGD, llvm::GlobalValue *OrigAddr, bool IsForDefinition)
Tries to emit declare variant function for OldGD from NewGD.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
Expr * getMapperVarRef()
Get the variable declared in the mapper.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr *> NumIterations)
Emit initialization for doacross loop nesting support.
void emitUserDefinedMapper(const OMPDeclareMapperDecl *D, CodeGenFunction *CGF=nullptr)
Emit the function for the user defined mapper construct.
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
QualType TgtOffloadEntryQTy
Type struct __tgt_offload_entry{ void *addr; // Pointer to the offload entry info.
static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM, const RecordDecl *RD, const CGRecordLayout &RL, ArrayRef< llvm::Constant *> Data)
static llvm::Function * emitParallelOrTeamsOutlinedFunction(CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen)
OpenMPContextSelectorSetKind
OpenMP context selector sets.
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, bool EmitDeclareReductionInit, const Expr *Init, const OMPDeclareReductionDecl *DRD, Address SrcAddr=Address::invalid())
Emit initialization of arrays of complex types.
static bool getAArch64MTV(QualType QT, ParamKindTy Kind)
Maps To Vector (MTV), as defined in 3.1.1 of the AAVFABI.
OpenMPScheduleClauseKind Schedule
Address CreateElementBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Cast the element type of the given address to a different type, preserving information like the align...
CharUnits - This is an opaque type for sizes expressed in character units.
void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) override
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
APValue Val
Val - This is the value the expression can be folded to.
bool requiresLandingPad() const
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, const Expr *Device, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter) override
Emit the target offloading code associated with D.
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info)
Emit the target data mapping code associated with D.
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Emits cleanup code for the reduction item.
RAII for correct setting/restoring of CapturedStmtInfo.
const Stmt * getPreInitStmt() const
Get pre-initialization statement for the clause.
CharUnits getAlignment() const
Return the alignment of this pointer.
void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override
Emits code for a taskyield directive.
llvm::StringSet ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
llvm::PointerType * VoidPtrTy
bool empty() const
Return true if a there are no entries defined.
String describing the source location.
void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) override
Emits a master region.
virtual std::pair< llvm::Function *, llvm::Function * > getUserDefinedReduction(const OMPDeclareReductionDecl *D)
Get combiner/initializer for the specified user-defined reduction, if any.
void actOnTargetRegionEntriesInfo(const OffloadTargetRegionEntryInfoActTy &Action)
OpenMPScheduleClauseModifier M2
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind...
SmallVector< const Expr *, 4 > PrivateCopies
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
llvm::StructType * getBaseSubobjectLLVMType() const
Return the "base subobject" LLVM type associated with this record.
unsigned NumberOfPtrs
The total number of pointers passed to the runtime library.
void operator()(CodeGenFunction &CGF) const
bool isConstexpr() const
Whether this variable is (C++11) constexpr.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
bool isLambda() const
Determine whether this class describes a lambda function object.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations...
Base class of the entries info.
Expr * getSizeExpr() const
Scope - A scope is a transient data structure that is used while parsing the program.
llvm::PointerType * VoidPtrPtrTy
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
virtual void emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn)
Marks function Fn with properly mangled versions of vector functions.
field_iterator field_begin() const
bool usesReductionInitializer(unsigned N) const
Returns true if the initialization of the reduction item uses initializer from declare reduction cons...
static std::tuple< unsigned, unsigned, bool > getNDSWDS(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
ArrayRef< MappableComponent > MappableExprComponentListRef
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
void addCompilerUsedGlobal(llvm::GlobalValue *GV)
Add a global to a list to be added to the llvm.compiler.used metadata.
SmallVector< const Expr *, 4 > FirstprivateCopies
OpenMPDependClauseKind getDependencyKind() const
Get dependency type.
static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool ForDup)
Emit initialization for private variables in task-based directives.
DiagnosticsEngine & getDiagnostics() const
static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn)
static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner)
virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D)
Emits OpenMP-specific function prolog.
static void emitOffloadingArraysArgument(CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg, llvm::Value *&PointersArrayArg, llvm::Value *&SizesArrayArg, llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info)
Emit the arguments to be passed to the runtime library based on the arrays of pointers, sizes and map types.
Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp)
Check if the combiner is a call to UDR combiner and if it is so return the UDR decl used for reductio...
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, unsigned Flags=0)
Emits object of ident_t type with info for source location.
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
CharUnits getPointerAlign() const
static IntegerLiteral * Create(const ASTContext &C, const llvm::APInt &V, QualType type, SourceLocation l)
Returns a new integer literal with value 'V' and type 'type'.
SmallVector< const Expr *, 4 > ReductionOps
This represents clause 'is_device_ptr' in the '#pragma omp ...' directives.
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
SmallVector< const Expr *, 4 > ReductionVars
virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS)
Checks if the variable has associated OMPAllocateDeclAttr attribute with the predefined allocator and...
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind)
Call the appropriate runtime routine to notify that we finished all the work with current loop...
virtual Decl * getCanonicalDecl()
Retrieves the "canonical" declaration of the given declaration.
static bool greaterCtxScore(const CompleteOMPContextSelectorData &LHS, const CompleteOMPContextSelectorData &RHS)
const CGFunctionInfo & arrangeGlobalDeclaration(GlobalDecl GD)
This represents clause 'from' in the '#pragma omp ...' directives.
LValue EmitLValueForField(LValue Base, const FieldDecl *Field)
const VarDecl * translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const override
Translates the native parameter of outlined function if this is required for target.
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
virtual bool emitDeclareVariant(GlobalDecl GD, bool IsForDefinition)
Emits the definition of the declare variant function.
OpenMP 4.0 [2.4, Array Sections].
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
ASTContext & getContext() const
void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr) override
Emits a critical region.
void emitTargetNumIterationsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Value *DeviceID, llvm::function_ref< llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter)
Emit code that pushes the trip count of loops associated with constructs 'target teams distribute' an...
Describes the capture of either a variable, or 'this', or variable-length array type.
const CodeGen::CGBlockInfo * BlockInfo
virtual Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const
Gets the address of the native argument basing on the address of the target-specific parameter...
CGBlockInfo - Information to generate a block literal.
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
LValueBaseInfo getBaseInfo() const
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
static llvm::Function * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Function *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
bool needCleanups(unsigned N)
Returns true if the private copy requires cleanups.
Class intended to support codegen of all kind of the reduction clauses.
llvm::FunctionCallee createRuntimeFunction(unsigned Function)
Returns specified OpenMP runtime function.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Expr * getCombiner()
Get combiner expression of the declare reduction construct.
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static chunked.
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
unsigned getNumLoops() const
Get number of loops associated with the clause.
This represents '#pragma omp requires...' directive.
virtual void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags, llvm::GlobalValue::LinkageTypes Linkage)
Creates offloading entry for the provided entry ID ID, address Addr, size Size, and flags Flags...
llvm::CallingConv::ID getRuntimeCC() const
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
This represents implicit clause 'depend' for the '#pragma omp task' directive.
virtual unsigned getDefaultFirstprivateAddressSpace() const
Returns default address space for the constant firstprivates, 0 by default.
KmpTaskTFields
Indexes of fields for type kmp_task_t.
void emitDeferredTargetDecls() const
Emit deferred declare target variables marked for deferred emission.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Required to resolve existing problems in the runtime.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr *> Vars, SourceLocation Loc)
Emit flush of the variables specified in 'omp flush' directive.
static void createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty, ArrayRef< llvm::Constant *> Data, T &Parent)
bool addPrivate(const VarDecl *LocalVD, const llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc) override
Emit code for 'taskwait' directive.
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
This represents one expression.
Allow any unmodeled side effect.
void loadOffloadInfoMetadata()
Loads all the offload entries information from the host IR metadata.
static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty)
Address getAddress(CodeGenFunction &CGF) const
static llvm::Value * emitReduceCombFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N, const Expr *ReductionOp, const Expr *LHS, const Expr *RHS, const Expr *PrivateRef)
Emits reduction combiner function:
const AnnotatedLine * Line
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top, if IgnoreCaptured is true.
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
const CGFunctionInfo & arrangeNullaryFunction()
A nullary function is a freestanding function of type 'void ()'.
std::string OMPHostIRFile
Name of the IR file that contains the result of the OpenMP target host code generation.
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
const T * castAs() const
Member-template castAs<specific type>.
llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts) override
Emits outlined function for the OpenMP task directive D.
llvm::BasicBlock * getBlock() const
unsigned getLine() const
Return the presumed line number of this location.
llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST) override
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD)
Checks if destructor function is required to be generated.
void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr *> CopyprivateVars, ArrayRef< const Expr *> DestExprs, ArrayRef< const Expr *> SrcExprs, ArrayRef< const Expr *> AssignmentOps) override
Emits a single region.
Represents a C++ destructor within a class.
VariableCaptureKind getCaptureKind() const
Determine the kind of capture.
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
QualType SavedKmpTaskloopTQTy
Saved kmp_task_t for taskloop-based directive.
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
VlaSizePair getVLASize(const VariableArrayType *vla)
Returns an LLVM value that corresponds to the size, in non-variably-sized elements, of a variable length array type, plus that largest non-variably-sized element type.
field_iterator field_end() const
llvm::PointerType * getType() const
Return the type of the pointer value.
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name) override
Creates artificial threadprivate variable with name Name and type VarType.
DeclContext * getDeclContext()
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause *> Clauses)
Mark the entry as a to declare target link.
llvm::FunctionCallee createDispatchNextFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_next_* runtime function for the specified size IVSize and sign IVSigned...
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr *> Vars, SourceLocation Loc) override
Emit flush of the variables specified in 'omp flush' directive.
This represents 'ordered' clause in the '#pragma omp ...' directive.
llvm::SmallDenseSet< CanonicalDeclPtr< const Decl > > NontemporalDeclsSet
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
llvm::IntegerType * Int32Ty
llvm::GlobalValue::LinkageTypes getLLVMLinkageVarDefinition(const VarDecl *VD, bool IsConstant)
Returns LLVM linkage for a declarator.
DefinitionKind hasDefinition(ASTContext &) const
Check whether this variable is defined in this translation unit.
TagDecl * getAsTagDecl() const
Retrieves the TagDecl that this type refers to, either because the type is a TagType or because it is...
Provides LLVM's BitmaskEnum facility to enumeration types declared in namespace clang.
SmallVector< const Expr *, 4 > FirstprivateVars
TBAAAccessInfo getTBAAInfo() const
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef< const Expr *> CopyprivateVars, ArrayRef< const Expr *> DestExprs, ArrayRef< const Expr *> SrcExprs, ArrayRef< const Expr *> AssignmentOps, SourceLocation Loc)
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
llvm::GlobalVariable::LinkageTypes getFunctionLinkage(GlobalDecl GD)
QualType getRecordType(const RecordDecl *Decl) const
Represents an unpacked "presumed" location which can be presented to the user.
void Emit(CodeGenFunction &CGF, Flags) override
Emit the cleanup.
virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Helper to emit outlined function for 'target' directive.
void registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, llvm::Constant *Addr, llvm::Constant *ID, OMPTargetRegionEntryKind Flags)
Register target region entry.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr) override
Emit a code for initialization of threadprivate variable.
llvm::Value * EmitCastToVoidPtr(llvm::Value *value)
Emit a cast to void* in the appropriate address space.
Allow UB that we can give a value, but not arbitrary unmodeled side effects.
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
Expr * getInitPriv()
Get Priv variable of the initializer.
const TargetInfo & getTarget() const
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
void SetCommonAttributes(GlobalDecl GD, llvm::GlobalValue *GV)
Set attributes which are common to any form of a global definition (alias, Objective-C method...
void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) override
Emit a taskgroup region.
virtual void emitProcBindClause(CodeGenFunction &CGF, llvm::omp::ProcBindKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
const Qualifiers & getQuals() const
fileinfo_iterator fileinfo_begin() const
Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc) override
Returns address of the threadprivate variable for the current thread.
const LangOptions & getLangOpts() const
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
ASTContext & getContext() const
bool isNull() const
Return true if this QualType doesn't point to a type yet.
static bool getAArch64PBV(QualType QT, ASTContext &C)
Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
GlobalDecl - represents a global declaration.
This file defines OpenMP AST classes for clauses.
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind. ...
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
void finishAndAddTo(AggregateBuilderBase &parent)
Given that this builder was created by beginning an array or struct component on the given parent bui...
CanQualType getCanonicalTypeUnqualified() const
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
The l-value was considered opaque, so the alignment was determined from a type.
RecordDecl * getDecl() const
const char * getFilename() const
Return the presumed filename of this location.
Address CreateConstGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = T* ...
virtual Address getAddrOfDeclareTargetVar(const VarDecl *VD)
Returns the address of the variable marked as declare target with link clause OR as declare target wi...
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp)
Emit reduction combiner.
Expr * getStrideVariable() const
bool UseOriginalIV
True if original lvalue for loop counter can be used in codegen (simd region or simd only mode) and n...
This represents 'num_teams' clause in the '#pragma omp ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
virtual void registerTargetGlobalVariable(const VarDecl *VD, llvm::Constant *Addr)
Checks if the provided global decl GD is a declare target variable and registers it when emitting cod...
CanProxy< U > castAs() const
This captures a statement into a function.
QualType getCanonicalType() const
decl_type * getPreviousDecl()
Return the previous declaration of this declaration or NULL if this is the first declaration.
llvm::Function * emitRequiresDirectiveRegFun()
Creates and returns a registration function for when at least one requires directives was used in the...
static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind)
Returns default flags for the barriers depending on the directive, for which this barier is going to ...
unsigned getColumn() const
Return the presumed column number of this location.
static with chunk adjustment (e.g., simd)
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
pushDestroy - Push the standard destructor for the given type as at least a normal cleanup...
void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancel' construct.
Encodes a location in the source.
static const OMPExecutableDirective * getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D)
Check for inner distribute directive.
llvm::FunctionCallee createDispatchFiniFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_fini_* runtime function for the specified size IVSize and sign IVSigned...
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
llvm::Value * MapTypesArray
The array of map types passed to the runtime library.
This represents '#pragma omp declare reduction ...' directive.
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr *> CopyprivateVars, ArrayRef< const Expr *> DestExprs, ArrayRef< const Expr *> SrcExprs, ArrayRef< const Expr *> AssignmentOps)
Emits a single region.
void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value *> CapturedVars, const Expr *IfCond) override
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
Address CreateConstInBoundsGEP2_32(Address Addr, unsigned Idx0, unsigned Idx1, const llvm::Twine &Name="")
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams...
llvm::OpenMPIRBuilder * getOpenMPIRBuilder()
Return a pointer to the configured OpenMPIRBuilder, if any.
void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false) override
Emit an implicit/explicit barrier for OpenMP threads.
This is a basic class for representing single OpenMP executable directive.
bool emitTargetGlobalVariable(GlobalDecl GD) override
Emit the global variable if it is a valid device global variable.
Lower bound for 'ordered' versions.
ASTContext & getASTContext() const LLVM_READONLY
llvm::Value * getPointer(CodeGenFunction &CGF) const
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
const Decl * getDecl() const
bool capturesVariable() const
Determine whether this capture handles a variable (by reference).
void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef< Expr *> NumIterations) override
Emit initialization for doacross loop nesting support.
Address CreateConstArrayGEP(Address Addr, uint64_t Index, const llvm::Twine &Name="")
Given addr = [n x T]* ...
Set if the nonmonotonic schedule modifier was present.
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
OpenMPLinearClauseKind Modifier
Modifier of 'linear' clause.
Target region entries info.
const ParmVarDecl * getParamDecl(unsigned i) const
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
Device global variable entries info.
void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) override
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value *> CapturedVars, const Expr *IfCond)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
This is a basic class for representing single OpenMP clause.
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param Data Additional data for task generation like final list of privates etc *void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) override
Call the appropriate runtime routine to initialize it before start of loop.
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc)
Gets the address of the global copy used for lastprivate conditional update, if any.
Maps the expression for the lastprivate variable to the global copy used to store new value because o...
bool isAnyPointerType() const
This declaration is only a declaration.
unsigned size() const
Return number of entries defined so far.
virtual void Enter(CodeGenFunction &CGF)
llvm::FunctionCallee createDispatchInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_init_* runtime function for the specified size IVSize and sign IVSigned...
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) override
Emit the target data mapping code associated with D.
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after...
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
MangleContext & getMangleContext()
Gets the mangle context.
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy)
Emit an aggregate assignment.
CharUnits getNaturalTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, bool forPointeeType=false)
llvm::Value * NewTaskNewTaskTTy
bool hasTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum) const
Return true if a target region entry with the provided information exists.
virtual void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
virtual void emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef< llvm::Value *> Args=llvm::None) const
Emits call of the outlined function with the provided arguments, translating these arguments to corre...
const Expr * getRefExpr(unsigned N) const
Returns the base declaration of the reduction item.
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
virtual void mangleTypeName(QualType T, raw_ostream &)=0
Generates a unique string for an externally visible type for use with TBAA or type uniquing...
void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) override
Required to resolve existing problems in the runtime.
Expr * getInitOrig()
Get Orig variable of the initializer.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
void registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr, CharUnits VarSize, OMPTargetGlobalVarEntryKind Flags, llvm::GlobalValue::LinkageTypes Linkage)
Register device global variable entry.
ConstantAddress GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr)
Returns a pointer to a character array containing the literal and a terminating '\0' character...
llvm::Constant * GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition=NotForDefinition)
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads) override
Emit an ordered region.
void addUsedGlobal(llvm::GlobalValue *GV)
Add a global to a list to be added to the llvm.used metadata.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
static void emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, const llvm::APSInt &VLENVal, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State)
void setAction(PrePostActionTy &Action) const
CGFunctionInfo - Class to encapsulate the information about a function definition.
This class organizes the cross-function state that is used while generating LLVM code.
QualType withRestrict() const
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
static const VarDecl * getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE)
StructBuilder beginStruct(llvm::StructType *ty=nullptr)
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
llvm::GlobalValue * GetGlobalValue(StringRef Ref)
Dataflow Directional Tag Classes.
static ApplyDebugLocation CreateDefaultArtificial(CodeGenFunction &CGF, SourceLocation TemporaryLocation)
Apply TemporaryLocation if it is valid.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
LValue EmitLoadOfReferenceLValue(LValue RefLVal)
static llvm::Value * emitReduceInitFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction initializer function:
This represents 'device' clause in the '#pragma omp ...' directive.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool emitTargetFunctions(GlobalDecl GD) override
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param Data Additional data for task generation like final list of privates etc *virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
EvalResult is a struct with detailed info about an evaluated expression.
NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S)
static std::string getName(const CallEvent &Call)
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
OpenMPScheduleClauseModifier M1
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
const Expr * getInit() const
llvm::Constant * getPointer() const
void initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, unsigned Order)
Initialize target region entry.
std::unique_ptr< DiagnosticConsumer > create(StringRef OutputFile, DiagnosticOptions *Diags, bool MergeChildRecords=false)
Returns a DiagnosticConsumer that serializes diagnostics to a bitcode file.
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::IntegerType * IntPtrTy
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name. ...
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::Module & getModule() const
void emitUDMapperArrayInitOrDel(CodeGenFunction &MapperCGF, llvm::Value *Handle, llvm::Value *BasePtr, llvm::Value *Ptr, llvm::Value *Size, llvm::Value *MapType, CharUnits ElementSize, llvm::BasicBlock *ExitBB, bool IsInit)
Emit the array initialization or deletion portion for user-defined mapper code generation.
Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...
Not really used in Fortran any more.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
llvm::StructType * ConvertRecordDeclType(const RecordDecl *TD)
ConvertRecordDeclType - Lay out a tagged decl type like struct or union.
virtual bool isDefaultLocationConstant() const
Check if the default location must be constant.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value *> CapturedVars) override
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
JumpDest ReturnBlock
ReturnBlock - Unified return block.
OffloadEntriesInfoManagerTy OffloadEntriesInfoManager
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
static bool isStrictSubset(const CompleteOMPContextSelectorData &LHS, const CompleteOMPContextSelectorData &RHS)
bool capturesVariableByCopy() const
Determine whether this capture handles a variable by copy.
Class that represents a component of a mappable expression.
static void emitForStaticInitCall(CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, const CGOpenMPRuntime::StaticRTInput &Values)
void EmitAutoVarCleanups(const AutoVarEmission &emission)
llvm::DenseSet< CanonicalDeclPtr< const Decl > > AlreadyEmittedTargetDecls
List of the emitted declarations.
API for captured statement code generation.
virtual bool emitDeclareTargetVarDefinition(const VarDecl *VD, llvm::GlobalVariable *Addr, bool PerformInit)
Emit a code for initialization of declare target variable.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef< llvm::Value *> CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values)
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
virtual StringRef getOutlinedHelperName() const
Get the function name of an outlined region.
static bool classof(const OMPClause *T)
static bool matchesContext(CodeGenModule &CGM, const CompleteOMPContextSelectorData &ContextData)
bool HasEmittedDeclareTargetRegion
Flag for keeping track of weather a device routine has been emitted.
ArraySubscriptExpr - [C99 6.5.2.1] Array Subscripting.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
This file defines OpenMP AST classes for executable directives and clauses.
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
CodeGenTypes & getTypes() const
StructBuilder beginStruct(llvm::StructType *structTy=nullptr)
virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS)
Checks if the provided LVal is lastprivate conditional and emits the code to update the value of the ...
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
CleanupTy(PrePostActionTy *Action)
llvm::Type * getElementType() const
Return the type of the values stored in this address.
void initializeDeviceGlobalVarEntryInfo(StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order)
Initialize device global variable entry.
static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef< PrivateDataTy > Privates)
Check if duplication function is required for taskloops.
llvm::Value * EmitCheckedInBoundsGEP(llvm::Value *Ptr, ArrayRef< llvm::Value *> IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc, unsigned &DeviceID, unsigned &FileID, unsigned &LineNum)
Obtain information that uniquely identifies a target entry.
llvm::PointerType * Int8PtrTy
Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal) override
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
OpenMPLocationFlags
Values for bit flags used in the ident_t to describe the fields.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Expr * getNumIterations() const
llvm::StringRef getName() const
Return the IR name of the pointer value.
llvm::StructType * getLLVMType() const
Return the "complete object" LLVM type associated with this record.
Base for LValueReferenceType and RValueReferenceType.
bool hasSameType(QualType T1, QualType T2) const
Determine whether the given types T1 and T2 are equivalent.
Entity that registers the offloading constants that were emitted so far.
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
void emitCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee Callee, ArrayRef< llvm::Value *> Args=llvm::None) const
Emits Callee function call with arguments Args with location Loc.
StringRef getMangledName(GlobalDecl GD)
llvm::Constant * getOrCreateInternalVariable(llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace=0)
Gets (if variable with the given name already exist) or creates internal global variable with the spe...
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArrayType::ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type...
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void addDecl(Decl *D)
Add the declaration D into this context.
FieldDecl * LambdaThisCaptureField
llvm::Constant * registerTargetFirstprivateCopy(CodeGenFunction &CGF, const VarDecl *VD)
Registers provided target firstprivate variable as global on the target.
void getCaptureFields(llvm::DenseMap< const VarDecl *, FieldDecl *> &Captures, FieldDecl *&ThisCapture) const
For a closure type, retrieve the mapping from captured variables and this to the non-static data memb...
Mark the entry as target region.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
SourceManager & getSourceManager()
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
llvm::iterator_range< specific_attr_iterator< T > > specific_attrs() const
Mark the entry as a global destructor.
Address LoadCXXThisAddress()
virtual void adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF, const OMPExecutableDirective &D) const
Adjust some parameters for the target-based directives, like addresses of the variables captured by r...
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
Lower bound for default (unordered) versions.
void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc) override
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams...
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
MemberExpr - [C99 6.5.2.3] Structure and Union Members.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr *> Privates, ArrayRef< const Expr *> LHSExprs, ArrayRef< const Expr *> RHSExprs, ArrayRef< const Expr *> ReductionOps, ReductionOptionsTy Options)
Emit a code for reduction clause.
This represents 'nowait' clause in the '#pragma omp ...' directive.
static bool isTrivial(ASTContext &Ctx, const Expr *E)
Checks if the expression is constant or does not have non-trivial function calls. ...
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
Represents a C++ struct/union/class.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
llvm::Function * CreateGlobalInitOrDestructFunction(llvm::FunctionType *ty, const Twine &name, const CGFunctionInfo &FI, SourceLocation Loc=SourceLocation(), bool TLS=false)
llvm::Value * BasePointersArray
The array of base pointer passed to the runtime library.
static QualType getBaseOriginalType(const Expr *Base)
Return original type of the base expression for array section.
static llvm::Value * emitReduceFiniFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N)
Emits reduction finalizer function:
~DisableAutoDeclareTargetRAII()
OpenMPOffloadingReservedDeviceIDs
llvm::Type * ConvertType(QualType T)
bool isTLSSupported() const
Whether the target supports thread-local storage.
void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) override
Emit code for doacross ordered directive with 'depend' clause.
Privates[]
Gets the list of initial values for linear variables.
OpenMPMapClauseKind
OpenMP mapping kind for 'map' clause.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
virtual llvm::Function * emitTeamsOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP teams directive D.
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
static bool checkContext(const OMPContextSelectorData &Data, Arguments... Params)
Checks current context and returns true if it matches the context selector.
A helper class of ConstantInitBuilder, used for building constant struct initializers.
void popTerminate()
Pops a terminate handler off the stack.
ASTImporterLookupTable & LT
CharUnits getNonVirtualSize() const
getNonVirtualSize - Get the non-virtual size (in chars) of an object, which is the size of the object...
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
unsigned getVirtualBaseIndex(const CXXRecordDecl *base) const
Return the LLVM field index corresponding to the given virtual base.
Entry is a declare target variable.
This represents '#pragma omp declare mapper ...' directive.
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) override
Emit outilined function for 'target' directive.
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool WithLastIter)
Emit task_dup function (for initialization of private/firstprivate/lastprivate vars and last_iter fla...
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues)
Call the appropriate runtime routine to initialize it before start of loop.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
llvm::Function * emitParallelOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) override
Emits outlined function for the specified OpenMP parallel directive D.
llvm::MapVector< CanonicalDeclPtr< const FunctionDecl >, std::pair< GlobalDecl, GlobalDecl > > DeferredVariantFunction
Mapping of the original functions to their variants and original global decl.
CGCapturedStmtInfo * CapturedStmtInfo
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
CGCXXABI & getCXXABI() const
Expr * getCombinerIn()
Get In variable of the combiner.
const VariableArrayType * getAsVariableArrayType(QualType T) const
Mark the entry as a global constructor.
decl_type * getMostRecentDecl()
Returns the most recent (re)declaration of this declaration.
bool isEvaluatable(const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
isEvaluatable - Call EvaluateAsRValue to see if this expression can be constant folded without side-e...
void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy, AggValueSlot::Overlap_t MayOverlap, bool isVolatile=false)
EmitAggregateCopy - Emit an aggregate copy.
A reference to a declared variable, function, enum, etc.
QualType getIntPtrType() const
Return a type compatible with "intptr_t" (C99 7.18.1.4), as defined by the target.
static RValue get(llvm::Value *V)
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values)
Call the appropriate runtime routine to initialize it before start of loop.
llvm::Value * emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr *> LHSExprs, ArrayRef< const Expr *> RHSExprs, const OMPTaskDataTy &Data) override
Emit a code for initialization of task reduction clause.
bool isPointerType() const
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
ParamKindTy
Kind of parameter in a function with 'declare simd' directive.
bool HasRequiresUnifiedSharedMemory
Flag for keeping track of weather a requires unified_shared_memory directive is present.
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
QualType KmpDimTy
struct kmp_dim { // loop bounds info casted to kmp_int64 kmp_int64 lo; // lower kmp_int64 up; // uppe...
void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) override
Call the appropriate runtime routine to notify that we finished all the work with current loop...
An l-value expression is a reference to an object with independent storage.
bool isFloatingType() const
static RValue getAggregate(Address addr, bool isVolatile=false)
bool isOpenMPLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a directive with an associated loop construct.
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param Data Additional data for task generation like final list of privates etc *void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) override
LValue - This represents an lvalue references.
virtual unsigned getDefaultLocationReserved2Flags() const
Returns additional flags that can be stored in reserved_2 field of the default location.
Information for lazily generating a cleanup.
QualType SavedKmpTaskTQTy
Saved kmp_task_t for task directive.
virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal)
Get the address of void * type of the privatue copy of the reduction item specified by the SharedLVal...
Represents a C array with a specified size that is not an integer-constant-expression.
bool isConstant(const ASTContext &Ctx) const
virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const
Choose default schedule type and chunk value for the schedule clause.
const LangOptions & getLangOpts() const
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::StringSet DeclareTargetWithDefinition
Set of declare target variables with the generated initializer.
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
virtual llvm::Function * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
Expr * getCombinerOut()
Get Out variable of the combiner.
unsigned getNumParams() const
Return the number of parameters this function must have based on its FunctionType.
const CGRecordLayout & getCGRecordLayout(const RecordDecl *)
getCGRecordLayout - Return record layout info for the given record decl.
This represents clause 'nontemporal' in the '#pragma omp ...' directives.
base_class_range vbases()
This class handles loading and caching of source files into memory.
void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) override
Emit code for 'cancellation point' construct.
void EmitGlobal(GlobalDecl D)
Emit code for a single global function or var decl.
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
bool capturesVariableArrayType() const
Determine whether this capture handles a variable-length array type.
bool isLocalVarDeclOrParm() const
Similar to isLocalVarDecl but also includes parameters.
Attr - This represents one attribute.
SmallVector< const Expr *, 4 > FirstprivateInits
SourceLocation getLocation() const
This represents clause 'use_device_ptr' in the '#pragma omp ...' directives.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
bool isExternallyVisible() const
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object...
llvm::Value * SizesArray
The array of sizes passed to the runtime library.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool Privatize()
Privatizes local variables previously registered as private.
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc)
Emit code for 'taskwait' directive.
unsigned getLLVMFieldNo(const FieldDecl *FD) const
Return llvm::StructType element number that corresponds to the field FD.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.