clang 8.0.0

CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit OpenMP nodes as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "CGCleanup.h"
15 #include "CGOpenMPRuntime.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/Stmt.h"
20 #include "clang/AST/StmtOpenMP.h"
21 #include "clang/AST/DeclOpenMP.h"
22 #include "llvm/IR/CallSite.h"
23 using namespace clang;
24 using namespace CodeGen;
25
26 namespace {
27 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
28 /// for captured expressions.
29 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
30 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
31 for (const auto *C : S.clauses()) {
32 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
33 if (const auto *PreInit =
34 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
35 for (const auto *I : PreInit->decls()) {
36 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
37 CGF.EmitVarDecl(cast<VarDecl>(*I));
38 } else {
39 CodeGenFunction::AutoVarEmission Emission =
40 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
41 CGF.EmitAutoVarCleanups(Emission);
42 }
43 }
44 }
45 }
46 }
47 }
48 CodeGenFunction::OMPPrivateScope InlinedShareds;
49
50 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
51 return CGF.LambdaCaptureFields.lookup(VD) ||
52 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
53 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
54 }
55
56 public:
57 OMPLexicalScope(
58 CodeGenFunction &CGF, const OMPExecutableDirective &S,
59 const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
60 const bool EmitPreInitStmt = true)
61 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
62 InlinedShareds(CGF) {
63 if (EmitPreInitStmt)
64 emitPreInitStmt(CGF, S);
65 if (!CapturedRegion.hasValue())
66 return;
67 assert(S.hasAssociatedStmt() &&
68 "Expected associated statement for inlined directive.");
69 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
70 for (const auto &C : CS->captures()) {
71 if (C.capturesVariable() || C.capturesVariableByCopy()) {
72 auto *VD = C.getCapturedVar();
73 assert(VD == VD->getCanonicalDecl() &&
74 "Canonical decl must be captured.");
75 DeclRefExpr DRE(
76 CGF.getContext(), const_cast<VarDecl *>(VD),
77 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
78 InlinedShareds.isGlobalVarCaptured(VD)),
79 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
80 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
81 return CGF.EmitLValue(&DRE).getAddress();
82 });
83 }
84 }
85 (void)InlinedShareds.Privatize();
86 }
87 };
88
89 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
90 /// for captured expressions.
91 class OMPParallelScope final : public OMPLexicalScope {
92 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
93 OpenMPDirectiveKind Kind = S.getDirectiveKind();
94 return !(isOpenMPTargetExecutionDirective(Kind) ||
95 isOpenMPLoopBoundSharingDirective(Kind)) &&
96 isOpenMPParallelDirective(Kind);
97 }
98
99 public:
100 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
101 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
102 EmitPreInitStmt(S)) {}
103 };
104
105 /// Lexical scope for OpenMP teams construct, that handles correct codegen
106 /// for captured expressions.
107 class OMPTeamsScope final : public OMPLexicalScope {
108 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
109 OpenMPDirectiveKind Kind = S.getDirectiveKind();
110 return !isOpenMPTargetExecutionDirective(Kind) &&
111 isOpenMPTeamsDirective(Kind);
112 }
113
114 public:
115 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
116 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
117 EmitPreInitStmt(S)) {}
118 };
119
120 /// Private scope for OpenMP loop-based directives, that supports capturing
121 /// of used expression from loop statement.
122 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
123 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
124 CodeGenFunction::OMPMapVars PreCondVars;
125 for (const auto *E : S.counters()) {
126 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
127 (void)PreCondVars.setVarAddr(
128 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
129 }
130 (void)PreCondVars.apply(CGF);
131 if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
132 for (const auto *I : PreInits->decls())
133 CGF.EmitVarDecl(cast<VarDecl>(*I));
134 }
135 PreCondVars.restore(CGF);
136 }
137
138 public:
139 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
140 : CodeGenFunction::RunCleanupsScope(CGF) {
141 emitPreInitStmt(CGF, S);
142 }
143 };
144
145 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
146 CodeGenFunction::OMPPrivateScope InlinedShareds;
147
148 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
149 return CGF.LambdaCaptureFields.lookup(VD) ||
150 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
151 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
152 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
153 }
154
155 public:
156 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
157 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
158 InlinedShareds(CGF) {
159 for (const auto *C : S.clauses()) {
160 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
161 if (const auto *PreInit =
162 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
163 for (const auto *I : PreInit->decls()) {
164 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
165 CGF.EmitVarDecl(cast<VarDecl>(*I));
166 } else {
167 CodeGenFunction::AutoVarEmission Emission =
168 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
169 CGF.EmitAutoVarCleanups(Emission);
170 }
171 }
172 }
173 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
174 for (const Expr *E : UDP->varlists()) {
175 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
176 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
177 CGF.EmitVarDecl(*OED);
178 }
179 }
180 }
181 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
182 CGF.EmitOMPPrivateClause(S, InlinedShareds);
183 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
184 if (const Expr *E = TG->getReductionRef())
185 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
186 }
187 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
188 while (CS) {
189 for (auto &C : CS->captures()) {
190 if (C.capturesVariable() || C.capturesVariableByCopy()) {
191 auto *VD = C.getCapturedVar();
192 assert(VD == VD->getCanonicalDecl() &&
193 "Canonical decl must be captured.");
194 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
195 isCapturedVar(CGF, VD) ||
196 (CGF.CapturedStmtInfo &&
197 InlinedShareds.isGlobalVarCaptured(VD)),
198 VD->getType().getNonReferenceType(), VK_LValue,
199 C.getLocation());
200 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
201 return CGF.EmitLValue(&DRE).getAddress();
202 });
203 }
204 }
205 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
206 }
207 (void)InlinedShareds.Privatize();
208 }
209 };
210
211 } // namespace
212
213 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
214 const OMPExecutableDirective &S,
215 const RegionCodeGenTy &CodeGen);
216
217 LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
218 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
219 if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
220 OrigVD = OrigVD->getCanonicalDecl();
221 bool IsCaptured =
222 LambdaCaptureFields.lookup(OrigVD) ||
223 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
224 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
225 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
226 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
227 return EmitLValue(&DRE);
228 }
229 }
230 return EmitLValue(E);
231 }
232
233 llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
234 ASTContext &C = getContext();
235 llvm::Value *Size = nullptr;
236 auto SizeInChars = C.getTypeSizeInChars(Ty);
237 if (SizeInChars.isZero()) {
238 // getTypeSizeInChars() returns 0 for a VLA.
239 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
240 VlaSizePair VlaSize = getVLASize(VAT);
241 Ty = VlaSize.Type;
242 Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
243 : VlaSize.NumElts;
244 }
245 SizeInChars = C.getTypeSizeInChars(Ty);
246 if (SizeInChars.isZero())
247 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
248 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
249 }
250 return CGM.getSize(SizeInChars);
251 }
252
253 void CodeGenFunction::GenerateOpenMPCapturedVars(
254 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
255 const RecordDecl *RD = S.getCapturedRecordDecl();
256 auto CurField = RD->field_begin();
257 auto CurCap = S.captures().begin();
258 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
259 E = S.capture_init_end();
260 I != E; ++I, ++CurField, ++CurCap) {
261 if (CurField->hasCapturedVLAType()) {
262 const VariableArrayType *VAT = CurField->getCapturedVLAType();
263 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
264 CapturedVars.push_back(Val);
265 } else if (CurCap->capturesThis()) {
266 CapturedVars.push_back(CXXThisValue);
267 } else if (CurCap->capturesVariableByCopy()) {
268 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
269
270 // If the field is not a pointer, we need to save the actual value
271 // and load it as a void pointer.
272 if (!CurField->getType()->isAnyPointerType()) {
273 ASTContext &Ctx = getContext();
274 Address DstAddr = CreateMemTemp(
275 Ctx.getUIntPtrType(),
276 Twine(CurCap->getCapturedVar()->getName(), ".casted"));
277 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
278
279 llvm::Value *SrcAddrVal = EmitScalarConversion(
280 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
281 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
282 LValue SrcLV =
283 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
284
285 // Store the value using the source type pointer.
286 EmitStoreThroughLValue(RValue::get(CV), SrcLV);
287
288 // Load the value using the destination type pointer.
289 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
290 }
291 CapturedVars.push_back(CV);
292 } else {
293 assert(CurCap->capturesVariable() && "Expected capture by reference.");
294 CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
295 }
296 }
297 }
298
299 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
300 QualType DstType, StringRef Name,
301 LValue AddrLV,
302 bool isReferenceType = false) {
303 ASTContext &Ctx = CGF.getContext();
304
305 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
306 AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
307 Ctx.getPointerType(DstType), Loc);
308 Address TmpAddr =
309 CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
310 .getAddress();
311
312 // If we are dealing with references we need to return the address of the
313 // reference instead of the reference of the value.
314 if (isReferenceType) {
315 QualType RefType = Ctx.getLValueReferenceType(DstType);
316 llvm::Value *RefVal = TmpAddr.getPointer();
317 TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref"));
318 LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
319 CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true);
320 }
321
322 return TmpAddr;
323 }
324
325 static QualType getCanonicalParamType(ASTContext &C, QualType T) {
326 if (T->isLValueReferenceType())
327 return C.getLValueReferenceType(
328 getCanonicalParamType(C, T.getNonReferenceType()),
329 /*SpelledAsLValue=*/false);
330 if (T->isPointerType())
331 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
332 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
333 if (const auto *VLA = dyn_cast<VariableArrayType>(A))
334 return getCanonicalParamType(C, VLA->getElementType());
335 if (!A->isVariablyModifiedType())
336 return C.getCanonicalType(T);
337 }
338 return C.getCanonicalParamType(T);
339 }
340
341 namespace {
342 /// Contains required data for proper outlined function codegen.
343 struct FunctionOptions {
344 /// Captured statement for which the function is generated.
345 const CapturedStmt *S = nullptr;
346 /// true if cast to/from UIntPtr is required for variables captured by
347 /// value.
348 const bool UIntPtrCastRequired = true;
349 /// true if only casted arguments must be registered as local args or VLA
350 /// sizes.
351 const bool RegisterCastedArgsOnly = false;
352 /// Name of the generated function.
353 const StringRef FunctionName;
354 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
355 bool RegisterCastedArgsOnly,
356 StringRef FunctionName)
357 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
358 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
359 FunctionName(FunctionName) {}
360 };
361 }
362
363 static llvm::Function *emitOutlinedFunctionPrologue(
364 CodeGenFunction &CGF, FunctionArgList &Args,
365 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
366 &LocalAddrs,
367 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
368 &VLASizes,
369 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
370 const CapturedDecl *CD = FO.S->getCapturedDecl();
371 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
372 assert(CD->hasBody() && "missing CapturedDecl body");
373
374 CXXThisValue = nullptr;
375 // Build the argument list.
376 CodeGenModule &CGM = CGF.CGM;
377 ASTContext &Ctx = CGM.getContext();
378 FunctionArgList TargetArgs;
379 Args.append(CD->param_begin(),
380 std::next(CD->param_begin(), CD->getContextParamPosition()));
381 TargetArgs.append(
382 CD->param_begin(),
383 std::next(CD->param_begin(), CD->getContextParamPosition()));
384 auto I = FO.S->captures().begin();
385 FunctionDecl *DebugFunctionDecl = nullptr;
386 if (!FO.UIntPtrCastRequired) {
387 FunctionProtoType::ExtProtoInfo EPI;
388 QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI);
389 DebugFunctionDecl = FunctionDecl::Create(
390 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
391 SourceLocation(), DeclarationName(), FunctionTy,
392 Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
393 /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
394 }
395 for (const FieldDecl *FD : RD->fields()) {
396 QualType ArgType = FD->getType();
397 IdentifierInfo *II = nullptr;
398 VarDecl *CapVar = nullptr;
399
400 // If this is a capture by copy and the type is not a pointer, the outlined
401 // function argument type should be uintptr and the value properly casted to
402 // uintptr. This is necessary given that the runtime library is only able to
403 // deal with pointers. We can pass in the same way the VLA type sizes to the
404 // outlined function.
405 if (FO.UIntPtrCastRequired &&
406 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
407 I->capturesVariableArrayType()))
408 ArgType = Ctx.getUIntPtrType();
409
410 if (I->capturesVariable() || I->capturesVariableByCopy()) {
411 CapVar = I->getCapturedVar();
412 II = CapVar->getIdentifier();
413 } else if (I->capturesThis()) {
414 II = &Ctx.Idents.get("this");
415 } else {
416 assert(I->capturesVariableArrayType());
417 II = &Ctx.Idents.get("vla");
418 }
419 if (ArgType->isVariablyModifiedType())
420 ArgType = getCanonicalParamType(Ctx, ArgType);
421 VarDecl *Arg;
422 if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
423 Arg = ParmVarDecl::Create(
424 Ctx, DebugFunctionDecl,
425 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
426 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
427 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
428 } else {
429 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
430 II, ArgType, ImplicitParamDecl::Other);
431 }
432 Args.emplace_back(Arg);
433 // Do not cast arguments if we emit function with non-original types.
434 TargetArgs.emplace_back(
435 FO.UIntPtrCastRequired
436 ? Arg
437 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
438 ++I;
439 }
440 Args.append(
441 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
442 CD->param_end());
443 TargetArgs.append(
444 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
445 CD->param_end());
446
447 // Create the function declaration.
448 const CGFunctionInfo &FuncInfo =
449 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
450 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
451
452 auto *F =
453 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
454 FO.FunctionName, &CGM.getModule());
455 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
456 if (CD->isNothrow())
457 F->setDoesNotThrow();
458 F->setDoesNotRecurse();
459
460 // Generate the function.
461 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
462 FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
463 unsigned Cnt = CD->getContextParamPosition();
464 I = FO.S->captures().begin();
465 for (const FieldDecl *FD : RD->fields()) {
466 // Do not map arguments if we emit function with non-original types.
467 Address LocalAddr(Address::invalid());
468 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
469 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
470 TargetArgs[Cnt]);
471 } else {
472 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
473 }
474 // If we are capturing a pointer by copy we don't need to do anything, just
475 // use the value that we get from the arguments.
476 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
477 const VarDecl *CurVD = I->getCapturedVar();
478 // If the variable is a reference we need to materialize it here.
479 if (CurVD->getType()->isReferenceType()) {
480 Address RefAddr = CGF.CreateMemTemp(
481 CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
482 CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
483 /*Volatile=*/false, CurVD->getType());
484 LocalAddr = RefAddr;
485 }
486 if (!FO.RegisterCastedArgsOnly)
487 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
488 ++Cnt;
489 ++I;
490 continue;
491 }
492
493 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
494 AlignmentSource::Decl);
495 if (FD->hasCapturedVLAType()) {
496 if (FO.UIntPtrCastRequired) {
497 ArgLVal = CGF.MakeAddrLValue(
498 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
499 Args[Cnt]->getName(), ArgLVal),
500 FD->getType(), AlignmentSource::Decl);
501 }
502 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
503 const VariableArrayType *VAT = FD->getCapturedVLAType();
504 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
505 } else if (I->capturesVariable()) {
506 const VarDecl *Var = I->getCapturedVar();
507 QualType VarTy = Var->getType();
508 Address ArgAddr = ArgLVal.getAddress();
509 if (!VarTy->isReferenceType()) {
510 if (ArgLVal.getType()->isLValueReferenceType()) {
511 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
512 } else if (!VarTy->isVariablyModifiedType() ||
513 !VarTy->isPointerType()) {
514 assert(ArgLVal.getType()->isPointerType());
515 ArgAddr = CGF.EmitLoadOfPointer(
516 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
517 }
518 }
519 if (!FO.RegisterCastedArgsOnly) {
520 LocalAddrs.insert(
521 {Args[Cnt],
522 {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
523 }
524 } else if (I->capturesVariableByCopy()) {
525 assert(!FD->getType()->isAnyPointerType() &&
526 "Not expecting a captured pointer.");
527 const VarDecl *Var = I->getCapturedVar();
528 QualType VarTy = Var->getType();
529 LocalAddrs.insert(
530 {Args[Cnt],
531 {Var, FO.UIntPtrCastRequired
532 ? castValueFromUintptr(CGF, I->getLocation(),
533 FD->getType(), Args[Cnt]->getName(),
534 ArgLVal, VarTy->isReferenceType())
535 : ArgLVal.getAddress()}});
536 } else {
537 // If 'this' is captured, load it into CXXThisValue.
538 assert(I->capturesThis());
539 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
540 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
541 }
542 ++Cnt;
543 ++I;
544 }
545
546 return F;
547 }
548
549 llvm::Function *
550 CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) {
551 assert(
552 CapturedStmtInfo &&
553 "CapturedStmtInfo should be set when generating the captured function");
554 const CapturedDecl *CD = S.getCapturedDecl();
555 // Build the argument list.
556 bool NeedWrapperFunction =
557 getDebugInfo() &&
558 CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
559 FunctionArgList Args;
560 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
561 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
562 SmallString<256> Buffer;
563 llvm::raw_svector_ostream Out(Buffer);
564 Out << CapturedStmtInfo->getHelperName();
565 if (NeedWrapperFunction)
566 Out << "_debug__";
567 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
568 Out.str());
569 llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
570 VLASizes, CXXThisValue, FO);
571 for (const auto &LocalAddrPair : LocalAddrs) {
572 if (LocalAddrPair.second.first) {
573 setAddrOfLocalVar(LocalAddrPair.second.first,
574 LocalAddrPair.second.second);
575 }
576 }
577 for (const auto &VLASizePair : VLASizes)
578 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
579 PGO.assignRegionCounters(GlobalDecl(CD), F);
580 CapturedStmtInfo->EmitBody(*this, CD->getBody());
581 FinishFunction(CD->getBodyRBrace());
582 if (!NeedWrapperFunction)
583 return F;
584
585 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
586 /*RegisterCastedArgsOnly=*/true,
587 CapturedStmtInfo->getHelperName());
588 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
589 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
590 Args.clear();
591 LocalAddrs.clear();
592 VLASizes.clear();
593 llvm::Function *WrapperF =
594 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
595 WrapperCGF.CXXThisValue, WrapperFO);
596 llvm::SmallVector<llvm::Value *, 4> CallArgs;
597 for (const auto *Arg : Args) {
598 llvm::Value *CallArg;
599 auto I = LocalAddrs.find(Arg);
600 if (I != LocalAddrs.end()) {
601 LValue LV = WrapperCGF.MakeAddrLValue(
602 I->second.second,
603 I->second.first ? I->second.first->getType() : Arg->getType(),
604 AlignmentSource::Decl);
605 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
606 } else {
607 auto EI = VLASizes.find(Arg);
608 if (EI != VLASizes.end()) {
609 CallArg = EI->second.second;
610 } else {
611 LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
612 Arg->getType(),
613 AlignmentSource::Decl);
614 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
615 }
616 }
617 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
618 }
619 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
620 F, CallArgs);
621 WrapperCGF.FinishFunction();
622 return WrapperF;
623 }
624
625 //===----------------------------------------------------------------------===//
626 // OpenMP Directive Emission
627 //===----------------------------------------------------------------------===//
628 void CodeGenFunction::EmitOMPAggregateAssign(
629 Address DestAddr, Address SrcAddr, QualType OriginalType,
630 const llvm::function_ref<void(Address, Address)> CopyGen) {
631 // Perform element-by-element initialization.
632 QualType ElementTy;
633
634 // Drill down to the base element type on both arrays.
635 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
636 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
637 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
638
639 llvm::Value *SrcBegin = SrcAddr.getPointer();
640 llvm::Value *DestBegin = DestAddr.getPointer();
641 // Cast from pointer to array type to pointer to single element.
642 llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
643 // The basic structure here is a while-do loop.
644 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
645 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
646 llvm::Value *IsEmpty =
647 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
648 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
649
650 // Enter the loop body, making that address the current address.
651 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
652 EmitBlock(BodyBB);
653
654 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
655
656 llvm::PHINode *SrcElementPHI =
657 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
658 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
659 Address SrcElementCurrent =
660 Address(SrcElementPHI,
661 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
662
663 llvm::PHINode *DestElementPHI =
664 Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
665 DestElementPHI->addIncoming(DestBegin, EntryBB);
666 Address DestElementCurrent =
667 Address(DestElementPHI,
668 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
669
670 // Emit copy.
671 CopyGen(DestElementCurrent, SrcElementCurrent);
672
673 // Shift the address forward by one element.
674 llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
675 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
676 llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
677 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
678 // Check whether we've reached the end.
679 llvm::Value *Done =
680 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
681 Builder.CreateCondBr(Done, DoneBB, BodyBB);
682 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
683 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
684
685 // Done.
686 EmitBlock(DoneBB, /*IsFinished=*/true);
687 }
688
689 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
690 Address SrcAddr, const VarDecl *DestVD,
691 const VarDecl *SrcVD, const Expr *Copy) {
692 if (OriginalType->isArrayType()) {
693 const auto *BO = dyn_cast<BinaryOperator>(Copy);
694 if (BO && BO->getOpcode() == BO_Assign) {
695 // Perform simple memcpy for simple copying.
696 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
697 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
698 EmitAggregateAssign(Dest, Src, OriginalType);
699 } else {
700 // For arrays with complex element types perform element by element
701 // copying.
702 EmitOMPAggregateAssign(
703 DestAddr, SrcAddr, OriginalType,
704 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
705 // Working with the single array element, so have to remap
706 // destination and source variables to corresponding array
707 // elements.
708 CodeGenFunction::OMPPrivateScope Remap(*this);
709 Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
710 Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
711 (void)Remap.Privatize();
712 EmitIgnoredExpr(Copy);
713 });
714 }
715 } else {
716 // Remap pseudo source variable to private copy.
717 CodeGenFunction::OMPPrivateScope Remap(*this);
718 Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
719 Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
720 (void)Remap.Privatize();
721 // Emit copying of the whole variable.
722 EmitIgnoredExpr(Copy);
723 }
724 }
725
726 bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
727 OMPPrivateScope &PrivateScope) {
728 if (!HaveInsertPoint())
729 return false;
730 bool FirstprivateIsLastprivate = false;
731 llvm::DenseSet<const VarDecl *> Lastprivates;
732 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
733 for (const auto *D : C->varlists())
734 Lastprivates.insert(
735 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
736 }
737 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
738 llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
739 getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
740 // Force emission of the firstprivate copy if the directive does not emit
741 // outlined function, like omp for, omp simd, omp distribute etc.
742 bool MustEmitFirstprivateCopy =
743 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
744 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
745 auto IRef = C->varlist_begin();
746 auto InitsRef = C->inits().begin();
747 for (const Expr *IInit : C->private_copies()) {
748 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
749 bool ThisFirstprivateIsLastprivate =
750 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
751 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
752 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
753 !FD->getType()->isReferenceType()) {
754 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
755 ++IRef;
756 ++InitsRef;
757 continue;
758 }
759 FirstprivateIsLastprivate =
760 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
761 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
762 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
763 const auto *VDInit =
764 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
765 bool IsRegistered;
766 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
767 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
768 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
769 LValue OriginalLVal = EmitLValue(&DRE);
770 QualType Type = VD->getType();
771 if (Type->isArrayType()) {
772 // Emit VarDecl with copy init for arrays.
773 // Get the address of the original variable captured in current
774 // captured region.
775 IsRegistered = PrivateScope.addPrivate(
776 OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
777 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
778 const Expr *Init = VD->getInit();
779 if (!isa<CXXConstructExpr>(Init) ||
780 isTrivialInitializer(Init)) {
781 // Perform simple memcpy.
782 LValue Dest =
783 MakeAddrLValue(Emission.getAllocatedAddress(), Type);
784 EmitAggregateAssign(Dest, OriginalLVal, Type);
785 } else {
786 EmitOMPAggregateAssign(
787 Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
788 Type,
789 [this, VDInit, Init](Address DestElement,
790 Address SrcElement) {
791 // Clean up any temporaries needed by the
792 // initialization.
793 RunCleanupsScope InitScope(*this);
794 // Emit initialization for single element.
795 setAddrOfLocalVar(VDInit, SrcElement);
796 EmitAnyExprToMem(Init, DestElement,
797 Init->getType().getQualifiers(),
798 /*IsInitializer*/ false);
799 LocalDeclMap.erase(VDInit);
800 });
801 }
802 EmitAutoVarCleanups(Emission);
803 return Emission.getAllocatedAddress();
804 });
805 } else {
806 Address OriginalAddr = OriginalLVal.getAddress();
807 IsRegistered = PrivateScope.addPrivate(
808 OrigVD, [this, VDInit, OriginalAddr, VD]() {
809 // Emit private VarDecl with copy init.
810 // Remap temp VDInit variable to the address of the original
811 // variable (for proper handling of captured global variables).
812 setAddrOfLocalVar(VDInit, OriginalAddr);
813 EmitDecl(*VD);
814 LocalDeclMap.erase(VDInit);
815 return GetAddrOfLocalVar(VD);
816 });
817 }
818 assert(IsRegistered &&
819 "firstprivate var already registered as private");
820 // Silence the warning about unused variable.
821 (void)IsRegistered;
822 }
823 ++IRef;
824 ++InitsRef;
825 }
826 }
827 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
828 }
829
830 void CodeGenFunction::EmitOMPPrivateClause(
831 const OMPExecutableDirective &D,
832 CodeGenFunction::OMPPrivateScope &PrivateScope) {
833 if (!HaveInsertPoint())
834 return;
835 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
836 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
837 auto IRef = C->varlist_begin();
838 for (const Expr *IInit : C->private_copies()) {
839 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
840 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
841 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
842 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
843 // Emit private VarDecl with copy init.
844 EmitDecl(*VD);
845 return GetAddrOfLocalVar(VD);
846 });
847 assert(IsRegistered && "private var already registered as private");
848 // Silence the warning about unused variable.
849 (void)IsRegistered;
850 }
851 ++IRef;
852 }
853 }
854 }
855
856 bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
857 if (!HaveInsertPoint())
858 return false;
859 // threadprivate_var1 = master_threadprivate_var1;
860 // operator=(threadprivate_var2, master_threadprivate_var2);
861 // ...
862 // __kmpc_barrier(&loc, global_tid);
863 llvm::DenseSet<const VarDecl *> CopiedVars;
864 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
865 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
866 auto IRef = C->varlist_begin();
867 auto ISrcRef = C->source_exprs().begin();
868 auto IDestRef = C->destination_exprs().begin();
869 for (const Expr *AssignOp : C->assignment_ops()) {
870 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
871 QualType Type = VD->getType();
872 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
873 // Get the address of the master variable. If we are emitting code with
874 // TLS support, the address is passed from the master as field in the
875 // captured declaration.
876 Address MasterAddr = Address::invalid();
877 if (getLangOpts().OpenMPUseTLS &&
878 getContext().getTargetInfo().isTLSSupported()) {
879 assert(CapturedStmtInfo->lookup(VD) &&
880 "Copyin threadprivates should have been captured!");
881 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
882 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
883 MasterAddr = EmitLValue(&DRE).getAddress();
884 LocalDeclMap.erase(VD);
885 } else {
886 MasterAddr =
887 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
888 : CGM.GetAddrOfGlobal(VD),
889 getContext().getDeclAlign(VD));
890 }
891 // Get the address of the threadprivate variable.
892 Address PrivateAddr = EmitLValue(*IRef).getAddress();
893 if (CopiedVars.size() == 1) {
894 // At first check if current thread is a master thread. If it is, no
895 // need to copy data.
896 CopyBegin = createBasicBlock("copyin.not.master");
897 CopyEnd = createBasicBlock("copyin.not.master.end");
898 Builder.CreateCondBr(
899 Builder.CreateICmpNE(
900 Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
901 Builder.CreatePtrToInt(PrivateAddr.getPointer(),
902 CGM.IntPtrTy)),
903 CopyBegin, CopyEnd);
904 EmitBlock(CopyBegin);
905 }
906 const auto *SrcVD =
907 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
908 const auto *DestVD =
909 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
910 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
911 }
912 ++IRef;
913 ++ISrcRef;
914 ++IDestRef;
915 }
916 }
917 if (CopyEnd) {
918 // Exit out of copying procedure for non-master thread.
919 EmitBlock(CopyEnd, /*IsFinished=*/true);
920 return true;
921 }
922 return false;
923 }
924
925 bool CodeGenFunction::EmitOMPLastprivateClauseInit(
926 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
927 if (!HaveInsertPoint())
928 return false;
929 bool HasAtLeastOneLastprivate = false;
930 llvm::DenseSet<const VarDecl *> SIMDLCVs;
931 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
932 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
933 for (const Expr *C : LoopDirective->counters()) {
934 SIMDLCVs.insert(
935 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
936 }
937 }
938 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
939 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
940 HasAtLeastOneLastprivate = true;
941 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
942 !getLangOpts().OpenMPSimd)
943 break;
944 auto IRef = C->varlist_begin();
945 auto IDestRef = C->destination_exprs().begin();
946 for (const Expr *IInit : C->private_copies()) {
947 // Keep the address of the original variable for future update at the end
948 // of the loop.
949 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
950 // Taskloops do not require additional initialization, it is done in
951 // runtime support library.
952 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
953 const auto *DestVD =
954 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
955 PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
956 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
957 /*RefersToEnclosingVariableOrCapture=*/
958 CapturedStmtInfo->lookup(OrigVD) != nullptr,
959 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
960 return EmitLValue(&DRE).getAddress();
961 });
962 // Check if the variable is also a firstprivate: in this case IInit is
963 // not generated. Initialization of this variable will happen in codegen
964 // for 'firstprivate' clause.
965 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
966 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
967 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
968 // Emit private VarDecl with copy init.
969 EmitDecl(*VD);
970 return GetAddrOfLocalVar(VD);
971 });
972 assert(IsRegistered &&
973 "lastprivate var already registered as private");
974 (void)IsRegistered;
975 }
976 }
977 ++IRef;
978 ++IDestRef;
979 }
980 }
981 return HasAtLeastOneLastprivate;
982 }
983
984 void CodeGenFunction::EmitOMPLastprivateClauseFinal(
985 const OMPExecutableDirective &D, bool NoFinals,
986 llvm::Value *IsLastIterCond) {
987 if (!HaveInsertPoint())
988 return;
989 // Emit following code:
990 // if (<IsLastIterCond>) {
991 // orig_var1 = private_orig_var1;
992 // ...
993 // orig_varn = private_orig_varn;
994 // }
995 llvm::BasicBlock *ThenBB = nullptr;
996 llvm::BasicBlock *DoneBB = nullptr;
997 if (IsLastIterCond) {
998 ThenBB = createBasicBlock(".omp.lastprivate.then");
999 DoneBB = createBasicBlock(".omp.lastprivate.done");
1000 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1001 EmitBlock(ThenBB);
1002 }
1003 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1004 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1005 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1006 auto IC = LoopDirective->counters().begin();
1007 for (const Expr *F : LoopDirective->finals()) {
1008 const auto *D =
1009 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1010 if (NoFinals)
1011 AlreadyEmittedVars.insert(D);
1012 else
1013 LoopCountersAndUpdates[D] = F;
1014 ++IC;
1015 }
1016 }
1017 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1018 auto IRef = C->varlist_begin();
1019 auto ISrcRef = C->source_exprs().begin();
1020 auto IDestRef = C->destination_exprs().begin();
1021 for (const Expr *AssignOp : C->assignment_ops()) {
1022 const auto *PrivateVD =
1023 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1024 QualType Type = PrivateVD->getType();
1025 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1026 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1027 // If lastprivate variable is a loop control variable for loop-based
1028 // directive, update its value before copyin back to original
1029 // variable.
1030 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1031 EmitIgnoredExpr(FinalExpr);
1032 const auto *SrcVD =
1033 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1034 const auto *DestVD =
1035 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1036 // Get the address of the original variable.
1037 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1038 // Get the address of the private variable.
1039 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1040 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1041 PrivateAddr =
1042 Address(Builder.CreateLoad(PrivateAddr),
1043 getNaturalTypeAlignment(RefTy->getPointeeType()));
1044 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1045 }
1046 ++IRef;
1047 ++ISrcRef;
1048 ++IDestRef;
1049 }
1050 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1051 EmitIgnoredExpr(PostUpdate);
1052 }
1053 if (IsLastIterCond)
1054 EmitBlock(DoneBB, /*IsFinished=*/true);
1055 }
1056
1057 void CodeGenFunction::EmitOMPReductionClauseInit(
1058 const OMPExecutableDirective &D,
1059 CodeGenFunction::OMPPrivateScope &PrivateScope) {
1060 if (!HaveInsertPoint())
1061 return;
1062 SmallVector<const Expr *, 4> Shareds;
1063 SmallVector<const Expr *, 4> Privates;
1064 SmallVector<const Expr *, 4> ReductionOps;
1065 SmallVector<const Expr *, 4> LHSs;
1066 SmallVector<const Expr *, 4> RHSs;
1067 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1068 auto IPriv = C->privates().begin();
1069 auto IRed = C->reduction_ops().begin();
1070 auto ILHS = C->lhs_exprs().begin();
1071 auto IRHS = C->rhs_exprs().begin();
1072 for (const Expr *Ref : C->varlists()) {
1073 Shareds.emplace_back(Ref);
1074 Privates.emplace_back(*IPriv);
1075 ReductionOps.emplace_back(*IRed);
1076 LHSs.emplace_back(*ILHS);
1077 RHSs.emplace_back(*IRHS);
1078 std::advance(IPriv, 1);
1079 std::advance(IRed, 1);
1080 std::advance(ILHS, 1);
1081 std::advance(IRHS, 1);
1082 }
1083 }
1084 ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1085 unsigned Count = 0;
1086 auto ILHS = LHSs.begin();
1087 auto IRHS = RHSs.begin();
1088 auto IPriv = Privates.begin();
1089 for (const Expr *IRef : Shareds) {
1090 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1091 // Emit private VarDecl with reduction init.
1092 RedCG.emitSharedLValue(*this, Count);
1093 RedCG.emitAggregateType(*this, Count);
1094 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1095 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1096 RedCG.getSharedLValue(Count),
1097 [&Emission](CodeGenFunction &CGF) {
1098 CGF.EmitAutoVarInit(Emission);
1099 return true;
1100 });
1101 EmitAutoVarCleanups(Emission);
1102 Address BaseAddr = RedCG.adjustPrivateAddress(
1103 *this, Count, Emission.getAllocatedAddress());
1104 bool IsRegistered = PrivateScope.addPrivate(
1105 RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1106 assert(IsRegistered && "private var already registered as private");
1107 // Silence the warning about unused variable.
1108 (void)IsRegistered;
1109
1110 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1111 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1112 QualType Type = PrivateVD->getType();
1113 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1114 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1115 // Store the address of the original variable associated with the LHS
1116 // implicit variable.
1117 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1118 return RedCG.getSharedLValue(Count).getAddress();
1119 });
1120 PrivateScope.addPrivate(
1121 RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1122 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1123 isa<ArraySubscriptExpr>(IRef)) {
1124 // Store the address of the original variable associated with the LHS
1125 // implicit variable.
1126 PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1127 return RedCG.getSharedLValue(Count).getAddress();
1128 });
1129 PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1130 return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1131 ConvertTypeForMem(RHSVD->getType()),
1132 "rhs.begin");
1133 });
1134 } else {
1135 QualType Type = PrivateVD->getType();
1136 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1137 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1138 // Store the address of the original variable associated with the LHS
1139 // implicit variable.
1140 if (IsArray) {
1141 OriginalAddr = Builder.CreateElementBitCast(
1142 OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1143 }
1144 PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1145 PrivateScope.addPrivate(
1146 RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1147 return IsArray
1148 ? Builder.CreateElementBitCast(
1149 GetAddrOfLocalVar(PrivateVD),
1150 ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1151 : GetAddrOfLocalVar(PrivateVD);
1152 });
1153 }
1154 ++ILHS;
1155 ++IRHS;
1156 ++IPriv;
1157 ++Count;
1158 }
1159 }
1160
1161 void CodeGenFunction::EmitOMPReductionClauseFinal(
1162 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1163 if (!HaveInsertPoint())
1164 return;
1165 llvm::SmallVector<const Expr *, 8> Privates;
1166 llvm::SmallVector<const Expr *, 8> LHSExprs;
1167 llvm::SmallVector<const Expr *, 8> RHSExprs;
1168 llvm::SmallVector<const Expr *, 8> ReductionOps;
1169 bool HasAtLeastOneReduction = false;
1170 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1171 HasAtLeastOneReduction = true;
1172 Privates.append(C->privates().begin(), C->privates().end());
1173 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1174 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1175 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1176 }
1177 if (HasAtLeastOneReduction) {
1178 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1179 isOpenMPParallelDirective(D.getDirectiveKind()) ||
1180 ReductionKind == OMPD_simd;
1181 bool SimpleReduction = ReductionKind == OMPD_simd;
1182 // Emit nowait reduction if nowait clause is present or directive is a
1183 // parallel directive (it always has implicit barrier).
1184 CGM.getOpenMPRuntime().emitReduction(
1185 *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1186 {WithNowait, SimpleReduction, ReductionKind});
1187 }
1188 }
1189
1190 static void emitPostUpdateForReductionClause(
1191 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1192 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1193 if (!CGF.HaveInsertPoint())
1194 return;
1195 llvm::BasicBlock *DoneBB = nullptr;
1196 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1197 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1198 if (!DoneBB) {
1199 if (llvm::Value *Cond = CondGen(CGF)) {
1200 // If the first post-update expression is found, emit conditional
1201 // block if it was requested.
1202 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1203 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1204 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1205 CGF.EmitBlock(ThenBB);
1206 }
1207 }
1208 CGF.EmitIgnoredExpr(PostUpdate);
1209 }
1210 }
1211 if (DoneBB)
1212 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1213 }
1214
1215 namespace {
1216 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1217 /// parallel function. This is necessary for combined constructs such as
1218 /// 'distribute parallel for'
1219 typedef llvm::function_ref<void(CodeGenFunction &,
1220 const OMPExecutableDirective &,
1221 llvm::SmallVectorImpl<llvm::Value *> &)>
1222 CodeGenBoundParametersTy;
1223 } // anonymous namespace
1224
1225 static void emitCommonOMPParallelDirective(
1226 CodeGenFunction &CGF, const OMPExecutableDirective &S,
1227 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1228 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1229 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1230 llvm::Value *OutlinedFn =
1231 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1232 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1233 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1234 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1235 llvm::Value *NumThreads =
1236 CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1237 /*IgnoreResultAssign=*/true);
1238 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1239 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1240 }
1241 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1242 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1243 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1244 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1245 }
1246 const Expr *IfCond = nullptr;
1247 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1248 if (C->getNameModifier() == OMPD_unknown ||
1249 C->getNameModifier() == OMPD_parallel) {
1250 IfCond = C->getCondition();
1251 break;
1252 }
1253 }
1254
1255 OMPParallelScope Scope(CGF, S);
1256 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
1257 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1258 // lower and upper bounds with the pragma 'for' chunking mechanism.
1259 // The following lambda takes care of appending the lower and upper bound
1260 // parameters when necessary
1261 CodeGenBoundParameters(CGF, S, CapturedVars);
1262 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1263 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1264 CapturedVars, IfCond);
1265 }
1266
1267 static void emitEmptyBoundParameters(CodeGenFunction &,
1268 const OMPExecutableDirective &,
1269 llvm::SmallVectorImpl<llvm::Value *> &) {}
1270
1271 void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
1272 // Emit parallel region as a standalone region.
1273 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1274 Action.Enter(CGF);
1275 OMPPrivateScope PrivateScope(CGF);
1276 bool Copyins = CGF.EmitOMPCopyinClause(S);
1277 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1278 if (Copyins) {
1279 // Emit implicit barrier to synchronize threads and avoid data races on
1280 // propagation master's thread values of threadprivate variables to local
1281 // instances of that variables of all other implicit threads.
1282 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1283 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1284 /*ForceSimpleCall=*/true);
1285 }
1286 CGF.EmitOMPPrivateClause(S, PrivateScope);
1287 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1288 (void)PrivateScope.Privatize();
1289 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1290 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1291 };
1292 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1293 emitEmptyBoundParameters);
1294 emitPostUpdateForReductionClause(*this, S,
1295 [](CodeGenFunction &) { return nullptr; });
1296 }
1297
1298 void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
1299 JumpDest LoopExit) {
1300 RunCleanupsScope BodyScope(*this);
1301 // Update counters values on current iteration.
1302 for (const Expr *UE : D.updates())
1303 EmitIgnoredExpr(UE);
1304 // Update the linear variables.
1305 // In distribute directives only loop counters may be marked as linear, no
1306 // need to generate the code for them.
1307 if (!isOpenMPDistributeDirective(D.getDirectiveKind())) {
1308 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1309 for (const Expr *UE : C->updates())
1310 EmitIgnoredExpr(UE);
1311 }
1312 }
1313
1314 // On a continue in the body, jump to the end.
1315 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1316 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1317 // Emit loop body.
1318 EmitStmt(D.getBody());
1319 // The end (updates/cleanups).
1320 EmitBlock(Continue.getBlock());
1321 BreakContinueStack.pop_back();
1322 }
1323
1324 void CodeGenFunction::EmitOMPInnerLoop(
1325 const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1326 const Expr *IncExpr,
1327 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1328 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1329 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1330
1331 // Start the loop with a block that tests the condition.
1332 auto CondBlock = createBasicBlock("omp.inner.for.cond");
1333 EmitBlock(CondBlock);
1334 const SourceRange R = S.getSourceRange();
1335 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1336 SourceLocToDebugLoc(R.getEnd()));
1337
1338 // If there are any cleanups between here and the loop-exit scope,
1339 // create a block to stage a loop exit along.
1340 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1341 if (RequiresCleanup)
1342 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1343
1344 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1345
1346 // Emit condition.
1347 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1348 if (ExitBlock != LoopExit.getBlock()) {
1349 EmitBlock(ExitBlock);
1350 EmitBranchThroughCleanup(LoopExit);
1351 }
1352
1353 EmitBlock(LoopBody);
1354 incrementProfileCounter(&S);
1355
1356 // Create a block for the increment.
1357 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1358 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1359
1360 BodyGen(*this);
1361
1362 // Emit "IV = IV + 1" and a back-edge to the condition block.
1363 EmitBlock(Continue.getBlock());
1364 EmitIgnoredExpr(IncExpr);
1365 PostIncGen(*this);
1366 BreakContinueStack.pop_back();
1367 EmitBranch(CondBlock);
1368 LoopStack.pop();
1369 // Emit the fall-through block.
1370 EmitBlock(LoopExit.getBlock());
1371 }
1372
1373 bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
1374 if (!HaveInsertPoint())
1375 return false;
1376 // Emit inits for the linear variables.
1377 bool HasLinears = false;
1378 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1379 for (const Expr *Init : C->inits()) {
1380 HasLinears = true;
1381 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1382 if (const auto *Ref =
1383 dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1384 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1385 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1386 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1387 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1388 VD->getInit()->getType(), VK_LValue,
1389 VD->getInit()->getExprLoc());
1390 EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1391 VD->getType()),
1392 /*capturedByInit=*/false);
1393 EmitAutoVarCleanups(Emission);
1394 } else {
1395 EmitVarDecl(*VD);
1396 }
1397 }
1398 // Emit the linear steps for the linear clauses.
1399 // If a step is not constant, it is pre-calculated before the loop.
1400 if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1401 if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1402 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1403 // Emit calculation of the linear step.
1404 EmitIgnoredExpr(CS);
1405 }
1406 }
1407 return HasLinears;
1408 }
1409
1410 void CodeGenFunction::EmitOMPLinearClauseFinal(
1411 const OMPLoopDirective &D,
1412 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1413 if (!HaveInsertPoint())
1414 return;
1415 llvm::BasicBlock *DoneBB = nullptr;
1416 // Emit the final values of the linear variables.
1417 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1418 auto IC = C->varlist_begin();
1419 for (const Expr *F : C->finals()) {
1420 if (!DoneBB) {
1421 if (llvm::Value *Cond = CondGen(*this)) {
1422 // If the first post-update expression is found, emit conditional
1423 // block if it was requested.
1424 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1425 DoneBB = createBasicBlock(".omp.linear.pu.done");
1426 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1427 EmitBlock(ThenBB);
1428 }
1429 }
1430 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1431 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1432 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1433 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1434 Address OrigAddr = EmitLValue(&DRE).getAddress();
1435 CodeGenFunction::OMPPrivateScope VarScope(*this);
1436 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1437 (void)VarScope.Privatize();
1438 EmitIgnoredExpr(F);
1439 ++IC;
1440 }
1441 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1442 EmitIgnoredExpr(PostUpdate);
1443 }
1444 if (DoneBB)
1445 EmitBlock(DoneBB, /*IsFinished=*/true);
1446 }
1447
1448 static void emitAlignedClause(CodeGenFunction &CGF,
1449 const OMPExecutableDirective &D) {
1450 if (!CGF.HaveInsertPoint())
1451 return;
1452 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1453 unsigned ClauseAlignment = 0;
1454 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1455 auto *AlignmentCI =
1456 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1457 ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1458 }
1459 for (const Expr *E : Clause->varlists()) {
1460 unsigned Alignment = ClauseAlignment;
1461 if (Alignment == 0) {
1462 // OpenMP [2.8.1, Description]
1463 // If no optional parameter is specified, implementation-defined default
1464 // alignments for SIMD instructions on the target platforms are assumed.
1465 Alignment =
1466 CGF.getContext()
1467 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
1468 E->getType()->getPointeeType()))
1469 .getQuantity();
1470 }
1471 assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1472 "alignment is not power of 2");
1473 if (Alignment != 0) {
1474 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1475 CGF.EmitAlignmentAssumption(
1476 PtrValue, E, /*No second loc needed*/ SourceLocation(), Alignment);
1477 }
1478 }
1479 }
1480 }
1481
1482 void CodeGenFunction::EmitOMPPrivateLoopCounters(
1483 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1484 if (!HaveInsertPoint())
1485 return;
1486 auto I = S.private_counters().begin();
1487 for (const Expr *E : S.counters()) {
1488 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1489 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1490 // Emit var without initialization.
1491 AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1492 EmitAutoVarCleanups(VarEmission);
1493 LocalDeclMap.erase(PrivateVD);
1494 (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1495 return VarEmission.getAllocatedAddress();
1496 });
1497 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1498 VD->hasGlobalStorage()) {
1499 (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1500 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
1501 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1502 E->getType(), VK_LValue, E->getExprLoc());
1503 return EmitLValue(&DRE).getAddress();
1504 });
1505 } else {
1506 (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1507 return VarEmission.getAllocatedAddress();
1508 });
1509 }
1510 ++I;
1511 }
1512 // Privatize extra loop counters used in loops for ordered(n) clauses.
1513 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1514 if (!C->getNumForLoops())
1515 continue;
1516 for (unsigned I = S.getCollapsedNumber(),
1517 E = C->getLoopNumIterations().size();
1518 I < E; ++I) {
1519 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
1520 const auto *VD = cast<VarDecl>(DRE->getDecl());
1521 // Override only those variables that are really emitted already.
1522 if (LocalDeclMap.count(VD)) {
1523 (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1524 return CreateMemTemp(DRE->getType(), VD->getName());
1525 });
1526 }
1527 }
1528 }
1529 }
1530
1531 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1532 const Expr *Cond, llvm::BasicBlock *TrueBlock,
1533 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1534 if (!CGF.HaveInsertPoint())
1535 return;
1536 {
1537 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1538 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1539 (void)PreCondScope.Privatize();
1540 // Get initial values of real counters.
1541 for (const Expr *I : S.inits()) {
1542 CGF.EmitIgnoredExpr(I);
1543 }
1544 }
1545 // Check that loop is executed at least one time.
1546 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1547 }
1548
1549 void CodeGenFunction::EmitOMPLinearClause(
1550 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1551 if (!HaveInsertPoint())
1552 return;
1553 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1554 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
1555 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1556 for (const Expr *C : LoopDirective->counters()) {
1557 SIMDLCVs.insert(
1558 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1559 }
1560 }
1561 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1562 auto CurPrivate = C->privates().begin();
1563 for (const Expr *E : C->varlists()) {
1564 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1565 const auto *PrivateVD =
1566 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1567 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1568 bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1569 // Emit private VarDecl with copy init.
1570 EmitVarDecl(*PrivateVD);
1571 return GetAddrOfLocalVar(PrivateVD);
1572 });
1573 assert(IsRegistered && "linear var already registered as private");
1574 // Silence the warning about unused variable.
1575 (void)IsRegistered;
1576 } else {
1577 EmitVarDecl(*PrivateVD);
1578 }
1579 ++CurPrivate;
1580 }
1581 }
1582 }
1583
1584 static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
1585 const OMPExecutableDirective &D,
1586 bool IsMonotonic) {
1587 if (!CGF.HaveInsertPoint())
1588 return;
1589 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1590 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1591 /*ignoreResult=*/true);
1592 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1593 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1594 // In presence of finite 'safelen', it may be unsafe to mark all
1595 // the memory instructions parallel, because loop-carried
1596 // dependences of 'safelen' iterations are possible.
1597 if (!IsMonotonic)
1598 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
1599 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1600 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1601 /*ignoreResult=*/true);
1602 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1603 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1604 // In presence of finite 'safelen', it may be unsafe to mark all
1605 // the memory instructions parallel, because loop-carried
1606 // dependences of 'safelen' iterations are possible.
1607 CGF.LoopStack.setParallel(/*Enable=*/false);
1608 }
1609 }
1610
1611 void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D,
1612 bool IsMonotonic) {
1613 // Walk clauses and process safelen/lastprivate.
1614 LoopStack.setParallel(!IsMonotonic);
1615 LoopStack.setVectorizeEnable();
1616 emitSimdlenSafelenClause(*this, D, IsMonotonic);
1617 }
1618
1619 void CodeGenFunction::EmitOMPSimdFinal(
1620 const OMPLoopDirective &D,
1621 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1622 if (!HaveInsertPoint())
1623 return;
1624 llvm::BasicBlock *DoneBB = nullptr;
1625 auto IC = D.counters().begin();
1626 auto IPC = D.private_counters().begin();
1627 for (const Expr *F : D.finals()) {
1628 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1629 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1630 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1631 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1632 OrigVD->hasGlobalStorage() || CED) {
1633 if (!DoneBB) {
1634 if (llvm::Value *Cond = CondGen(*this)) {
1635 // If the first post-update expression is found, emit conditional
1636 // block if it was requested.
1637 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1638 DoneBB = createBasicBlock(".omp.final.done");
1639 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1640 EmitBlock(ThenBB);
1641 }
1642 }
1643 Address OrigAddr = Address::invalid();
1644 if (CED) {
1645 OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1646 } else {
1647 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
1648 /*RefersToEnclosingVariableOrCapture=*/false,
1649 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1650 OrigAddr = EmitLValue(&DRE).getAddress();
1651 }
1652 OMPPrivateScope VarScope(*this);
1653 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1654 (void)VarScope.Privatize();
1655 EmitIgnoredExpr(F);
1656 }
1657 ++IC;
1658 ++IPC;
1659 }
1660 if (DoneBB)
1661 EmitBlock(DoneBB, /*IsFinished=*/true);
1662 }
1663
1664 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
1665 const OMPLoopDirective &S,
1666 CodeGenFunction::JumpDest LoopExit) {
1667 CGF.EmitOMPLoopBody(S, LoopExit);
1668 CGF.EmitStopPoint(&S);
1669 }
1670
1671 /// Emit a helper variable and return corresponding lvalue.
1672 static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
1673 const DeclRefExpr *Helper) {
1674 auto VDecl = cast<VarDecl>(Helper->getDecl());
1675 CGF.EmitVarDecl(*VDecl);
1676 return CGF.EmitLValue(Helper);
1677 }
1678
1679 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
1680 PrePostActionTy &Action) {
1681 Action.Enter(CGF);
1682 assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&
1683 "Expected simd directive");
1684 OMPLoopScope PreInitScope(CGF, S);
1685 // if (PreCond) {
1686 // for (IV in 0..LastIteration) BODY;
1687 // <Final counter/linear vars updates>;
1688 // }
1689 //
1690 if (isOpenMPDistributeDirective(S.getDirectiveKind()) ||
1691 isOpenMPWorksharingDirective(S.getDirectiveKind()) ||
1692 isOpenMPTaskLoopDirective(S.getDirectiveKind())) {
1693 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1694 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1695 }
1696
1697 // Emit: if (PreCond) - begin.
1698 // If the condition constant folds and can be elided, avoid emitting the
1699 // whole loop.
1700 bool CondConstant;
1701 llvm::BasicBlock *ContBlock = nullptr;
1702 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1703 if (!CondConstant)
1704 return;
1705 } else {
1706 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1707 ContBlock = CGF.createBasicBlock("simd.if.end");
1708 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1709 CGF.getProfileCount(&S));
1710 CGF.EmitBlock(ThenBlock);
1711 CGF.incrementProfileCounter(&S);
1712 }
1713
1714 // Emit the loop iteration variable.
1715 const Expr *IVExpr = S.getIterationVariable();
1716 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1717 CGF.EmitVarDecl(*IVDecl);
1718 CGF.EmitIgnoredExpr(S.getInit());
1719
1720 // Emit the iterations count variable.
1721 // If it is not a variable, Sema decided to calculate iterations count on
1722 // each iteration (e.g., it is foldable into a constant).
1723 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1724 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1725 // Emit calculation of the iterations count.
1726 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
1727 }
1728
1729 CGF.EmitOMPSimdInit(S);
1730
1731 emitAlignedClause(CGF, S);
1732 (void)CGF.EmitOMPLinearClauseInit(S);
1733 {
1734 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1735 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1736 CGF.EmitOMPLinearClause(S, LoopScope);
1737 CGF.EmitOMPPrivateClause(S, LoopScope);
1738 CGF.EmitOMPReductionClauseInit(S, LoopScope);
1739 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1740 (void)LoopScope.Privatize();
1741 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
1742 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
1743 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1744 S.getInc(),
1745 [&S](CodeGenFunction &CGF) {
1746 CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest());
1747 CGF.EmitStopPoint(&S);
1748 },
1749 [](CodeGenFunction &) {});
1750 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
1751 // Emit final copy of the lastprivate variables at the end of loops.
1752 if (HasLastprivateClause)
1753 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1754 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1755 emitPostUpdateForReductionClause(CGF, S,
1756 [](CodeGenFunction &) { return nullptr; });
1757 }
1758 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
1759 // Emit: if (PreCond) - end.
1760 if (ContBlock) {
1761 CGF.EmitBranch(ContBlock);
1762 CGF.EmitBlock(ContBlock, true);
1763 }
1764 }
1765
1766 void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
1767 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1768 emitOMPSimdRegion(CGF, S, Action);
1769 };
1770 OMPLexicalScope Scope(*this, S, OMPD_unknown);
1771 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1772 }
1773
1774 void CodeGenFunction::EmitOMPOuterLoop(
1775 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1776 CodeGenFunction::OMPPrivateScope &LoopScope,
1777 const CodeGenFunction::OMPLoopArguments &LoopArgs,
1778 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1779 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1780 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1781
1782 const Expr *IVExpr = S.getIterationVariable();
1783 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1784 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1785
1786 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1787
1788 // Start the loop with a block that tests the condition.
1789 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
1790 EmitBlock(CondBlock);
1791 const SourceRange R = S.getSourceRange();
1792 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1793 SourceLocToDebugLoc(R.getEnd()));
1794
1795 llvm::Value *BoolCondVal = nullptr;
1796 if (!DynamicOrOrdered) {
1797 // UB = min(UB, GlobalUB) or
1798 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1799 // 'distribute parallel for')
1800 EmitIgnoredExpr(LoopArgs.EUB);
1801 // IV = LB
1802 EmitIgnoredExpr(LoopArgs.Init);
1803 // IV < UB
1804 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1805 } else {
1806 BoolCondVal =
1807 RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
1808 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1809 }
1810
1811 // If there are any cleanups between here and the loop-exit scope,
1812 // create a block to stage a loop exit along.
1813 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1814 if (LoopScope.requiresCleanups())
1815 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1816
1817 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
1818 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1819 if (ExitBlock != LoopExit.getBlock()) {
1820 EmitBlock(ExitBlock);
1821 EmitBranchThroughCleanup(LoopExit);
1822 }
1823 EmitBlock(LoopBody);
1824
1825 // Emit "IV = LB" (in case of static schedule, we have already calculated new
1826 // LB for loop condition and emitted it above).
1827 if (DynamicOrOrdered)
1828 EmitIgnoredExpr(LoopArgs.Init);
1829
1830 // Create a block for the increment.
1831 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1832 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1833
1834 // Generate !llvm.loop.parallel metadata for loads and stores for loops
1835 // with dynamic/guided scheduling and without ordered clause.
1836 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
1837 LoopStack.setParallel(!IsMonotonic);
1838 else
1839 EmitOMPSimdInit(S, IsMonotonic);
1840
1841 SourceLocation Loc = S.getBeginLoc();
1842
1843 // when 'distribute' is not combined with a 'for':
1844 // while (idx <= UB) { BODY; ++idx; }
1845 // when 'distribute' is combined with a 'for'
1846 // (e.g. 'distribute parallel for')
1847 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1848 EmitOMPInnerLoop(
1849 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1850 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1851 CodeGenLoop(CGF, S, LoopExit);
1852 },
1853 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1854 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1855 });
1856
1857 EmitBlock(Continue.getBlock());
1858 BreakContinueStack.pop_back();
1859 if (!DynamicOrOrdered) {
1860 // Emit "LB = LB + Stride", "UB = UB + Stride".
1861 EmitIgnoredExpr(LoopArgs.NextLB);
1862 EmitIgnoredExpr(LoopArgs.NextUB);
1863 }
1864
1865 EmitBranch(CondBlock);
1866 LoopStack.pop();
1867 // Emit the fall-through block.
1868 EmitBlock(LoopExit.getBlock());
1869
1870 // Tell the runtime we are done.
1871 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1872 if (!DynamicOrOrdered)
1873 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
1874 S.getDirectiveKind());
1875 };
1876 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1877 }
1878
1879 void CodeGenFunction::EmitOMPForOuterLoop(
1880 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1881 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1882 const OMPLoopArguments &LoopArgs,
1883 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1884 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1885
1886 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1887 const bool DynamicOrOrdered =
1888 Ordered || RT.isDynamic(ScheduleKind.Schedule);
1889
1890 assert((Ordered ||
1891 !RT.isStaticNonchunked(ScheduleKind.Schedule,
1892 LoopArgs.Chunk != nullptr)) &&
1893 "static non-chunked schedule does not need outer loop");
1894
1895 // Emit outer loop.
1896 //
1897 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1898 // When schedule(dynamic,chunk_size) is specified, the iterations are
1899 // distributed to threads in the team in chunks as the threads request them.
1900 // Each thread executes a chunk of iterations, then requests another chunk,
1901 // until no chunks remain to be distributed. Each chunk contains chunk_size
1902 // iterations, except for the last chunk to be distributed, which may have
1903 // fewer iterations. When no chunk_size is specified, it defaults to 1.
1904 //
1905 // When schedule(guided,chunk_size) is specified, the iterations are assigned
1906 // to threads in the team in chunks as the executing threads request them.
1907 // Each thread executes a chunk of iterations, then requests another chunk,
1908 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1909 // each chunk is proportional to the number of unassigned iterations divided
1910 // by the number of threads in the team, decreasing to 1. For a chunk_size
1911 // with value k (greater than 1), the size of each chunk is determined in the
1912 // same way, with the restriction that the chunks do not contain fewer than k
1913 // iterations (except for the last chunk to be assigned, which may have fewer
1914 // than k iterations).
1915 //
1916 // When schedule(auto) is specified, the decision regarding scheduling is
1917 // delegated to the compiler and/or runtime system. The programmer gives the
1918 // implementation the freedom to choose any possible mapping of iterations to
1919 // threads in the team.
1920 //
1921 // When schedule(runtime) is specified, the decision regarding scheduling is
1922 // deferred until run time, and the schedule and chunk size are taken from the
1923 // run-sched-var ICV. If the ICV is set to auto, the schedule is
1924 // implementation defined
1925 //
1926 // while(__kmpc_dispatch_next(&LB, &UB)) {
1927 // idx = LB;
1928 // while (idx <= UB) { BODY; ++idx;
1929 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1930 // } // inner loop
1931 // }
1932 //
1933 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1934 // When schedule(static, chunk_size) is specified, iterations are divided into
1935 // chunks of size chunk_size, and the chunks are assigned to the threads in
1936 // the team in a round-robin fashion in the order of the thread number.
1937 //
1938 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1939 // while (idx <= UB) { BODY; ++idx; } // inner loop
1940 // LB = LB + ST;
1941 // UB = UB + ST;
1942 // }
1943 //
1944
1945 const Expr *IVExpr = S.getIterationVariable();
1946 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1947 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1948
1949 if (DynamicOrOrdered) {
1950 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
1951 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1952 llvm::Value *LBVal = DispatchBounds.first;
1953 llvm::Value *UBVal = DispatchBounds.second;
1954 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1955 LoopArgs.Chunk};
1956 RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
1957 IVSigned, Ordered, DipatchRTInputValues);
1958 } else {
1959 CGOpenMPRuntime::StaticRTInput StaticInit(
1960 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1961 LoopArgs.ST, LoopArgs.Chunk);
1962 RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
1963 ScheduleKind, StaticInit);
1964 }
1965
1966 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1967 const unsigned IVSize,
1968 const bool IVSigned) {
1969 if (Ordered) {
1970 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1971 IVSigned);
1972 }
1973 };
1974
1975 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1976 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1977 OuterLoopArgs.IncExpr = S.getInc();
1978 OuterLoopArgs.Init = S.getInit();
1979 OuterLoopArgs.Cond = S.getCond();
1980 OuterLoopArgs.NextLB = S.getNextLowerBound();
1981 OuterLoopArgs.NextUB = S.getNextUpperBound();
1982 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1983 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1984 }
1985
1986 static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
1987 const unsigned IVSize, const bool IVSigned) {}
1988
1989 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1990 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
1991 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
1992 const CodeGenLoopTy &CodeGenLoopContent) {
1993
1994 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1995
1996 // Emit outer loop.
1997 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
1998 // dynamic
1999 //
2000
2001 const Expr *IVExpr = S.getIterationVariable();
2002 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2003 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2004
2005 CGOpenMPRuntime::StaticRTInput StaticInit(
2006 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2007 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2008 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
2009
2010 // for combined 'distribute' and 'for' the increment expression of distribute
2011 // is stored in DistInc. For 'distribute' alone, it is in Inc.
2012 Expr *IncExpr;
2013 if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
2014 IncExpr = S.getDistInc();
2015 else
2016 IncExpr = S.getInc();
2017
2018 // this routine is shared by 'omp distribute parallel for' and
2019 // 'omp distribute': select the right EUB expression depending on the
2020 // directive
2021 OMPLoopArguments OuterLoopArgs;
2022 OuterLoopArgs.LB = LoopArgs.LB;
2023 OuterLoopArgs.UB = LoopArgs.UB;
2024 OuterLoopArgs.ST = LoopArgs.ST;
2025 OuterLoopArgs.IL = LoopArgs.IL;
2026 OuterLoopArgs.Chunk = LoopArgs.Chunk;
2027 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2028 ? S.getCombinedEnsureUpperBound()
2029 : S.getEnsureUpperBound();
2030 OuterLoopArgs.IncExpr = IncExpr;
2031 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2032 ? S.getCombinedInit()
2033 : S.getInit();
2034 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2035 ? S.getCombinedCond()
2036 : S.getCond();
2037 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2038 ? S.getCombinedNextLowerBound()
2039 : S.getNextLowerBound();
2040 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2041 ? S.getCombinedNextUpperBound()
2042 : S.getNextUpperBound();
2043
2044 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2045 LoopScope, OuterLoopArgs, CodeGenLoopContent,
2046 emitEmptyOrdered);
2047 }
2048
2049 static std::pair<LValue, LValue>
2050 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
2051 const OMPExecutableDirective &S) {
2052 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2053 LValue LB =
2054 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2055 LValue UB =
2056 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2057
2058 // When composing 'distribute' with 'for' (e.g. as in 'distribute
2059 // parallel for') we need to use the 'distribute'
2060 // chunk lower and upper bounds rather than the whole loop iteration
2061 // space. These are parameters to the outlined function for 'parallel'
2062 // and we copy the bounds of the previous schedule into the
2063 // the current ones.
2064 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2065 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2066 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2067 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2068 PrevLBVal = CGF.EmitScalarConversion(
2069 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2070 LS.getIterationVariable()->getType(),
2071 LS.getPrevLowerBoundVariable()->getExprLoc());
2072 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2073 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2074 PrevUBVal = CGF.EmitScalarConversion(
2075 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2076 LS.getIterationVariable()->getType(),
2077 LS.getPrevUpperBoundVariable()->getExprLoc());
2078
2079 CGF.EmitStoreOfScalar(PrevLBVal, LB);
2080 CGF.EmitStoreOfScalar(PrevUBVal, UB);
2081
2082 return {LB, UB};
2083 }
2084
2085 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2086 /// we need to use the LB and UB expressions generated by the worksharing
2087 /// code generation support, whereas in non combined situations we would
2088 /// just emit 0 and the LastIteration expression
2089 /// This function is necessary due to the difference of the LB and UB
2090 /// types for the RT emission routines for 'for_static_init' and
2091 /// 'for_dispatch_init'
2092 static std::pair<llvm::Value *, llvm::Value *>
2093 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
2094 const OMPExecutableDirective &S,
2095 Address LB, Address UB) {
2096 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2097 const Expr *IVExpr = LS.getIterationVariable();
2098 // when implementing a dynamic schedule for a 'for' combined with a
2099 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2100 // is not normalized as each team only executes its own assigned
2101 // distribute chunk
2102 QualType IteratorTy = IVExpr->getType();
2103 llvm::Value *LBVal =
2104 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2105 llvm::Value *UBVal =
2106 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2107 return {LBVal, UBVal};
2108 }
2109
2110 static void emitDistributeParallelForDistributeInnerBoundParams(
2111 CodeGenFunction &CGF, const OMPExecutableDirective &S,
2112 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2113 const auto &Dir = cast<OMPLoopDirective>(S);
2114 LValue LB =
2115 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2116 llvm::Value *LBCast = CGF.Builder.CreateIntCast(
2117 CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2118 CapturedVars.push_back(LBCast);
2119 LValue UB =
2120 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2121
2122 llvm::Value *UBCast = CGF.Builder.CreateIntCast(
2123 CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2124 CapturedVars.push_back(UBCast);
2125 }
2126
2127 static void
2128 emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
2129 const OMPLoopDirective &S,
2130 CodeGenFunction::JumpDest LoopExit) {
2131 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2132 PrePostActionTy &Action) {
2133 Action.Enter(CGF);
2134 bool HasCancel = false;
2135 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
2136 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2137 HasCancel = D->hasCancel();
2138 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2139 HasCancel = D->hasCancel();
2140 else if (const auto *D =
2141 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2142 HasCancel = D->hasCancel();
2143 }
2144 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
2145 HasCancel);
2146 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
2147 emitDistributeParallelForInnerBounds,
2148 emitDistributeParallelForDispatchBounds);
2149 };
2150
2151 emitCommonOMPParallelDirective(
2152 CGF, S,
2153 isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2154 CGInlinedWorksharingLoop,
2155 emitDistributeParallelForDistributeInnerBoundParams);
2156 }
2157
2158 void CodeGenFunction::EmitOMPDistributeParallelForDirective(
2159 const OMPDistributeParallelForDirective &S) {
2160 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2161 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2162 S.getDistInc());
2163 };
2164 OMPLexicalScope Scope(*this, S, OMPD_parallel);
2165 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2166 }
2167
2168 void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
2169 const OMPDistributeParallelForSimdDirective &S) {
2170 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2171 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2172 S.getDistInc());
2173 };
2174 OMPLexicalScope Scope(*this, S, OMPD_parallel);
2175 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2176 }
2177
2178 void CodeGenFunction::EmitOMPDistributeSimdDirective(
2179 const OMPDistributeSimdDirective &S) {
2180 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2181 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
2182 };
2183 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2184 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2185 }
2186
2187 void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
2188 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2189 // Emit SPMD target parallel for region as a standalone region.
2190 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2191 emitOMPSimdRegion(CGF, S, Action);
2192 };
2193 llvm::Function *Fn;
2194 llvm::Constant *Addr;
2195 // Emit target region as a standalone region.
2196 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2197 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2198 assert(Fn && Addr && "Target device function emission failed.");
2199 }
2200
2201 void CodeGenFunction::EmitOMPTargetSimdDirective(
2202 const OMPTargetSimdDirective &S) {
2203 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2204 emitOMPSimdRegion(CGF, S, Action);
2205 };
2206 emitCommonOMPTargetDirective(*this, S, CodeGen);
2207 }
2208
2209 namespace {
2210 struct ScheduleKindModifiersTy {
2211 OpenMPScheduleClauseKind Kind;
2212 OpenMPScheduleClauseModifier M1;
2213 OpenMPScheduleClauseModifier M2;
2214 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2215 OpenMPScheduleClauseModifier M1,
2216 OpenMPScheduleClauseModifier M2)
2217 : Kind(Kind), M1(M1), M2(M2) {}
2218 };
2219 } // namespace
2220
2221 bool CodeGenFunction::EmitOMPWorksharingLoop(
2222 const OMPLoopDirective &S, Expr *EUB,
2223 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2224 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2225 // Emit the loop iteration variable.
2226 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2227 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2228 EmitVarDecl(*IVDecl);
2229
2230 // Emit the iterations count variable.
2231 // If it is not a variable, Sema decided to calculate iterations count on each
2232 // iteration (e.g., it is foldable into a constant).
2233 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2234 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2235 // Emit calculation of the iterations count.
2236 EmitIgnoredExpr(S.getCalcLastIteration());
2237 }
2238
2239 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2240
2241 bool HasLastprivateClause;
2242 // Check pre-condition.
2243 {
2244 OMPLoopScope PreInitScope(*this, S);
2245 // Skip the entire loop if we don't meet the precondition.
2246 // If the condition constant folds and can be elided, avoid emitting the
2247 // whole loop.
2248 bool CondConstant;
2249 llvm::BasicBlock *ContBlock = nullptr;
2250 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2251 if (!CondConstant)
2252 return false;
2253 } else {
2254 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2255 ContBlock = createBasicBlock("omp.precond.end");
2256 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2257 getProfileCount(&S));
2258 EmitBlock(ThenBlock);
2259 incrementProfileCounter(&S);
2260 }
2261
2262 RunCleanupsScope DoacrossCleanupScope(*this);
2263 bool Ordered = false;
2264 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2265 if (OrderedClause->getNumForLoops())
2266 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
2267 else
2268 Ordered = true;
2269 }
2270
2271 llvm::DenseSet<const Expr *> EmittedFinals;
2272 emitAlignedClause(*this, S);
2273 bool HasLinears = EmitOMPLinearClauseInit(S);
2274 // Emit helper vars inits.
2275
2276 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2277 LValue LB = Bounds.first;
2278 LValue UB = Bounds.second;
2279 LValue ST =
2280 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2281 LValue IL =
2282 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2283
2284 // Emit 'then' code.
2285 {
2286 OMPPrivateScope LoopScope(*this);
2287 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2288 // Emit implicit barrier to synchronize threads and avoid data races on
2289 // initialization of firstprivate variables and post-update of
2290 // lastprivate variables.
2291 CGM.getOpenMPRuntime().emitBarrierCall(
2292 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2293 /*ForceSimpleCall=*/true);
2294 }
2295 EmitOMPPrivateClause(S, LoopScope);
2296 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2297 EmitOMPReductionClauseInit(S, LoopScope);
2298 EmitOMPPrivateLoopCounters(S, LoopScope);
2299 EmitOMPLinearClause(S, LoopScope);
2300 (void)LoopScope.Privatize();
2301 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2302 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
2303
2304 // Detect the loop schedule kind and chunk.
2305 const Expr *ChunkExpr = nullptr;
2306 OpenMPScheduleTy ScheduleKind;
2307 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2308 ScheduleKind.Schedule = C->getScheduleKind();
2309 ScheduleKind.M1 = C->getFirstScheduleModifier();
2310 ScheduleKind.M2 = C->getSecondScheduleModifier();
2311 ChunkExpr = C->getChunkSize();
2312 } else {
2313 // Default behaviour for schedule clause.
2314 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
2315 *this, S, ScheduleKind.Schedule, ChunkExpr);
2316 }
2317 bool HasChunkSizeOne = false;
2318 llvm::Value *Chunk = nullptr;
2319 if (ChunkExpr) {
2320 Chunk = EmitScalarExpr(ChunkExpr);
2321 Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
2322 S.getIterationVariable()->getType(),
2323 S.getBeginLoc());
2324 Expr::EvalResult Result;
2325 if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
2326 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
2327 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
2328 }
2329 }
2330 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2331 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2332 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2333 // If the static schedule kind is specified or if the ordered clause is
2334 // specified, and if no monotonic modifier is specified, the effect will
2335 // be as if the monotonic modifier was specified.
2336 bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule,
2337 /* Chunked */ Chunk != nullptr) && HasChunkSizeOne &&
2338 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
2339 if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
2340 /* Chunked */ Chunk != nullptr) ||
2341 StaticChunkedOne) &&
2342 !Ordered) {
2343 if (isOpenMPSimdDirective(S.getDirectiveKind()))
2344 EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2345 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2346 // When no chunk_size is specified, the iteration space is divided into
2347 // chunks that are approximately equal in size, and at most one chunk is
2348 // distributed to each thread. Note that the size of the chunks is
2349 // unspecified in this case.
2350 CGOpenMPRuntime::StaticRTInput StaticInit(
2351 IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2352 UB.getAddress(), ST.getAddress(),
2353 StaticChunkedOne ? Chunk : nullptr);
2354 RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2355 ScheduleKind, StaticInit);
2356 JumpDest LoopExit =
2357 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2358 // UB = min(UB, GlobalUB);
2359 if (!StaticChunkedOne)
2360 EmitIgnoredExpr(S.getEnsureUpperBound());
2361 // IV = LB;
2362 EmitIgnoredExpr(S.getInit());
2363 // For unchunked static schedule generate:
2364 //
2365 // while (idx <= UB) {
2366 // BODY;
2367 // ++idx;
2368 // }
2369 //
2370 // For static schedule with chunk one:
2371 //
2372 // while (IV <= PrevUB) {
2373 // BODY;
2374 // IV += ST;
2375 // }
2376 EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
2377 StaticChunkedOne ? S.getCombinedParForInDistCond() : S.getCond(),
2378 StaticChunkedOne ? S.getDistInc() : S.getInc(),
2379 [&S, LoopExit](CodeGenFunction &CGF) {
2380 CGF.EmitOMPLoopBody(S, LoopExit);
2381 CGF.EmitStopPoint(&S);
2382 },
2383 [](CodeGenFunction &) {});
2384 EmitBlock(LoopExit.getBlock());
2385 // Tell the runtime we are done.
2386 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2387 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2388 S.getDirectiveKind());
2389 };
2390 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2391 } else {
2392 const bool IsMonotonic =
2393 Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2394 ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2395 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2396 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2397 // Emit the outer loop, which requests its work chunk [LB..UB] from
2398 // runtime and runs the inner loop to process it.
2399 const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2400 ST.getAddress(), IL.getAddress(),
2401 Chunk, EUB);
2402 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2403 LoopArguments, CGDispatchBounds);
2404 }
2405 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
2406 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
2407 return CGF.Builder.CreateIsNotNull(
2408 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2409 });
2410 }
2411 EmitOMPReductionClauseFinal(
2412 S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2413 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2414 : /*Parallel only*/ OMPD_parallel);
2415 // Emit post-update of the reduction variables if IsLastIter != 0.
2416 emitPostUpdateForReductionClause(
2417 *this, S, [IL, &S](CodeGenFunction &CGF) {
2418 return CGF.Builder.CreateIsNotNull(
2419 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2420 });
2421 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2422 if (HasLastprivateClause)
2423 EmitOMPLastprivateClauseFinal(
2424 S, isOpenMPSimdDirective(S.getDirectiveKind()),
2425 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
2426 }
2427 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2428 return CGF.Builder.CreateIsNotNull(
2429 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2430 });
2431 DoacrossCleanupScope.ForceCleanup();
2432 // We're now done with the loop, so jump to the continuation block.
2433 if (ContBlock) {
2434 EmitBranch(ContBlock);
2435 EmitBlock(ContBlock, /*IsFinished=*/true);
2436 }
2437 }
2438 return HasLastprivateClause;
2439 }
2440
2441 /// The following two functions generate expressions for the loop lower
2442 /// and upper bounds in case of static and dynamic (dispatch) schedule
2443 /// of the associated 'for' or 'distribute' loop.
2444 static std::pair<LValue, LValue>
2445 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
2446 const auto &LS = cast<OMPLoopDirective>(S);
2447 LValue LB =
2448 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2449 LValue UB =
2450 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2451 return {LB, UB};
2452 }
2453
2454 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2455 /// consider the lower and upper bound expressions generated by the
2456 /// worksharing loop support, but we use 0 and the iteration space size as
2457 /// constants
2458 static std::pair<llvm::Value *, llvm::Value *>
2459 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
2460 Address LB, Address UB) {
2461 const auto &LS = cast<OMPLoopDirective>(S);
2462 const Expr *IVExpr = LS.getIterationVariable();
2463 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2464 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2465 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2466 return {LBVal, UBVal};
2467 }
2468
2469 void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
2470 bool HasLastprivates = false;
2471 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2472 PrePostActionTy &) {
2473 OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2474 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2475 emitForLoopBounds,
2476 emitDispatchForLoopBounds);
2477 };
2478 {
2479 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2480 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2481 S.hasCancel());
2482 }
2483
2484 // Emit an implicit barrier at the end.
2485 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2486 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2487 }
2488
2489 void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
2490 bool HasLastprivates = false;
2491 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2492 PrePostActionTy &) {
2493 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2494 emitForLoopBounds,
2495 emitDispatchForLoopBounds);
2496 };
2497 {
2498 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2499 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2500 }
2501
2502 // Emit an implicit barrier at the end.
2503 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2504 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2505 }
2506
2507 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
2508 const Twine &Name,
2509 llvm::Value *Init = nullptr) {
2510 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2511 if (Init)
2512 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2513 return LVal;
2514 }
2515
2516 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2517 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
2518 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2519 bool HasLastprivates = false;
2520 auto &&CodeGen = [&S, CapturedStmt, CS,
2521 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2522 ASTContext &C = CGF.getContext();
2523 QualType KmpInt32Ty =
2524 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2525 // Emit helper vars inits.
2526 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2527 CGF.Builder.getInt32(0));
2528 llvm::ConstantInt *GlobalUBVal = CS != nullptr
2529 ? CGF.Builder.getInt32(CS->size() - 1)
2530 : CGF.Builder.getInt32(0);
2531 LValue UB =
2532 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2533 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2534 CGF.Builder.getInt32(1));
2535 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2536 CGF.Builder.getInt32(0));
2537 // Loop counter.
2538 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2539 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2540 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2541 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2542 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2543 // Generate condition for loop.
2544 BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2545 OK_Ordinary, S.getBeginLoc(), FPOptions());
2546 // Increment for loop counter.
2547 UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2548 S.getBeginLoc(), true);
2549 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2550 // Iterate through all sections and emit a switch construct:
2551 // switch (IV) {
2552 // case 0:
2553 // <SectionStmt[0]>;
2554 // break;
2555 // ...
2556 // case <NumSection> - 1:
2557 // <SectionStmt[<NumSection> - 1]>;
2558 // break;
2559 // }
2560 // .omp.sections.exit:
2561 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2562 llvm::SwitchInst *SwitchStmt =
2563 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
2564 ExitBB, CS == nullptr ? 1 : CS->size());
2565 if (CS) {
2566 unsigned CaseNumber = 0;
2567 for (const Stmt *SubStmt : CS->children()) {
2568 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2569 CGF.EmitBlock(CaseBB);
2570 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2571 CGF.EmitStmt(SubStmt);
2572 CGF.EmitBranch(ExitBB);
2573 ++CaseNumber;
2574 }
2575 } else {
2576 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2577 CGF.EmitBlock(CaseBB);
2578 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2579 CGF.EmitStmt(CapturedStmt);
2580 CGF.EmitBranch(ExitBB);
2581 }
2582 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2583 };
2584
2585 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2586 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2587 // Emit implicit barrier to synchronize threads and avoid data races on
2588 // initialization of firstprivate variables and post-update of lastprivate
2589 // variables.
2590 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2591 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2592 /*ForceSimpleCall=*/true);
2593 }
2594 CGF.EmitOMPPrivateClause(S, LoopScope);
2595 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2596 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2597 (void)LoopScope.Privatize();
2598 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2599 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2600
2601 // Emit static non-chunked loop.
2602 OpenMPScheduleTy ScheduleKind;
2603 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2604 CGOpenMPRuntime::StaticRTInput StaticInit(
2605 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2606 LB.getAddress(), UB.getAddress(), ST.getAddress());
2607 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2608 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2609 // UB = min(UB, GlobalUB);
2610 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
2611 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2612 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2613 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2614 // IV = LB;
2615 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
2616 // while (idx <= UB) { BODY; ++idx; }
2617 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2618 [](CodeGenFunction &) {});
2619 // Tell the runtime we are done.
2620 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2621 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2622 S.getDirectiveKind());
2623 };
2624 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2625 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2626 // Emit post-update of the reduction variables if IsLastIter != 0.
2627 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2628 return CGF.Builder.CreateIsNotNull(
2629 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2630 });
2631
2632 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2633 if (HasLastprivates)
2634 CGF.EmitOMPLastprivateClauseFinal(
2635 S, /*NoFinals=*/false,
2636 CGF.Builder.CreateIsNotNull(
2637 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
2638 };
2639
2640 bool HasCancel = false;
2641 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2642 HasCancel = OSD->hasCancel();
2643 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2644 HasCancel = OPSD->hasCancel();
2645 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2646 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2647 HasCancel);
2648 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2649 // clause. Otherwise the barrier will be generated by the codegen for the
2650 // directive.
2651 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2652 // Emit implicit barrier to synchronize threads and avoid data races on
2653 // initialization of firstprivate variables.
2654 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2655 OMPD_unknown);
2656 }
2657 }
2658
2659 void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
2660 {
2661 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2662 EmitSections(S);
2663 }
2664 // Emit an implicit barrier at the end.
2665 if (!S.getSingleClause<OMPNowaitClause>()) {
2666 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2667 OMPD_sections);
2668 }
2669 }
2670
2671 void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
2672 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2673 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2674 };
2675 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2676 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2677 S.hasCancel());
2678 }
2679
2680 void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
2681 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2682 llvm::SmallVector<const Expr *, 8> DestExprs;
2683 llvm::SmallVector<const Expr *, 8> SrcExprs;
2684 llvm::SmallVector<const Expr *, 8> AssignmentOps;
2685 // Check if there are any 'copyprivate' clauses associated with this
2686 // 'single' construct.
2687 // Build a list of copyprivate variables along with helper expressions
2688 // (<source>, <destination>, <destination>=<source> expressions)
2689 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2690 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2691 DestExprs.append(C->destination_exprs().begin(),
2692 C->destination_exprs().end());
2693 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2694 AssignmentOps.append(C->assignment_ops().begin(),
2695 C->assignment_ops().end());
2696 }
2697 // Emit code for 'single' region along with 'copyprivate' clauses
2698 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2699 Action.Enter(CGF);
2700 OMPPrivateScope SingleScope(CGF);
2701 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2702 CGF.EmitOMPPrivateClause(S, SingleScope);
2703 (void)SingleScope.Privatize();
2704 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2705 };
2706 {
2707 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2708 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
2709 CopyprivateVars, DestExprs,
2710 SrcExprs, AssignmentOps);
2711 }
2712 // Emit an implicit barrier at the end (to avoid data race on firstprivate
2713 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2714 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2715 CGM.getOpenMPRuntime().emitBarrierCall(
2716 *this, S.getBeginLoc(),
2717 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2718 }
2719 }
2720
2721 void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
2722 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2723 Action.Enter(CGF);
2724 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2725 };
2726 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2727 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
2728 }
2729
2730 void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
2731 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2732 Action.Enter(CGF);
2733 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2734 };
2735 const Expr *Hint = nullptr;
2736 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
2737 Hint = HintClause->getHint();
2738 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2739 CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2740 S.getDirectiveName().getAsString(),
2741 CodeGen, S.getBeginLoc(), Hint);
2742 }
2743
2744 void CodeGenFunction::EmitOMPParallelForDirective(
2745 const OMPParallelForDirective &S) {
2746 // Emit directive as a combined directive that consists of two implicit
2747 // directives: 'parallel' with 'for' directive.
2748 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2749 Action.Enter(CGF);
2750 OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2751 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2752 emitDispatchForLoopBounds);
2753 };
2754 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2755 emitEmptyBoundParameters);
2756 }
2757
2758 void CodeGenFunction::EmitOMPParallelForSimdDirective(
2759 const OMPParallelForSimdDirective &S) {
2760 // Emit directive as a combined directive that consists of two implicit
2761 // directives: 'parallel' with 'for' directive.
2762 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2763 Action.Enter(CGF);
2764 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
2765 emitDispatchForLoopBounds);
2766 };
2767 emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2768 emitEmptyBoundParameters);
2769 }
2770
2771 void CodeGenFunction::EmitOMPParallelSectionsDirective(
2772 const OMPParallelSectionsDirective &S) {
2773 // Emit directive as a combined directive that consists of two implicit
2774 // directives: 'parallel' with 'sections' directive.
2775 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2776 Action.Enter(CGF);
2777 CGF.EmitSections(S);
2778 };
2779 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2780 emitEmptyBoundParameters);
2781 }
2782
2783 void CodeGenFunction::EmitOMPTaskBasedDirective(
2784 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2785 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2786 OMPTaskDataTy &Data) {
2787 // Emit outlined function for task construct.
2788 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
2789 auto I = CS->getCapturedDecl()->param_begin();
2790 auto PartId = std::next(I);
2791 auto TaskT = std::next(I, 4);
2792 // Check if the task is final
2793 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2794 // If the condition constant folds and can be elided, try to avoid emitting
2795 // the condition and the dead arm of the if/else.
2796 const Expr *Cond = Clause->getCondition();
2797 bool CondConstant;
2798 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2799 Data.Final.setInt(CondConstant);
2800 else
2801 Data.Final.setPointer(EvaluateExprAsBool(Cond));
2802 } else {
2803 // By default the task is not final.
2804 Data.Final.setInt(/*IntVal=*/false);
2805 }
2806 // Check if the task has 'priority' clause.
2807 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2808 const Expr *Prio = Clause->getPriority();
2809 Data.Priority.setInt(/*IntVal=*/true);
2810 Data.Priority.setPointer(EmitScalarConversion(
2811 EmitScalarExpr(Prio), Prio->getType(),
2812 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2813 Prio->getExprLoc()));
2814 }
2815 // The first function argument for tasks is a thread id, the second one is a
2816 // part id (0 for tied tasks, >=0 for untied task).
2817 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2818 // Get list of private variables.
2819 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2820 auto IRef = C->varlist_begin();
2821 for (const Expr *IInit : C->private_copies()) {
2822 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2823 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2824 Data.PrivateVars.push_back(*IRef);
2825 Data.PrivateCopies.push_back(IInit);
2826 }
2827 ++IRef;
2828 }
2829 }
2830 EmittedAsPrivate.clear();
2831 // Get list of firstprivate variables.
2832 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2833 auto IRef = C->varlist_begin();
2834 auto IElemInitRef = C->inits().begin();
2835 for (const Expr *IInit : C->private_copies()) {
2836 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2837 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2838 Data.FirstprivateVars.push_back(*IRef);
2839 Data.FirstprivateCopies.push_back(IInit);
2840 Data.FirstprivateInits.push_back(*IElemInitRef);
2841 }
2842 ++IRef;
2843 ++IElemInitRef;
2844 }
2845 }
2846 // Get list of lastprivate variables (for taskloops).
2847 llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2848 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2849 auto IRef = C->varlist_begin();
2850 auto ID = C->destination_exprs().begin();
2851 for (const Expr *IInit : C->private_copies()) {
2852 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2853 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2854 Data.LastprivateVars.push_back(*IRef);
2855 Data.LastprivateCopies.push_back(IInit);
2856 }
2857 LastprivateDstsOrigs.insert(
2858 {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2859 cast<DeclRefExpr>(*IRef)});
2860 ++IRef;
2861 ++ID;
2862 }
2863 }
2864 SmallVector<const Expr *, 4> LHSs;
2865 SmallVector<const Expr *, 4> RHSs;
2866 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2867 auto IPriv = C->privates().begin();
2868 auto IRed = C->reduction_ops().begin();
2869 auto ILHS = C->lhs_exprs().begin();
2870 auto IRHS = C->rhs_exprs().begin();
2871 for (const Expr *Ref : C->varlists()) {
2872 Data.ReductionVars.emplace_back(Ref);
2873 Data.ReductionCopies.emplace_back(*IPriv);
2874 Data.ReductionOps.emplace_back(*IRed);
2875 LHSs.emplace_back(*ILHS);
2876 RHSs.emplace_back(*IRHS);
2877 std::advance(IPriv, 1);
2878 std::advance(IRed, 1);
2879 std::advance(ILHS, 1);
2880 std::advance(IRHS, 1);
2881 }
2882 }
2883 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2884 *this, S.getBeginLoc(), LHSs, RHSs, Data);
2885 // Build list of dependences.
2886 for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2887 for (const Expr *IRef : C->varlists())
2888 Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
2889 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
2890 CapturedRegion](CodeGenFunction &CGF,
2891 PrePostActionTy &Action) {
2892 // Set proper addresses for generated private copies.
2893 OMPPrivateScope Scope(CGF);
2894 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2895 !Data.LastprivateVars.empty()) {
2896 enum { PrivatesParam = 2, CopyFnParam = 3 };
2897 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
2898 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
2899 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
2900 CS->getCapturedDecl()->getParam(PrivatesParam)));
2901 // Map privates.
2902 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
2903 llvm::SmallVector<llvm::Value *, 16> CallArgs;
2904 CallArgs.push_back(PrivatesPtr);
2905 for (const Expr *E : Data.PrivateVars) {
2906 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2907 Address PrivatePtr = CGF.CreateMemTemp(
2908 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2909 PrivatePtrs.emplace_back(VD, PrivatePtr);
2910 CallArgs.push_back(PrivatePtr.getPointer());
2911 }
2912 for (const Expr *E : Data.FirstprivateVars) {
2913 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2914 Address PrivatePtr =
2915 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2916 ".firstpriv.ptr.addr");
2917 PrivatePtrs.emplace_back(VD, PrivatePtr);
2918 CallArgs.push_back(PrivatePtr.getPointer());
2919 }
2920 for (const Expr *E : Data.LastprivateVars) {
2921 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2922 Address PrivatePtr =
2923 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2924 ".lastpriv.ptr.addr");
2925 PrivatePtrs.emplace_back(VD, PrivatePtr);
2926 CallArgs.push_back(PrivatePtr.getPointer());
2927 }
2928 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
2929 CopyFn, CallArgs);
2930 for (const auto &Pair : LastprivateDstsOrigs) {
2931 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2932 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
2933 /*RefersToEnclosingVariableOrCapture=*/
2934 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
2935 Pair.second->getType(), VK_LValue,
2936 Pair.second->getExprLoc());
2937 Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2938 return CGF.EmitLValue(&DRE).getAddress();
2939 });
2940 }
2941 for (const auto &Pair : PrivatePtrs) {
2942 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2943 CGF.getContext().getDeclAlign(Pair.first));
2944 Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2945 }
2946 }
2947 if (Data.Reductions) {
2948 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
2949 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionCopies,
2950 Data.ReductionOps);
2951 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2952 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2953 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2954 RedCG.emitSharedLValue(CGF, Cnt);
2955 RedCG.emitAggregateType(CGF, Cnt);
2956 // FIXME: This must removed once the runtime library is fixed.
2957 // Emit required threadprivate variables for
2958 // initializer/combiner/finalizer.
2959 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
2960 RedCG, Cnt);
2961 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2962 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2963 Replacement =
2964 Address(CGF.EmitScalarConversion(
2965 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2966 CGF.getContext().getPointerType(
2967 Data.ReductionCopies[Cnt]->getType()),
2968 Data.ReductionCopies[Cnt]->getExprLoc()),
2969 Replacement.getAlignment());
2970 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2971 Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2972 [Replacement]() { return Replacement; });
2973 }
2974 }
2975 // Privatize all private variables except for in_reduction items.
2976 (void)Scope.Privatize();
2977 SmallVector<const Expr *, 4> InRedVars;
2978 SmallVector<const Expr *, 4> InRedPrivs;
2979 SmallVector<const Expr *, 4> InRedOps;
2980 SmallVector<const Expr *, 4> TaskgroupDescriptors;
2981 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2982 auto IPriv = C->privates().begin();
2983 auto IRed = C->reduction_ops().begin();
2984 auto ITD = C->taskgroup_descriptors().begin();
2985 for (const Expr *Ref : C->varlists()) {
2986 InRedVars.emplace_back(Ref);
2987 InRedPrivs.emplace_back(*IPriv);
2988 InRedOps.emplace_back(*IRed);
2989 TaskgroupDescriptors.emplace_back(*ITD);
2990 std::advance(IPriv, 1);
2991 std::advance(IRed, 1);
2992 std::advance(ITD, 1);
2993 }
2994 }
2995 // Privatize in_reduction items here, because taskgroup descriptors must be
2996 // privatized earlier.
2997 OMPPrivateScope InRedScope(CGF);
2998 if (!InRedVars.empty()) {
2999 ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
3000 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
3001 RedCG.emitSharedLValue(CGF, Cnt);
3002 RedCG.emitAggregateType(CGF, Cnt);
3003 // The taskgroup descriptor variable is always implicit firstprivate and
3004 // privatized already during processing of the firstprivates.
3005 // FIXME: This must removed once the runtime library is fixed.
3006 // Emit required threadprivate variables for
3007 // initializer/combiner/finalizer.
3008 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3009 RedCG, Cnt);
3010 llvm::Value *ReductionsPtr =
3011 CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
3012 TaskgroupDescriptors[Cnt]->getExprLoc());
3013 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3014 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3015 Replacement = Address(
3016 CGF.EmitScalarConversion(
3017 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3018 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
3019 InRedPrivs[Cnt]->getExprLoc()),
3020 Replacement.getAlignment());
3021 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3022 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
3023 [Replacement]() { return Replacement; });
3024 }
3025 }
3026 (void)InRedScope.Privatize();
3027
3028 Action.Enter(CGF);
3029 BodyGen(CGF);
3030 };
3031 llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3032 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
3033 Data.NumberOfParts);
3034 OMPLexicalScope Scope(*this, S);
3035 TaskGen(*this, OutlinedFn, Data);
3036 }
3037
3038 static ImplicitParamDecl *
3039 createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
3040 QualType Ty, CapturedDecl *CD,
3041 SourceLocation Loc) {
3042 auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3043 ImplicitParamDecl::Other);
3044 auto *OrigRef = DeclRefExpr::Create(
3045 C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3046 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3047 auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3048 ImplicitParamDecl::Other);
3049 auto *PrivateRef = DeclRefExpr::Create(
3050 C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
3051 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3052 QualType ElemType = C.getBaseElementType(Ty);
3053 auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3054 ImplicitParamDecl::Other);
3055 auto *InitRef = DeclRefExpr::Create(
3056 C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3057 /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3058 PrivateVD->setInitStyle(VarDecl::CInit);
3059 PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3060 InitRef, /*BasePath=*/nullptr,
3061 VK_RValue));
3062 Data.FirstprivateVars.emplace_back(OrigRef);
3063 Data.FirstprivateCopies.emplace_back(PrivateRef);
3064 Data.FirstprivateInits.emplace_back(InitRef);
3065 return OrigVD;
3066 }
3067
3068 void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
3069 const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3070 OMPTargetDataInfo &InputInfo) {
3071 // Emit outlined function for task construct.
3072 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3073 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3074 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3075 auto I = CS->getCapturedDecl()->param_begin();
3076 auto PartId = std::next(I);
3077 auto TaskT = std::next(I, 4);
3078 OMPTaskDataTy Data;
3079 // The task is not final.
3080 Data.Final.setInt(/*IntVal=*/false);
3081 // Get list of firstprivate variables.
3082 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3083 auto IRef = C->varlist_begin();
3084 auto IElemInitRef = C->inits().begin();
3085 for (auto *IInit : C->private_copies()) {
3086 Data.FirstprivateVars.push_back(*IRef);
3087 Data.FirstprivateCopies.push_back(IInit);
3088 Data.FirstprivateInits.push_back(*IElemInitRef);
3089 ++IRef;
3090 ++IElemInitRef;
3091 }
3092 }
3093 OMPPrivateScope TargetScope(*this);
3094 VarDecl *BPVD = nullptr;
3095 VarDecl *PVD = nullptr;
3096 VarDecl *SVD = nullptr;
3097 if (InputInfo.NumberOfTargetItems > 0) {
3098 auto *CD = CapturedDecl::Create(
3099 getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3100 llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3101 QualType BaseAndPointersType = getContext().getConstantArrayType(
3102 getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3103 /*IndexTypeQuals=*/0);
3104 BPVD = createImplicitFirstprivateForType(
3105 getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3106 PVD = createImplicitFirstprivateForType(
3107 getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3108 QualType SizesType = getContext().getConstantArrayType(
3109 getContext().getSizeType(), ArrSize, ArrayType::Normal,
3110 /*IndexTypeQuals=*/0);
3111 SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3112 S.getBeginLoc());
3113 TargetScope.addPrivate(
3114 BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3115 TargetScope.addPrivate(PVD,
3116 [&InputInfo]() { return InputInfo.PointersArray; });
3117 TargetScope.addPrivate(SVD,
3118 [&InputInfo]() { return InputInfo.SizesArray; });
3119 }
3120 (void)TargetScope.Privatize();
3121 // Build list of dependences.
3122 for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3123 for (const Expr *IRef : C->varlists())
3124 Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3125 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3126 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3127 // Set proper addresses for generated private copies.
3128 OMPPrivateScope Scope(CGF);
3129 if (!Data.FirstprivateVars.empty()) {
3130 enum { PrivatesParam = 2, CopyFnParam = 3 };
3131 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3132 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3133 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3134 CS->getCapturedDecl()->getParam(PrivatesParam)));
3135 // Map privates.
3136 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
3137 llvm::SmallVector<llvm::Value *, 16> CallArgs;
3138 CallArgs.push_back(PrivatesPtr);
3139 for (const Expr *E : Data.FirstprivateVars) {
3140 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3141 Address PrivatePtr =
3142 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3143 ".firstpriv.ptr.addr");
3144 PrivatePtrs.emplace_back(VD, PrivatePtr);
3145 CallArgs.push_back(PrivatePtr.getPointer());
3146 }
3147 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3148 CopyFn, CallArgs);
3149 for (const auto &Pair : PrivatePtrs) {
3150 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3151 CGF.getContext().getDeclAlign(Pair.first));
3152 Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3153 }
3154 }
3155 // Privatize all private variables except for in_reduction items.
3156 (void)Scope.Privatize();
3157 if (InputInfo.NumberOfTargetItems > 0) {
3158 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3159 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize());
3160 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3161 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize());
3162 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3163 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize());
3164 }
3165
3166 Action.Enter(CGF);
3167 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3168 BodyGen(CGF);
3169 };
3170 llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3171 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3172 Data.NumberOfParts);
3173 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3174 IntegerLiteral IfCond(getContext(), TrueOrFalse,
3175 getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3176 SourceLocation());
3177
3178 CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
3179 SharedsTy, CapturedStruct, &IfCond, Data);
3180 }
3181
3182 void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
3183 // Emit outlined function for task construct.
3184 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3185 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3186 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3187 const Expr *IfCond = nullptr;
3188 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3189 if (C->getNameModifier() == OMPD_unknown ||
3190 C->getNameModifier() == OMPD_task) {
3191 IfCond = C->getCondition();
3192 break;
3193 }
3194 }
3195
3196 OMPTaskDataTy Data;
3197 // Check if we should emit tied or untied task.
3198 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3199 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3200 CGF.EmitStmt(CS->getCapturedStmt());
3201 };
3202 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3203 IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
3204 const OMPTaskDataTy &Data) {
3205 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
3206 SharedsTy, CapturedStruct, IfCond,
3207 Data);
3208 };
3209 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3210 }
3211
3212 void CodeGenFunction::EmitOMPTaskyieldDirective(
3213 const OMPTaskyieldDirective &S) {
3214 CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
3215 }
3216
3217 void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
3218 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
3219 }
3220
3221 void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
3222 CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
3223 }
3224
3225 void CodeGenFunction::EmitOMPTaskgroupDirective(
3226 const OMPTaskgroupDirective &S) {
3227 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3228 Action.Enter(CGF);
3229 if (const Expr *E = S.getReductionRef()) {
3230 SmallVector<const Expr *, 4> LHSs;
3231 SmallVector<const Expr *, 4> RHSs;
3232 OMPTaskDataTy Data;
3233 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3234 auto IPriv = C->privates().begin();
3235 auto IRed = C->reduction_ops().begin();
3236 auto ILHS = C->lhs_exprs().begin();
3237 auto IRHS = C->rhs_exprs().begin();
3238 for (const Expr *Ref : C->varlists()) {
3239 Data.ReductionVars.emplace_back(Ref);
3240 Data.ReductionCopies.emplace_back(*IPriv);
3241 Data.ReductionOps.emplace_back(*IRed);
3242 LHSs.emplace_back(*ILHS);
3243 RHSs.emplace_back(*IRHS);
3244 std::advance(IPriv, 1);
3245 std::advance(IRed, 1);
3246 std::advance(ILHS, 1);
3247 std::advance(IRHS, 1);
3248 }
3249 }
3250 llvm::Value *ReductionDesc =
3251 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
3252 LHSs, RHSs, Data);
3253 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3254 CGF.EmitVarDecl(*VD);
3255 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3256 /*Volatile=*/false, E->getType());
3257 }
3258 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
3259 };
3260 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3261 CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
3262 }
3263
3264 void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
3265 CGM.getOpenMPRuntime().emitFlush(
3266 *this,
3267 [&S]() -> ArrayRef<const Expr *> {
3268 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3269 return llvm::makeArrayRef(FlushClause->varlist_begin(),
3270 FlushClause->varlist_end());
3271 return llvm::None;
3272 }(),
3273 S.getBeginLoc());
3274 }
3275
3276 void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
3277 const CodeGenLoopTy &CodeGenLoop,
3278 Expr *IncExpr) {
3279 // Emit the loop iteration variable.
3280 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3281 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3282 EmitVarDecl(*IVDecl);
3283
3284 // Emit the iterations count variable.
3285 // If it is not a variable, Sema decided to calculate iterations count on each
3286 // iteration (e.g., it is foldable into a constant).
3287 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3288 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3289 // Emit calculation of the iterations count.
3290 EmitIgnoredExpr(S.getCalcLastIteration());
3291 }
3292
3293 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3294
3295 bool HasLastprivateClause = false;
3296 // Check pre-condition.
3297 {
3298 OMPLoopScope PreInitScope(*this, S);
3299 // Skip the entire loop if we don't meet the precondition.
3300 // If the condition constant folds and can be elided, avoid emitting the
3301 // whole loop.
3302 bool CondConstant;
3303 llvm::BasicBlock *ContBlock = nullptr;
3304 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3305 if (!CondConstant)
3306 return;
3307 } else {
3308 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3309 ContBlock = createBasicBlock("omp.precond.end");
3310 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3311 getProfileCount(&S));
3312 EmitBlock(ThenBlock);
3313 incrementProfileCounter(&S);
3314 }
3315
3316 emitAlignedClause(*this, S);
3317 // Emit 'then' code.
3318 {
3319 // Emit helper vars inits.
3320
3321 LValue LB = EmitOMPHelperVar(
3322 *this, cast<DeclRefExpr>(
3323 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3324 ? S.getCombinedLowerBoundVariable()
3325 : S.getLowerBoundVariable())));
3326 LValue UB = EmitOMPHelperVar(
3327 *this, cast<DeclRefExpr>(
3328 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3329 ? S.getCombinedUpperBoundVariable()
3330 : S.getUpperBoundVariable())));
3331 LValue ST =
3332 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3333 LValue IL =
3334 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3335
3336 OMPPrivateScope LoopScope(*this);
3337 if (EmitOMPFirstprivateClause(S, LoopScope)) {
3338 // Emit implicit barrier to synchronize threads and avoid data races
3339 // on initialization of firstprivate variables and post-update of
3340 // lastprivate variables.
3341 CGM.getOpenMPRuntime().emitBarrierCall(
3342 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3343 /*ForceSimpleCall=*/true);
3344 }
3345 EmitOMPPrivateClause(S, LoopScope);
3346 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
3347 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
3348 !isOpenMPTeamsDirective(S.getDirectiveKind()))
3349 EmitOMPReductionClauseInit(S, LoopScope);
3350 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3351 EmitOMPPrivateLoopCounters(S, LoopScope);
3352 (void)LoopScope.Privatize();
3353 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
3354 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
3355
3356 // Detect the distribute schedule kind and chunk.
3357 llvm::Value *Chunk = nullptr;
3358 OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
3359 if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3360 ScheduleKind = C->getDistScheduleKind();
3361 if (const Expr *Ch = C->getChunkSize()) {
3362 Chunk = EmitScalarExpr(Ch);
3363 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3364 S.getIterationVariable()->getType(),
3365 S.getBeginLoc());
3366 }
3367 } else {
3368 // Default behaviour for dist_schedule clause.
3369 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
3370 *this, S, ScheduleKind, Chunk);
3371 }
3372 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3373 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3374
3375 // OpenMP [2.10.8, distribute Construct, Description]
3376 // If dist_schedule is specified, kind must be static. If specified,
3377 // iterations are divided into chunks of size chunk_size, chunks are
3378 // assigned to the teams of the league in a round-robin fashion in the
3379 // order of the team number. When no chunk_size is specified, the
3380 // iteration space is divided into chunks that are approximately equal
3381 // in size, and at most one chunk is distributed to each team of the
3382 // league. The size of the chunks is unspecified in this case.
3383 bool StaticChunked = RT.isStaticChunked(
3384 ScheduleKind, /* Chunked */ Chunk != nullptr) &&
3385 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
3386 if (RT.isStaticNonchunked(ScheduleKind,
3387 /* Chunked */ Chunk != nullptr) ||
3388 StaticChunked) {
3389 if (isOpenMPSimdDirective(S.getDirectiveKind()))
3390 EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3391 CGOpenMPRuntime::StaticRTInput StaticInit(
3392 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3393 LB.getAddress(), UB.getAddress(), ST.getAddress(),
3394 StaticChunked ? Chunk : nullptr);
3395 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
3396 StaticInit);
3397 JumpDest LoopExit =
3398 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3399 // UB = min(UB, GlobalUB);
3400 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3401 ? S.getCombinedEnsureUpperBound()
3402 : S.getEnsureUpperBound());
3403 // IV = LB;
3404 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3405 ? S.getCombinedInit()
3406 : S.getInit());
3407
3408 const Expr *Cond =
3409 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3410 ? S.getCombinedCond()
3411 : S.getCond();
3412
3413 if (StaticChunked)
3414 Cond = S.getCombinedDistCond();
3415
3416 // For static unchunked schedules generate:
3417 //
3418 // 1. For distribute alone, codegen
3419 // while (idx <= UB) {
3420 // BODY;
3421 // ++idx;
3422 // }
3423 //
3424 // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
3425 // while (idx <= UB) {
3426 // <CodeGen rest of pragma>(LB, UB);
3427 // idx += ST;
3428 // }
3429 //
3430 // For static chunk one schedule generate:
3431 //
3432 // while (IV <= GlobalUB) {
3433 // <CodeGen rest of pragma>(LB, UB);
3434 // LB += ST;
3435 // UB += ST;
3436 // UB = min(UB, GlobalUB);
3437 // IV = LB;
3438 // }
3439 //
3440 EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3441 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3442 CodeGenLoop(CGF, S, LoopExit);
3443 },
3444 [&S, StaticChunked](CodeGenFunction &CGF) {
3445 if (StaticChunked) {
3446 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
3447 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
3448 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
3449 CGF.EmitIgnoredExpr(S.getCombinedInit());
3450 }
3451 });
3452 EmitBlock(LoopExit.getBlock());
3453 // Tell the runtime we are done.
3454 RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
3455 } else {
3456 // Emit the outer loop, which requests its work chunk [LB..UB] from
3457 // runtime and runs the inner loop to process it.
3458 const OMPLoopArguments LoopArguments = {
3459 LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3460 Chunk};
3461 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3462 CodeGenLoop);
3463 }
3464 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3465 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3466 return CGF.Builder.CreateIsNotNull(
3467 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3468 });
3469 }
3470 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
3471 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
3472 !isOpenMPTeamsDirective(S.getDirectiveKind())) {
3473 EmitOMPReductionClauseFinal(S, OMPD_simd);
3474 // Emit post-update of the reduction variables if IsLastIter != 0.
3475 emitPostUpdateForReductionClause(
3476 *this, S, [IL, &S](CodeGenFunction &CGF) {
3477 return CGF.Builder.CreateIsNotNull(
3478 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3479 });
3480 }
3481 // Emit final copy of the lastprivate variables if IsLastIter != 0.
3482 if (HasLastprivateClause) {
3483 EmitOMPLastprivateClauseFinal(
3484 S, /*NoFinals=*/false,
3485 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3486 }
3487 }
3488
3489 // We're now done with the loop, so jump to the continuation block.
3490 if (ContBlock) {
3491 EmitBranch(ContBlock);
3492 EmitBlock(ContBlock, true);
3493 }
3494 }
3495 }
3496
3497 void CodeGenFunction::EmitOMPDistributeDirective(
3498 const OMPDistributeDirective &S) {
3499 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3500 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
3501 };
3502 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3503 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3504 }
3505
3506 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3507 const CapturedStmt *S) {
3508 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3509 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
3510 CGF.CapturedStmtInfo = &CapStmtInfo;
3511 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3512 Fn->setDoesNotRecurse();
3513 return Fn;
3514 }
3515
3516 void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
3517 if (S.hasClausesOfKind<OMPDependClause>()) {
3518 assert(!S.getAssociatedStmt() &&
3519 "No associated statement must be in ordered depend construct.");
3520 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3521 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3522 return;
3523 }
3524 const auto *C = S.getSingleClause<OMPSIMDClause>();
3525 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3526 PrePostActionTy &Action) {
3527 const CapturedStmt *CS = S.getInnermostCapturedStmt();
3528 if (C) {
3529 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3530 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3531 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3532 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3533 OutlinedFn, CapturedVars);
3534 } else {
3535 Action.Enter(CGF);
3536 CGF.EmitStmt(CS->getCapturedStmt());
3537 }
3538 };
3539 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3540 CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
3541 }
3542
3543 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
3544 QualType SrcType, QualType DestType,
3545 SourceLocation Loc) {
3546 assert(CGF.hasScalarEvaluationKind(DestType) &&
3547 "DestType must have scalar evaluation kind.");
3548 assert(!Val.isAggregate() && "Must be a scalar or complex.");
3549 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3550 DestType, Loc)
3551 : CGF.EmitComplexToScalarConversion(
3552 Val.getComplexVal(), SrcType, DestType, Loc);
3553 }
3554
3555 static CodeGenFunction::ComplexPairTy
3556 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
3557 QualType DestType, SourceLocation Loc) {
3558 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3559 "DestType must have complex evaluation kind.");
3560 CodeGenFunction::ComplexPairTy ComplexVal;
3561 if (Val.isScalar()) {
3562 // Convert the input element to the element type of the complex.
3563 QualType DestElementType =
3564 DestType->castAs<ComplexType>()->getElementType();
3565 llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3566 Val.getScalarVal(), SrcType, DestElementType, Loc);
3567 ComplexVal = CodeGenFunction::ComplexPairTy(
3568 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3569 } else {
3570 assert(Val.isComplex() && "Must be a scalar or complex.");
3571 QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3572 QualType DestElementType =
3573 DestType->castAs<ComplexType>()->getElementType();
3574 ComplexVal.first = CGF.EmitScalarConversion(
3575 Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3576 ComplexVal.second = CGF.EmitScalarConversion(
3577 Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3578 }
3579 return ComplexVal;
3580 }
3581
3582 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3583 LValue LVal, RValue RVal) {
3584 if (LVal.isGlobalReg()) {
3585 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3586 } else {
3587 CGF.EmitAtomicStore(RVal, LVal,
3588 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3589 : llvm::AtomicOrdering::Monotonic,
3590 LVal.isVolatile(), /*IsInit=*/false);
3591 }
3592 }
3593
3594 void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
3595 QualType RValTy, SourceLocation Loc) {
3596 switch (getEvaluationKind(LVal.getType())) {
3597 case TEK_Scalar:
3598 EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3599 *this, RVal, RValTy, LVal.getType(), Loc)),
3600 LVal);
3601 break;
3602 case TEK_Complex:
3603 EmitStoreOfComplex(
3604 convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3605 /*isInit=*/false);
3606 break;
3607 case TEK_Aggregate:
3608 llvm_unreachable("Must be a scalar or complex.");
3609 }
3610 }
3611
3612 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3613 const Expr *X, const Expr *V,
3614 SourceLocation Loc) {
3615 // v = x;
3616 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3617 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3618 LValue XLValue = CGF.EmitLValue(X);
3619 LValue VLValue = CGF.EmitLValue(V);
3620 RValue Res = XLValue.isGlobalReg()
3621 ? CGF.EmitLoadOfLValue(XLValue, Loc)
3622 : CGF.EmitAtomicLoad(
3623 XLValue, Loc,
3624 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3625 : llvm::AtomicOrdering::Monotonic,
3626 XLValue.isVolatile());
3627 // OpenMP, 2.12.6, atomic Construct
3628 // Any atomic construct with a seq_cst clause forces the atomically
3629 // performed operation to include an implicit flush operation without a
3630 // list.
3631 if (IsSeqCst)
3632 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3633 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3634 }
3635
3636 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3637 const Expr *X, const Expr *E,
3638 SourceLocation Loc) {
3639 // x = expr;
3640 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3641 emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3642 // OpenMP, 2.12.6, atomic Construct
3643 // Any atomic construct with a seq_cst clause forces the atomically
3644 // performed operation to include an implicit flush operation without a
3645 // list.
3646 if (IsSeqCst)
3647 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3648 }
3649
3650 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3651 RValue Update,
3652 BinaryOperatorKind BO,
3653 llvm::AtomicOrdering AO,
3654 bool IsXLHSInRHSPart) {
3655 ASTContext &Context = CGF.getContext();
3656 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3657 // expression is simple and atomic is allowed for the given type for the
3658 // target platform.
3659 if (BO == BO_Comma || !Update.isScalar() ||
3660 !Update.getScalarVal()->getType()->isIntegerTy() ||
3661 !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3662 (Update.getScalarVal()->getType() !=
3663 X.getAddress().getElementType())) ||
3664 !X.getAddress().getElementType()->isIntegerTy() ||
3665 !Context.getTargetInfo().hasBuiltinAtomic(
3666 Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3667 return std::make_pair(false, RValue::get(nullptr));
3668
3669 llvm::AtomicRMWInst::BinOp RMWOp;
3670 switch (BO) {
3671 case BO_Add:
3672 RMWOp = llvm::AtomicRMWInst::Add;
3673 break;
3674 case BO_Sub:
3675 if (!IsXLHSInRHSPart)
3676 return std::make_pair(false, RValue::get(nullptr));
3677 RMWOp = llvm::AtomicRMWInst::Sub;
3678 break;
3679 case BO_And:
3680 RMWOp = llvm::AtomicRMWInst::And;
3681 break;
3682 case BO_Or:
3683 RMWOp = llvm::AtomicRMWInst::Or;
3684 break;
3685 case BO_Xor:
3686 RMWOp = llvm::AtomicRMWInst::Xor;
3687 break;
3688 case BO_LT:
3689 RMWOp = X.getType()->hasSignedIntegerRepresentation()
3690 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3691 : llvm::AtomicRMWInst::Max)
3692 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3693 : llvm::AtomicRMWInst::UMax);
3694 break;
3695 case BO_GT:
3696 RMWOp = X.getType()->hasSignedIntegerRepresentation()
3697 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3698 : llvm::AtomicRMWInst::Min)
3699 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3700 : llvm::AtomicRMWInst::UMin);
3701 break;
3702 case BO_Assign:
3703 RMWOp = llvm::AtomicRMWInst::Xchg;
3704 break;
3705 case BO_Mul:
3706 case BO_Div:
3707 case BO_Rem:
3708 case BO_Shl:
3709 case BO_Shr:
3710 case BO_LAnd:
3711 case BO_LOr:
3712 return std::make_pair(false, RValue::get(nullptr));
3713 case BO_PtrMemD:
3714 case BO_PtrMemI:
3715 case BO_LE:
3716 case BO_GE:
3717 case BO_EQ:
3718 case BO_NE:
3719 case BO_Cmp:
3720 case BO_AddAssign:
3721 case BO_SubAssign:
3722 case BO_AndAssign:
3723 case BO_OrAssign:
3724 case BO_XorAssign:
3725 case BO_MulAssign:
3726 case BO_DivAssign:
3727 case BO_RemAssign:
3728 case BO_ShlAssign:
3729 case BO_ShrAssign:
3730 case BO_Comma:
3731 llvm_unreachable("Unsupported atomic update operation");
3732 }
3733 llvm::Value *UpdateVal = Update.getScalarVal();
3734 if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3735 UpdateVal = CGF.Builder.CreateIntCast(
3736 IC, X.getAddress().getElementType(),
3737 X.getType()->hasSignedIntegerRepresentation());
3738 }
3739 llvm::Value *Res =
3740 CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3741 return std::make_pair(true, RValue::get(Res));
3742 }
3743
3744 std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
3745 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3746 llvm::AtomicOrdering AO, SourceLocation Loc,
3747 const llvm::function_ref<RValue(RValue)> CommonGen) {
3748 // Update expressions are allowed to have the following forms:
3749 // x binop= expr; -> xrval + expr;
3750 // x++, ++x -> xrval + 1;
3751 // x--, --x -> xrval - 1;
3752 // x = x binop expr; -> xrval binop expr
3753 // x = expr Op x; - > expr binop xrval;
3754 auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3755 if (!Res.first) {
3756 if (X.isGlobalReg()) {
3757 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3758 // 'xrval'.
3759 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3760 } else {
3761 // Perform compare-and-swap procedure.
3762 EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3763 }
3764 }
3765 return Res;
3766 }
3767
3768 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3769 const Expr *X, const Expr *E,
3770 const Expr *UE, bool IsXLHSInRHSPart,
3771 SourceLocation Loc) {
3772 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3773 "Update expr in 'atomic update' must be a binary operator.");
3774 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3775 // Update expressions are allowed to have the following forms:
3776 // x binop= expr; -> xrval + expr;
3777 // x++, ++x -> xrval + 1;
3778 // x--, --x -> xrval - 1;
3779 // x = x binop expr; -> xrval binop expr
3780 // x = expr Op x; - > expr binop xrval;
3781 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3782 LValue XLValue = CGF.EmitLValue(X);
3783 RValue ExprRValue = CGF.EmitAnyExpr(E);
3784 llvm::AtomicOrdering AO = IsSeqCst
3785 ? llvm::AtomicOrdering::SequentiallyConsistent
3786 : llvm::AtomicOrdering::Monotonic;
3787 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3788 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3789 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3790 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3791 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
3792 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3793 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3794 return CGF.EmitAnyExpr(UE);
3795 };
3796 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
3797 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3798 // OpenMP, 2.12.6, atomic Construct
3799 // Any atomic construct with a seq_cst clause forces the atomically
3800 // performed operation to include an implicit flush operation without a
3801 // list.
3802 if (IsSeqCst)
3803 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3804 }
3805
3806 static RValue convertToType(CodeGenFunction &CGF, RValue Value,
3807 QualType SourceType, QualType ResType,
3808 SourceLocation Loc) {
3809 switch (CGF.getEvaluationKind(ResType)) {
3810 case TEK_Scalar:
3811 return RValue::get(
3812 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3813 case TEK_Complex: {
3814 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3815 return RValue::getComplex(Res.first, Res.second);
3816 }
3817 case TEK_Aggregate:
3818 break;
3819 }
3820 llvm_unreachable("Must be a scalar or complex.");
3821 }
3822
3823 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3824 bool IsPostfixUpdate, const Expr *V,
3825 const Expr *X, const Expr *E,
3826 const Expr *UE, bool IsXLHSInRHSPart,
3827 SourceLocation Loc) {
3828 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3829 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3830 RValue NewVVal;
3831 LValue VLValue = CGF.EmitLValue(V);
3832 LValue XLValue = CGF.EmitLValue(X);
3833 RValue ExprRValue = CGF.EmitAnyExpr(E);
3834 llvm::AtomicOrdering AO = IsSeqCst
3835 ? llvm::AtomicOrdering::SequentiallyConsistent
3836 : llvm::AtomicOrdering::Monotonic;
3837 QualType NewVValType;
3838 if (UE) {
3839 // 'x' is updated with some additional value.
3840 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3841 "Update expr in 'atomic capture' must be a binary operator.");
3842 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3843 // Update expressions are allowed to have the following forms:
3844 // x binop= expr; -> xrval + expr;
3845 // x++, ++x -> xrval + 1;
3846 // x--, --x -> xrval - 1;
3847 // x = x binop expr; -> xrval binop expr
3848 // x = expr Op x; - > expr binop xrval;
3849 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3850 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3851 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3852 NewVValType = XRValExpr->getType();
3853 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3854 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3855 IsPostfixUpdate](RValue XRValue) {
3856 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3857 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3858 RValue Res = CGF.EmitAnyExpr(UE);
3859 NewVVal = IsPostfixUpdate ? XRValue : Res;
3860 return Res;
3861 };
3862 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3863 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3864 if (Res.first) {
3865 // 'atomicrmw' instruction was generated.
3866 if (IsPostfixUpdate) {
3867 // Use old value from 'atomicrmw'.
3868 NewVVal = Res.second;
3869 } else {
3870 // 'atomicrmw' does not provide new value, so evaluate it using old
3871 // value of 'x'.
3872 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3873 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3874 NewVVal = CGF.EmitAnyExpr(UE);
3875 }
3876 }
3877 } else {
3878 // 'x' is simply rewritten with some 'expr'.
3879 NewVValType = X->getType().getNonReferenceType();
3880 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3881 X->getType().getNonReferenceType(), Loc);
3882 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
3883 NewVVal = XRValue;
3884 return ExprRValue;
3885 };
3886 // Try to perform atomicrmw xchg, otherwise simple exchange.
3887 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3888 XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3889 Loc, Gen);
3890 if (Res.first) {
3891 // 'atomicrmw' instruction was generated.
3892 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3893 }
3894 }
3895 // Emit post-update store to 'v' of old/new 'x' value.
3896 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3897 // OpenMP, 2.12.6, atomic Construct
3898 // Any atomic construct with a seq_cst clause forces the atomically
3899 // performed operation to include an implicit flush operation without a
3900 // list.
3901 if (IsSeqCst)
3902 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3903 }
3904
3905 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
3906 bool IsSeqCst, bool IsPostfixUpdate,
3907 const Expr *X, const Expr *V, const Expr *E,
3908 const Expr *UE, bool IsXLHSInRHSPart,
3909 SourceLocation Loc) {
3910 switch (Kind) {
3911 case OMPC_read:
3912 emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3913 break;
3914 case OMPC_write:
3915 emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3916 break;
3917 case OMPC_unknown:
3918 case OMPC_update:
3919 emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3920 break;
3921 case OMPC_capture:
3922 emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3923 IsXLHSInRHSPart, Loc);
3924 break;
3925 case OMPC_if:
3926 case OMPC_final:
3927 case OMPC_num_threads:
3928 case OMPC_private:
3929 case OMPC_firstprivate:
3930 case OMPC_lastprivate:
3931 case OMPC_reduction:
3932 case OMPC_task_reduction:
3933 case OMPC_in_reduction:
3934 case OMPC_safelen:
3935 case OMPC_simdlen:
3936 case OMPC_collapse:
3937 case OMPC_default:
3938 case OMPC_seq_cst:
3939 case OMPC_shared:
3940 case OMPC_linear:
3941 case OMPC_aligned:
3942 case OMPC_copyin:
3943 case OMPC_copyprivate:
3944 case OMPC_flush:
3945 case OMPC_proc_bind:
3946 case OMPC_schedule:
3947 case OMPC_ordered:
3948 case OMPC_nowait:
3949 case OMPC_untied:
3950 case OMPC_threadprivate:
3951 case OMPC_depend:
3952 case OMPC_mergeable:
3953 case OMPC_device:
3954 case OMPC_threads:
3955 case OMPC_simd:
3956 case OMPC_map:
3957 case OMPC_num_teams:
3958 case OMPC_thread_limit:
3959 case OMPC_priority:
3960 case OMPC_grainsize:
3961 case OMPC_nogroup:
3962 case OMPC_num_tasks:
3963 case OMPC_hint:
3964 case OMPC_dist_schedule:
3965 case OMPC_defaultmap:
3966 case OMPC_uniform:
3967 case OMPC_to:
3968 case OMPC_from:
3969 case OMPC_use_device_ptr:
3970 case OMPC_is_device_ptr:
3971 case OMPC_unified_address:
3972 case OMPC_unified_shared_memory:
3973 case OMPC_reverse_offload:
3974 case OMPC_dynamic_allocators:
3975 case OMPC_atomic_default_mem_order:
3976 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3977 }
3978 }
3979
3980 void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
3981 bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
3982 OpenMPClauseKind Kind = OMPC_unknown;
3983 for (const OMPClause *C : S.clauses()) {
3984 // Find first clause (skip seq_cst clause, if it is first).
3985 if (C->getClauseKind() != OMPC_seq_cst) {
3986 Kind = C->getClauseKind();
3987 break;
3988 }
3989 }
3990
3991 const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
3992 if (const auto *FE = dyn_cast<FullExpr>(CS))
3993 enterFullExpression(FE);
3994 // Processing for statements under 'atomic capture'.
3995 if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3996 for (const Stmt *C : Compound->body()) {
3997 if (const auto *FE = dyn_cast<FullExpr>(C))
3998 enterFullExpression(FE);
3999 }
4000 }
4001
4002 auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
4003 PrePostActionTy &) {
4004 CGF.EmitStopPoint(CS);
4005 emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
4006 S.getV(), S.getExpr(), S.getUpdateExpr(),
4007 S.isXLHSInRHSPart(), S.getBeginLoc());
4008 };
4009 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4010 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
4011 }
4012
4013 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
4014 const OMPExecutableDirective &S,
4015 const RegionCodeGenTy &CodeGen) {
4016 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
4017 CodeGenModule &CGM = CGF.CGM;
4018
4019 // On device emit this construct as inlined code.
4020 if (CGM.getLangOpts().OpenMPIsDevice) {
4021 OMPLexicalScope Scope(CGF, S, OMPD_target);
4022 CGM.getOpenMPRuntime().emitInlinedDirective(
4023 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4024 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4025 });
4026 return;
4027 }
4028
4029 llvm::Function *Fn = nullptr;
4030 llvm::Constant *FnID = nullptr;
4031
4032 const Expr *IfCond = nullptr;
4033 // Check for the at most one if clause associated with the target region.
4034 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4035 if (C->getNameModifier() == OMPD_unknown ||
4036 C->getNameModifier() == OMPD_target) {
4037 IfCond = C->getCondition();
4038 break;
4039 }
4040 }
4041
4042 // Check if we have any device clause associated with the directive.
4043 const Expr *Device = nullptr;
4044 if (auto *C = S.getSingleClause<OMPDeviceClause>())
4045 Device = C->getDevice();
4046
4047 // Check if we have an if clause whose conditional always evaluates to false
4048 // or if we do not have any targets specified. If so the target region is not
4049 // an offload entry point.
4050 bool IsOffloadEntry = true;
4051 if (IfCond) {
4052 bool Val;
4053 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
4054 IsOffloadEntry = false;
4055 }
4056 if (CGM.getLangOpts().OMPTargetTriples.empty())
4057 IsOffloadEntry = false;
4058
4059 assert(CGF.CurFuncDecl && "No parent declaration for target region!");
4060 StringRef ParentName;
4061 // In case we have Ctors/Dtors we use the complete type variant to produce
4062 // the mangling of the device outlined kernel.
4063 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
4064 ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
4065 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
4066 ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
4067 else
4068 ParentName =
4069 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
4070
4071 // Emit target region as a standalone region.
4072 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4073 IsOffloadEntry, CodeGen);
4074 OMPLexicalScope Scope(CGF, S, OMPD_task);
4075 auto &&SizeEmitter = [](CodeGenFunction &CGF, const OMPLoopDirective &D) {
4076 OMPLoopScope(CGF, D);
4077 // Emit calculation of the iterations count.
4078 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
4079 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
4080 /*IsSigned=*/false);
4081 return NumIterations;
4082 };
4083 CGM.getOpenMPRuntime().emitTargetNumIterationsCall(CGF, S, Device,
4084 SizeEmitter);
4085 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
4086 }
4087
4088 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
4089 PrePostActionTy &Action) {
4090 Action.Enter(CGF);
4091 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4092 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4093 CGF.EmitOMPPrivateClause(S, PrivateScope);
4094 (void)PrivateScope.Privatize();
4095 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4096 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4097
4098 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4099 }
4100
4101 void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
4102 StringRef ParentName,
4103 const OMPTargetDirective &S) {
4104 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4105 emitTargetRegion(CGF, S, Action);
4106 };
4107 llvm::Function *Fn;
4108 llvm::Constant *Addr;
4109 // Emit target region as a standalone region.
4110 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4111 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4112 assert(Fn && Addr && "Target device function emission failed.");
4113 }
4114
4115 void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
4116 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4117 emitTargetRegion(CGF, S, Action);
4118 };
4119 emitCommonOMPTargetDirective(*this, S, CodeGen);
4120 }
4121
4122 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
4123 const OMPExecutableDirective &S,
4124 OpenMPDirectiveKind InnermostKind,
4125 const RegionCodeGenTy &CodeGen) {
4126 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4127 llvm::Value *OutlinedFn =
4128 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4129 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4130
4131 const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4132 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4133 if (NT || TL) {
4134 const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4135 const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
4136
4137 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4138 S.getBeginLoc());
4139 }
4140
4141 OMPTeamsScope Scope(CGF, S);
4142 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
4143 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4144 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
4145 CapturedVars);
4146 }
4147
4148 void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
4149 // Emit teams region as a standalone region.
4150 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4151 Action.Enter(CGF);
4152 OMPPrivateScope PrivateScope(CGF);
4153 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4154 CGF.EmitOMPPrivateClause(S, PrivateScope);
4155 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4156 (void)PrivateScope.Privatize();
4157 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4158 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4159 };
4160 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4161 emitPostUpdateForReductionClause(*this, S,
4162 [](CodeGenFunction &) { return nullptr; });
4163 }
4164
4165 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
4166 const OMPTargetTeamsDirective &S) {
4167 auto *CS = S.getCapturedStmt(OMPD_teams);
4168 Action.Enter(CGF);
4169 // Emit teams region as a standalone region.
4170 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4171 Action.Enter(CGF);
4172 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4173 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4174 CGF.EmitOMPPrivateClause(S, PrivateScope);
4175 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4176 (void)PrivateScope.Privatize();
4177 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4178 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4179 CGF.EmitStmt(CS->getCapturedStmt());
4180 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4181 };
4182 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4183 emitPostUpdateForReductionClause(CGF, S,
4184 [](CodeGenFunction &) { return nullptr; });
4185 }
4186
4187 void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
4188 CodeGenModule &CGM, StringRef ParentName,
4189 const OMPTargetTeamsDirective &S) {
4190 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4191 emitTargetTeamsRegion(CGF, Action, S);
4192 };
4193 llvm::Function *Fn;
4194 llvm::Constant *Addr;
4195 // Emit target region as a standalone region.
4196 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4197 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4198 assert(Fn && Addr && "Target device function emission failed.");
4199 }
4200
4201 void CodeGenFunction::EmitOMPTargetTeamsDirective(
4202 const OMPTargetTeamsDirective &S) {
4203 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4204 emitTargetTeamsRegion(CGF, Action, S);
4205 };
4206 emitCommonOMPTargetDirective(*this, S, CodeGen);
4207 }
4208
4209 static void
4210 emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
4211 const OMPTargetTeamsDistributeDirective &S) {
4212 Action.Enter(CGF);
4213 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4214 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4215 };
4216
4217 // Emit teams region as a standalone region.
4218 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4219 PrePostActionTy &Action) {
4220 Action.Enter(CGF);
4221 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4222 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4223 (void)PrivateScope.Privatize();
4224 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4225 CodeGenDistribute);
4226 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4227 };
4228 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4229 emitPostUpdateForReductionClause(CGF, S,
4230 [](CodeGenFunction &) { return nullptr; });
4231 }
4232
4233 void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
4234 CodeGenModule &CGM, StringRef ParentName,
4235 const OMPTargetTeamsDistributeDirective &S) {
4236 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4237 emitTargetTeamsDistributeRegion(CGF, Action, S);
4238 };
4239 llvm::Function *Fn;
4240 llvm::Constant *Addr;
4241 // Emit target region as a standalone region.
4242 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4243 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4244 assert(Fn && Addr && "Target device function emission failed.");
4245 }
4246
4247 void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
4248 const OMPTargetTeamsDistributeDirective &S) {
4249 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4250 emitTargetTeamsDistributeRegion(CGF, Action, S);
4251 };
4252 emitCommonOMPTargetDirective(*this, S, CodeGen);
4253 }
4254
4255 static void emitTargetTeamsDistributeSimdRegion(
4256 CodeGenFunction &CGF, PrePostActionTy &Action,
4257 const OMPTargetTeamsDistributeSimdDirective &S) {
4258 Action.Enter(CGF);
4259 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4260 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4261 };
4262
4263 // Emit teams region as a standalone region.
4264 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4265 PrePostActionTy &Action) {
4266 Action.Enter(CGF);
4267 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4268 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4269 (void)PrivateScope.Privatize();
4270 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4271 CodeGenDistribute);
4272 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4273 };
4274 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4275 emitPostUpdateForReductionClause(CGF, S,
4276 [](CodeGenFunction &) { return nullptr; });
4277 }
4278
4279 void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
4280 CodeGenModule &CGM, StringRef ParentName,
4281 const OMPTargetTeamsDistributeSimdDirective &S) {
4282 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4283 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4284 };
4285 llvm::Function *Fn;
4286 llvm::Constant *Addr;
4287 // Emit target region as a standalone region.
4288 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4289 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4290 assert(Fn && Addr && "Target device function emission failed.");
4291 }
4292
4293 void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
4294 const OMPTargetTeamsDistributeSimdDirective &S) {
4295 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4296 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4297 };
4298 emitCommonOMPTargetDirective(*this, S, CodeGen);
4299 }
4300
4301 void CodeGenFunction::EmitOMPTeamsDistributeDirective(
4302 const OMPTeamsDistributeDirective &S) {
4303
4304 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4305 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4306 };
4307
4308 // Emit teams region as a standalone region.
4309 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4310 PrePostActionTy &Action) {
4311 Action.Enter(CGF);
4312 OMPPrivateScope PrivateScope(CGF);
4313 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4314 (void)PrivateScope.Privatize();
4315 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4316 CodeGenDistribute);
4317 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4318 };
4319 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4320 emitPostUpdateForReductionClause(*this, S,
4321 [](CodeGenFunction &) { return nullptr; });
4322 }
4323
4324 void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
4325 const OMPTeamsDistributeSimdDirective &S) {
4326 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4327 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4328 };
4329
4330 // Emit teams region as a standalone region.
4331 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4332 PrePostActionTy &Action) {
4333 Action.Enter(CGF);
4334 OMPPrivateScope PrivateScope(CGF);
4335 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4336 (void)PrivateScope.Privatize();
4337 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4338 CodeGenDistribute);
4339 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4340 };
4341 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4342 emitPostUpdateForReductionClause(*this, S,
4343 [](CodeGenFunction &) { return nullptr; });
4344 }
4345
4346 void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
4347 const OMPTeamsDistributeParallelForDirective &S) {
4348 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4349 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4350 S.getDistInc());
4351 };
4352
4353 // Emit teams region as a standalone region.
4354 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4355 PrePostActionTy &Action) {
4356 Action.Enter(CGF);
4357 OMPPrivateScope PrivateScope(CGF);
4358 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4359 (void)PrivateScope.Privatize();
4360 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4361 CodeGenDistribute);
4362 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4363 };
4364 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4365 emitPostUpdateForReductionClause(*this, S,
4366 [](CodeGenFunction &) { return nullptr; });
4367 }
4368
4369 void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
4370 const OMPTeamsDistributeParallelForSimdDirective &S) {
4371 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4372 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4373 S.getDistInc());
4374 };
4375
4376 // Emit teams region as a standalone region.
4377 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4378 PrePostActionTy &Action) {
4379 Action.Enter(CGF);
4380 OMPPrivateScope PrivateScope(CGF);
4381 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4382 (void)PrivateScope.Privatize();
4383 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4384 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4385 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4386 };
4387 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4388 emitPostUpdateForReductionClause(*this, S,
4389 [](CodeGenFunction &) { return nullptr; });
4390 }
4391
4392 static void emitTargetTeamsDistributeParallelForRegion(
4393 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,
4394 PrePostActionTy &Action) {
4395 Action.Enter(CGF);
4396 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4397 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4398 S.getDistInc());
4399 };
4400
4401 // Emit teams region as a standalone region.
4402 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4403 PrePostActionTy &Action) {
4404 Action.Enter(CGF);
4405 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4406 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4407 (void)PrivateScope.Privatize();
4408 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4409 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4410 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4411 };
4412
4413 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4414 CodeGenTeams);
4415 emitPostUpdateForReductionClause(CGF, S,
4416 [](CodeGenFunction &) { return nullptr; });
4417 }
4418
4419 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
4420 CodeGenModule &CGM, StringRef ParentName,
4421 const OMPTargetTeamsDistributeParallelForDirective &S) {
4422 // Emit SPMD target teams distribute parallel for region as a standalone
4423 // region.
4424 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4425 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
4426 };
4427 llvm::Function *Fn;
4428 llvm::Constant *Addr;
4429 // Emit target region as a standalone region.
4430 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4431 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4432 assert(Fn && Addr && "Target device function emission failed.");
4433 }
4434
4435 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
4436 const OMPTargetTeamsDistributeParallelForDirective &S) {
4437 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4438 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
4439 };
4440 emitCommonOMPTargetDirective(*this, S, CodeGen);
4441 }
4442
4443 static void emitTargetTeamsDistributeParallelForSimdRegion(
4444 CodeGenFunction &CGF,
4445 const OMPTargetTeamsDistributeParallelForSimdDirective &S,
4446 PrePostActionTy &Action) {
4447 Action.Enter(CGF);
4448 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4449 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4450 S.getDistInc());
4451 };
4452
4453 // Emit teams region as a standalone region.
4454 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4455 PrePostActionTy &Action) {
4456 Action.Enter(CGF);
4457 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4458 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4459 (void)PrivateScope.Privatize();
4460 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4461 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4462 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4463 };
4464
4465 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4466 CodeGenTeams);
4467 emitPostUpdateForReductionClause(CGF, S,
4468 [](CodeGenFunction &) { return nullptr; });
4469 }
4470
4471 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
4472 CodeGenModule &CGM, StringRef ParentName,
4473 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
4474 // Emit SPMD target teams distribute parallel for simd region as a standalone
4475 // region.
4476 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4477 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
4478 };
4479 llvm::Function *Fn;
4480 llvm::Constant *Addr;
4481 // Emit target region as a standalone region.
4482 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4483 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4484 assert(Fn && Addr && "Target device function emission failed.");
4485 }
4486
4487 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
4488 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
4489 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4490 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
4491 };
4492 emitCommonOMPTargetDirective(*this, S, CodeGen);
4493 }
4494
4495 void CodeGenFunction::EmitOMPCancellationPointDirective(
4496 const OMPCancellationPointDirective &S) {
4497 CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
4498 S.getCancelRegion());
4499 }
4500
4501 void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
4502 const Expr *IfCond = nullptr;
4503 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4504 if (C->getNameModifier() == OMPD_unknown ||
4505 C->getNameModifier() == OMPD_cancel) {
4506 IfCond = C->getCondition();
4507 break;
4508 }
4509 }
4510 CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
4511 S.getCancelRegion());
4512 }
4513
4514 CodeGenFunction::JumpDest
4515 CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
4516 if (Kind == OMPD_parallel || Kind == OMPD_task ||
4517 Kind == OMPD_target_parallel)
4518 return ReturnBlock;
4519 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4520 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4521 Kind == OMPD_distribute_parallel_for ||
4522 Kind == OMPD_target_parallel_for ||
4523 Kind == OMPD_teams_distribute_parallel_for ||
4524 Kind == OMPD_target_teams_distribute_parallel_for);
4525 return OMPCancelStack.getExitBlock();
4526 }
4527
4528 void CodeGenFunction::EmitOMPUseDevicePtrClause(
4529 const OMPClause &NC, OMPPrivateScope &PrivateScope,
4530 const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4531 const auto &C = cast<OMPUseDevicePtrClause>(NC);
4532 auto OrigVarIt = C.varlist_begin();
4533 auto InitIt = C.inits().begin();
4534 for (const Expr *PvtVarIt : C.private_copies()) {
4535 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4536 const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4537 const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4538
4539 // In order to identify the right initializer we need to match the
4540 // declaration used by the mapping logic. In some cases we may get
4541 // OMPCapturedExprDecl that refers to the original declaration.
4542 const ValueDecl *MatchingVD = OrigVD;
4543 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4544 // OMPCapturedExprDecl are used to privative fields of the current
4545 // structure.
4546 const auto *ME = cast<MemberExpr>(OED->getInit());
4547 assert(isa<CXXThisExpr>(ME->getBase()) &&
4548 "Base should be the current struct!");
4549 MatchingVD = ME->getMemberDecl();
4550 }
4551
4552 // If we don't have information about the current list item, move on to
4553 // the next one.
4554 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4555 if (InitAddrIt == CaptureDeviceAddrMap.end())
4556 continue;
4557
4558 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4559 InitAddrIt, InitVD,
4560 PvtVD]() {
4561 // Initialize the temporary initialization variable with the address we
4562 // get from the runtime library. We have to cast the source address
4563 // because it is always a void *. References are materialized in the
4564 // privatization scope, so the initialization here disregards the fact
4565 // the original variable is a reference.
4566 QualType AddrQTy =
4567 getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4568 llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4569 Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4570 setAddrOfLocalVar(InitVD, InitAddr);
4571
4572 // Emit private declaration, it will be initialized by the value we
4573 // declaration we just added to the local declarations map.
4574 EmitDecl(*PvtVD);
4575
4576 // The initialization variables reached its purpose in the emission
4577 // of the previous declaration, so we don't need it anymore.
4578 LocalDeclMap.erase(InitVD);
4579
4580 // Return the address of the private variable.
4581 return GetAddrOfLocalVar(PvtVD);
4582 });
4583 assert(IsRegistered && "firstprivate var already registered as private");
4584 // Silence the warning about unused variable.
4585 (void)IsRegistered;
4586
4587 ++OrigVarIt;
4588 ++InitIt;
4589 }
4590 }
4591
4592 // Generate the instructions for '#pragma omp target data' directive.
4593 void CodeGenFunction::EmitOMPTargetDataDirective(
4594 const OMPTargetDataDirective &S) {
4595 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4596
4597 // Create a pre/post action to signal the privatization of the device pointer.
4598 // This action can be replaced by the OpenMP runtime code generation to
4599 // deactivate privatization.
4600 bool PrivatizeDevicePointers = false;
4601 class DevicePointerPrivActionTy : public PrePostActionTy {
4602 bool &PrivatizeDevicePointers;
4603
4604 public:
4605 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4606 : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4607 void Enter(CodeGenFunction &CGF) override {
4608 PrivatizeDevicePointers = true;
4609 }
4610 };
4611 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4612
4613 auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4614 CodeGenFunction &CGF, PrePostActionTy &Action) {
4615 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4616 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4617 };
4618
4619 // Codegen that selects whether to generate the privatization code or not.
4620 auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4621 &InnermostCodeGen](CodeGenFunction &CGF,
4622 PrePostActionTy &Action) {
4623 RegionCodeGenTy RCG(InnermostCodeGen);
4624 PrivatizeDevicePointers = false;
4625
4626 // Call the pre-action to change the status of PrivatizeDevicePointers if
4627 // needed.
4628 Action.Enter(CGF);
4629
4630 if (PrivatizeDevicePointers) {
4631 OMPPrivateScope PrivateScope(CGF);
4632 // Emit all instances of the use_device_ptr clause.
4633 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4634 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4635 Info.CaptureDeviceAddrMap);
4636 (void)PrivateScope.Privatize();
4637 RCG(CGF);
4638 } else {
4639 RCG(CGF);
4640 }
4641 };
4642
4643 // Forward the provided action to the privatization codegen.
4644 RegionCodeGenTy PrivRCG(PrivCodeGen);
4645 PrivRCG.setAction(Action);
4646
4647 // Notwithstanding the body of the region is emitted as inlined directive,
4648 // we don't use an inline scope as changes in the references inside the
4649 // region are expected to be visible outside, so we do not privative them.
4650 OMPLexicalScope Scope(CGF, S);
4651 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4652 PrivRCG);
4653 };
4654
4655 RegionCodeGenTy RCG(CodeGen);
4656
4657 // If we don't have target devices, don't bother emitting the data mapping
4658 // code.
4659 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4660 RCG(*this);
4661 return;
4662 }
4663
4664 // Check if we have any if clause associated with the directive.
4665 const Expr *IfCond = nullptr;
4666 if (const auto *C = S.getSingleClause<OMPIfClause>())
4667 IfCond = C->getCondition();
4668
4669 // Check if we have any device clause associated with the directive.
4670 const Expr *Device = nullptr;
4671 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4672 Device = C->getDevice();
4673
4674 // Set the action to signal privatization of device pointers.
4675 RCG.setAction(PrivAction);
4676
4677 // Emit region code.
4678 CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4679 Info);
4680 }
4681
4682 void CodeGenFunction::EmitOMPTargetEnterDataDirective(
4683 const OMPTargetEnterDataDirective &S) {
4684 // If we don't have target devices, don't bother emitting the data mapping
4685 // code.
4686 if (CGM.getLangOpts().OMPTargetTriples.empty())
4687 return;
4688
4689 // Check if we have any if clause associated with the directive.
4690 const Expr *IfCond = nullptr;
4691 if (const auto *C = S.getSingleClause<OMPIfClause>())
4692 IfCond = C->getCondition();
4693
4694 // Check if we have any device clause associated with the directive.
4695 const Expr *Device = nullptr;
4696 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4697 Device = C->getDevice();
4698
4699 OMPLexicalScope Scope(*this, S, OMPD_task);
4700 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4701 }
4702
4703 void CodeGenFunction::EmitOMPTargetExitDataDirective(
4704 const OMPTargetExitDataDirective &S) {
4705 // If we don't have target devices, don't bother emitting the data mapping
4706 // code.
4707 if (CGM.getLangOpts().OMPTargetTriples.empty())
4708 return;
4709
4710 // Check if we have any if clause associated with the directive.
4711 const Expr *IfCond = nullptr;
4712 if (const auto *C = S.getSingleClause<OMPIfClause>())
4713 IfCond = C->getCondition();
4714
4715 // Check if we have any device clause associated with the directive.
4716 const Expr *Device = nullptr;
4717 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4718 Device = C->getDevice();
4719
4720 OMPLexicalScope Scope(*this, S, OMPD_task);
4721 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4722 }
4723
4724 static void emitTargetParallelRegion(CodeGenFunction &CGF,
4725 const OMPTargetParallelDirective &S,
4726 PrePostActionTy &Action) {
4727 // Get the captured statement associated with the 'parallel' region.
4728 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
4729 Action.Enter(CGF);
4730 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4731 Action.Enter(CGF);
4732 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4733 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4734 CGF.EmitOMPPrivateClause(S, PrivateScope);
4735 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4736 (void)PrivateScope.Privatize();
4737 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4738 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4739 // TODO: Add support for clauses.
4740 CGF.EmitStmt(CS->getCapturedStmt());
4741 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4742 };
4743 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4744 emitEmptyBoundParameters);
4745 emitPostUpdateForReductionClause(CGF, S,
4746 [](CodeGenFunction &) { return nullptr; });
4747 }
4748
4749 void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
4750 CodeGenModule &CGM, StringRef ParentName,
4751 const OMPTargetParallelDirective &S) {
4752 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4753 emitTargetParallelRegion(CGF, S, Action);
4754 };
4755 llvm::Function *Fn;
4756 llvm::Constant *Addr;
4757 // Emit target region as a standalone region.
4758 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4759 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4760 assert(Fn && Addr && "Target device function emission failed.");
4761 }
4762
4763 void CodeGenFunction::EmitOMPTargetParallelDirective(
4764 const OMPTargetParallelDirective &S) {
4765 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4766 emitTargetParallelRegion(CGF, S, Action);
4767 };
4768 emitCommonOMPTargetDirective(*this, S, CodeGen);
4769 }
4770
4771 static void emitTargetParallelForRegion(CodeGenFunction &CGF,
4772 const OMPTargetParallelForDirective &S,
4773 PrePostActionTy &Action) {
4774 Action.Enter(CGF);
4775 // Emit directive as a combined directive that consists of two implicit
4776 // directives: 'parallel' with 'for' directive.
4777 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4778 Action.Enter(CGF);
4779 CodeGenFunction::OMPCancelStackRAII CancelRegion(
4780 CGF, OMPD_target_parallel_for, S.hasCancel());
4781 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
4782 emitDispatchForLoopBounds);
4783 };
4784 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4785 emitEmptyBoundParameters);
4786 }
4787
4788 void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
4789 CodeGenModule &CGM, StringRef ParentName,
4790 const OMPTargetParallelForDirective &S) {
4791 // Emit SPMD target parallel for region as a standalone region.
4792 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4793 emitTargetParallelForRegion(CGF, S, Action);
4794 };
4795 llvm::Function *Fn;
4796 llvm::Constant *Addr;
4797 // Emit target region as a standalone region.
4798 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4799 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4800 assert(Fn && Addr && "Target device function emission failed.");
4801 }
4802
4803 void CodeGenFunction::EmitOMPTargetParallelForDirective(
4804 const OMPTargetParallelForDirective &S) {
4805 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4806 emitTargetParallelForRegion(CGF, S, Action);
4807 };
4808 emitCommonOMPTargetDirective(*this, S, CodeGen);
4809 }
4810
4811 static void
4812 emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
4813 const OMPTargetParallelForSimdDirective &S,
4814 PrePostActionTy &Action) {
4815 Action.Enter(CGF);
4816 // Emit directive as a combined directive that consists of two implicit
4817 // directives: 'parallel' with 'for' directive.
4818 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4819 Action.Enter(CGF);
4820 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
4821 emitDispatchForLoopBounds);
4822 };
4823 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4824 emitEmptyBoundParameters);
4825 }
4826
4827 void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
4828 CodeGenModule &CGM, StringRef ParentName,
4829 const OMPTargetParallelForSimdDirective &S) {
4830 // Emit SPMD target parallel for region as a standalone region.
4831 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4832 emitTargetParallelForSimdRegion(CGF, S, Action);
4833 };
4834 llvm::Function *Fn;
4835 llvm::Constant *Addr;
4836 // Emit target region as a standalone region.
4837 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4838 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4839 assert(Fn && Addr && "Target device function emission failed.");
4840 }
4841
4842 void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
4843 const OMPTargetParallelForSimdDirective &S) {
4844 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4845 emitTargetParallelForSimdRegion(CGF, S, Action);
4846 };
4847 emitCommonOMPTargetDirective(*this, S, CodeGen);
4848 }
4849
4850 /// Emit a helper variable and return corresponding lvalue.
4851 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4852 const ImplicitParamDecl *PVD,
4853 CodeGenFunction::OMPPrivateScope &Privates) {
4854 const auto *VDecl = cast<VarDecl>(Helper->getDecl());
4855 Privates.addPrivate(VDecl,
4856 [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
4857 }
4858
4859 void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
4860 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
4861 // Emit outlined function for task construct.
4862 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
4863 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4864 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4865 const Expr *IfCond = nullptr;
4866 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4867 if (C->getNameModifier() == OMPD_unknown ||
4868 C->getNameModifier() == OMPD_taskloop) {
4869 IfCond = C->getCondition();
4870 break;
4871 }
4872 }
4873
4874 OMPTaskDataTy Data;
4875 // Check if taskloop must be emitted without taskgroup.
4876 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
4877 // TODO: Check if we should emit tied or untied task.
4878 Data.Tied = true;
4879 // Set scheduling for taskloop
4880 if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4881 // grainsize clause
4882 Data.Schedule.setInt(/*IntVal=*/false);
4883 Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4884 } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4885 // num_tasks clause
4886 Data.Schedule.setInt(/*IntVal=*/true);
4887 Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4888 }
4889
4890 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4891 // if (PreCond) {
4892 // for (IV in 0..LastIteration) BODY;
4893 // <Final counter/linear vars updates>;
4894 // }
4895 //
4896
4897 // Emit: if (PreCond) - begin.
4898 // If the condition constant folds and can be elided, avoid emitting the
4899 // whole loop.
4900 bool CondConstant;
4901 llvm::BasicBlock *ContBlock = nullptr;
4902 OMPLoopScope PreInitScope(CGF, S);
4903 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4904 if (!CondConstant)
4905 return;
4906 } else {
4907 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4908 ContBlock = CGF.createBasicBlock("taskloop.if.end");
4909 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4910 CGF.getProfileCount(&S));
4911 CGF.EmitBlock(ThenBlock);
4912 CGF.incrementProfileCounter(&S);
4913 }
4914
4915 if (isOpenMPSimdDirective(S.getDirectiveKind()))
4916 CGF.EmitOMPSimdInit(S);
4917
4918 OMPPrivateScope LoopScope(CGF);
4919 // Emit helper vars inits.
4920 enum { LowerBound = 5, UpperBound, Stride, LastIter };
4921 auto *I = CS->getCapturedDecl()->param_begin();
4922 auto *LBP = std::next(I, LowerBound);
4923 auto *UBP = std::next(I, UpperBound);
4924 auto *STP = std::next(I, Stride);
4925 auto *LIP = std::next(I, LastIter);
4926 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4927 LoopScope);
4928 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4929 LoopScope);
4930 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4931 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4932 LoopScope);
4933 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4934 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4935 (void)LoopScope.Privatize();
4936 // Emit the loop iteration variable.
4937 const Expr *IVExpr = S.getIterationVariable();
4938 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4939 CGF.EmitVarDecl(*IVDecl);
4940 CGF.EmitIgnoredExpr(S.getInit());
4941
4942 // Emit the iterations count variable.
4943 // If it is not a variable, Sema decided to calculate iterations count on
4944 // each iteration (e.g., it is foldable into a constant).
4945 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4946 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4947 // Emit calculation of the iterations count.
4948 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4949 }
4950
4951 CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4952 S.getInc(),
4953 [&S](CodeGenFunction &CGF) {
4954 CGF.EmitOMPLoopBody(S, JumpDest());
4955 CGF.EmitStopPoint(&S);
4956 },
4957 [](CodeGenFunction &) {});
4958 // Emit: if (PreCond) - end.
4959 if (ContBlock) {
4960 CGF.EmitBranch(ContBlock);
4961 CGF.EmitBlock(ContBlock, true);
4962 }
4963 // Emit final copy of the lastprivate variables if IsLastIter != 0.
4964 if (HasLastprivateClause) {
4965 CGF.EmitOMPLastprivateClauseFinal(
4966 S, isOpenMPSimdDirective(S.getDirectiveKind()),
4967 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4968 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4969 (*LIP)->getType(), S.getBeginLoc())));
4970 }
4971 };
4972 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4973 IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
4974 const OMPTaskDataTy &Data) {
4975 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
4976 &Data](CodeGenFunction &CGF, PrePostActionTy &) {
4977 OMPLoopScope PreInitScope(CGF, S);
4978 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
4979 OutlinedFn, SharedsTy,
4980 CapturedStruct, IfCond, Data);
4981 };
4982 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4983 CodeGen);
4984 };
4985 if (Data.Nogroup) {
4986 EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
4987 } else {
4988 CGM.getOpenMPRuntime().emitTaskgroupRegion(
4989 *this,
4990 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
4991 PrePostActionTy &Action) {
4992 Action.Enter(CGF);
4993 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
4994 Data);
4995 },
4996 S.getBeginLoc());
4997 }
4998 }
4999
5000 void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
5001 EmitOMPTaskLoopBasedDirective(S);
5002 }
5003
5004 void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
5005 const OMPTaskLoopSimdDirective &S) {
5006 EmitOMPTaskLoopBasedDirective(S);
5007 }
5008
5009 // Generate the instructions for '#pragma omp target update' directive.
5010 void CodeGenFunction::EmitOMPTargetUpdateDirective(
5011 const OMPTargetUpdateDirective &S) {
5012 // If we don't have target devices, don't bother emitting the data mapping
5013 // code.
5014 if (CGM.getLangOpts().OMPTargetTriples.empty())
5015 return;
5016
5017 // Check if we have any if clause associated with the directive.
5018 const Expr *IfCond = nullptr;
5019 if (const auto *C = S.getSingleClause<OMPIfClause>())
5020 IfCond = C->getCondition();
5021
5022 // Check if we have any device clause associated with the directive.
5023 const Expr *Device = nullptr;
5024 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
5025 Device = C->getDevice();
5026
5027 OMPLexicalScope Scope(*this, S, OMPD_task);
5028 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
5029 }
5030
5031 void CodeGenFunction::EmitSimpleOMPExecutableDirective(
5032 const OMPExecutableDirective &D) {
5033 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
5034 return;
5035 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
5036 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
5037 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
5038 } else {
5039 OMPPrivateScope LoopGlobals(CGF);
5040 if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
5041 for (const Expr *E : LD->counters()) {
5042 const auto *VD = dyn_cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5043 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
5044 LValue GlobLVal = CGF.EmitLValue(E);
5045 LoopGlobals.addPrivate(
5046 VD, [&GlobLVal]() { return GlobLVal.getAddress(); });
5047 }
5048 if (isa<OMPCapturedExprDecl>(VD)) {
5049 // Emit only those that were not explicitly referenced in clauses.
5050 if (!CGF.LocalDeclMap.count(VD))
5051 CGF.EmitVarDecl(*VD);
5052 }
5053 }
5054 for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
5055 if (!C->getNumForLoops())
5056 continue;
5057 for (unsigned I = LD->getCollapsedNumber(),
5058 E = C->getLoopNumIterations().size();
5059 I < E; ++I) {
5060 if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
5061 cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
5062 // Emit only those that were not explicitly referenced in clauses.
5063 if (!CGF.LocalDeclMap.count(VD))
5064 CGF.EmitVarDecl(*VD);
5065 }
5066 }
5067 }
5068 }
5069 LoopGlobals.Privatize();
5070 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());
5071 }
5072 };
5073 OMPSimdLexicalScope Scope(*this, D);
5074 CGM.getOpenMPRuntime().emitInlinedDirective(
5075 *this,
5076 isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
5077 : D.getDirectiveKind(),
5078 CodeGen);
5079 }
5080
clang::CodeGen::CodeGenTypes::arrangeBuiltinFunctionDeclaration
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:659
clang::CodeGen::RValue::isAggregate
bool isAggregate() const
Definition: CGValue.h:54
clang::Builtin::ID
ID
Definition: Builtins.h:49
clang::OMPDistributeSimdDirective
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3248
clang::OMPLoopDirective::getNextUpperBound
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:865
clang::OMPMasterDirective
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1431
clang::OMPTaskDirective
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1771
getCanonicalDecl
static const Decl * getCanonicalDecl(const Decl *D)
Definition: IndexingContext.cpp:282
clang::FunctionDecl
Represents a function declaration or definition.
Definition: Decl.h:1738
OMPThreadLimitClause
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:4342
clang::CodeGen::CodeGenFunction::VlaSizePair
Definition: CodeGenFunction.h:2407
clang::OMPLoopDirective::getUpperBoundVariable
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:833
clang::ImplicitParamDecl::Other
Other implicit parameter.
Definition: Decl.h:1510
clang::CodeGen::CodeGenFunction::EmitOMPTaskwaitDirective
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
Definition: CGStmtOpenMP.cpp:3221
clang::CodeGen::CodeGenFunction::EmitOMPTargetTaskBasedDirective
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
Definition: CGStmtOpenMP.cpp:3068
clang::OMPD_unknown
Definition: OpenMPKinds.h:29
emitOMPAtomicWriteExpr
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3636
clang::CodeGen::OMPTaskDataTy
Definition: CGOpenMPRuntime.h:92
OMPCopyinClause
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:2931
clang::Ctor_Complete
Complete object ctor.
Definition: ABI.h:26
clang::PointerType
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2537
clang::OpenMPScheduleTy
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:141
clang::QualType
A (possibly-)qualified type.
Definition: Type.h:638
clang::CapturedStmt::capture_init_begin
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:3279
clang::CodeGen::CodeGenModule::getTypes
CodeGenTypes & getTypes()
Definition: CodeGenModule.h:676
clang::Type::isArrayType
bool isArrayType() const
Definition: Type.h:6345
clang::OMPExecutableDirective::clauses
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:260
clang::CodeGen::CodeGenFunction::EmitBranchOnBoolExpr
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
Definition: CodeGenFunction.cpp:1532
emitOMPAtomicReadExpr
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3612
clang::CodeGen::CodeGenFunction::EmitVarDecl
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:158
clang::CapturedDecl::param_begin
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:4124
clang::CodeGen::CodeGenFunction::CreateMemTemp
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:139
clang::CodeGen::CodeGenFunction::HaveInsertPoint
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
Definition: CodeGenFunction.h:2078
clang::CodeGen::CodeGenFunction::EmitOMPDistributeDirective
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
Definition: CGStmtOpenMP.cpp:3497
clang::CodeGen::CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
Definition: CGStmtOpenMP.cpp:4827
clang::diff::Update
Definition: ASTDiff.h:31
llvm
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:30
emitDistributeParallelForInnerBounds
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Definition: CGStmtOpenMP.cpp:2050
clang::Expr::EvaluateAsInt
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
Definition: ExprConstant.cpp:10951
clang::OMPClauseWithPreInit::get
static OMPClauseWithPreInit * get(OMPClause *C)
Definition: OpenMPClause.cpp:39
clang::Stmt
Stmt - This represents one statement.
Definition: Stmt.h:66
clang::OMPInReductionClause
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Definition: OpenMPClause.h:2362
clang::OMPLoopDirective::getLowerBoundVariable
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:825
clang::CodeGen::CodeGenFunction::EmitStoreThroughGlobalRegLValue
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2149
clang::Type::getPointeeType
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:505
clang::CodeGen::CodeGenFunction::OMPTargetDataInfo
Definition: CodeGenFunction.h:3094
clang::CodeGen::CodeGenFunction::RunCleanupsScope::requiresCleanups
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
Definition: CodeGenFunction.h:735
clang::CodeGen::CodeGenFunction::EmitOMPTargetSimdDirective
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
Definition: CGStmtOpenMP.cpp:2201
clang::CodeGen::CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
Definition: CGStmtOpenMP.cpp:3744
clang::CodeGen::OMPTaskDataTy::Dependences
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
Definition: CGOpenMPRuntime.h:103
clang::OMPForSimdDirective
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1181
clang::CodeGen::OMPTaskDataTy::LastprivateCopies
SmallVector< const Expr *, 4 > LastprivateCopies
Definition: CGOpenMPRuntime.h:99
clang::ASTContext::getLValueReferenceType
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Definition: ASTContext.cpp:2967
clang::Decl
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:87
OMPGrainsizeClause
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:4464
clang::OMPTeamsDistributeParallelForDirective
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3659
clang::OMPExecutableDirective::getBeginLoc
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:168
clang::CapturedDecl::isNothrow
bool isNothrow() const
Definition: Decl.cpp:4469
clang::CodeGen::CodeGenFunction::CurCodeDecl
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
Definition: CodeGenFunction.h:298
emitOMPAtomicUpdateExpr
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3768
clang::CodeGen::CodeGenFunction::getTypeSize
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
Definition: CGStmtOpenMP.cpp:233
clang::OMPIfClause
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:240
llvm::DenseSet
Definition: Sema.h:66
clang::OMPExecutableDirective::getInnermostCapturedStmt
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
Definition: StmtOpenMP.h:226
clang::QualType::getNonReferenceType
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:6245
clang::CodeGen::CodeGenFunction::EmitOMPOrderedDirective
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
Definition: CGStmtOpenMP.cpp:3516
clang::CodeGen::LValue::isVolatile
bool isVolatile() const
Definition: CGValue.h:301
clang::IntegerLiteral
Definition: Expr.h:1308
OMPPriorityClause
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:4407
mapParam
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
Definition: CGStmtOpenMP.cpp:4851
clang::Type
The base class of the type hierarchy.
Definition: Type.h:1407
clang::OMPTargetTeamsDistributeDirective
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:3796
clang::OMPC_DIST_SCHEDULE_unknown
Definition: OpenMPKinds.h:112
clang::CodeGen::CodeGenFunction::EmitStoreThroughLValue
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1914
emitOMPAtomicExpr
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3905
clang::CapturedStmt::getCapturedRecordDecl
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:3223
clang::ArrayType
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition: Type.h:2812
clang::CodeGen::CodeGenFunction::CGCapturedStmtInfo::lookup
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
Definition: CodeGenFunction.h:374
clang::ASTContext::getTargetInfo
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:690
clang::OMPLoopDirective::getCombinedParForInDistCond
Expr * getCombinedParForInDistCond() const
Definition: StmtOpenMP.h:953
clang::CodeGen::AlignmentSource::Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
emitCommonOMPParallelDirective
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Definition: CGStmtOpenMP.cpp:1225
clang::CodeGen::CodeGenFunction::EmitLoadOfPointer
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition: CGExpr.cpp:2322
clang::FPOptions
Floating point control options.
Definition: LangOptions.h:307
clang::OMPParallelForDirective
This represents &#39;#pragma omp parallel for&#39; directive.
Definition: StmtOpenMP.h:1552
clang::CodeGen::CodeGenFunction::GenerateOpenMPCapturedVars
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
Definition: CGStmtOpenMP.cpp:253
clang::OMPTargetTeamsDistributeParallelForDirective
This represents &#39;#pragma omp target teams distribute parallel for&#39; combined directive.
Definition: StmtOpenMP.h:3864
clang::OMPLoopDirective::getCombinedEnsureUpperBound
Expr * getCombinedEnsureUpperBound() const
Definition: StmtOpenMP.h:917
clang::CodeGen::CodeGenFunction::EmitAtomicStore
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1895
clang::CodeGen::CodeGenFunction::ConstantFoldsToSimpleInteger
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Definition: CodeGenFunction.cpp:1495
clang::LoopExit
Represents a point when we exit a loop.
Definition: ProgramPoint.h:715
clang::CodeGen::CodeGenFunction::EmitOMPCopy
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
Definition: CGStmtOpenMP.cpp:689
Value
Value
Definition: UninitializedValues.cpp:102
clang::CodeGen::OMPTaskDataTy::ReductionCopies
SmallVector< const Expr *, 4 > ReductionCopies
Definition: CGOpenMPRuntime.h:101
clang::OMPExecutableDirective::getEndLoc
SourceLocation getEndLoc() const
Returns ending location of directive.
Definition: StmtOpenMP.h:170
clang::CodeGen::CodeGenFunction::EmitOMPParallelForSimdDirective
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
Definition: CGStmtOpenMP.cpp:2758
createSectionLVal
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
Definition: CGStmtOpenMP.cpp:2507
clang::OMPTargetExitDataDirective
This represents &#39;#pragma omp target exit data&#39; directive.
Definition: StmtOpenMP.h:2463
clang::CodeGen::CodeGenFunction::GetAddrOfLocalVar
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
Definition: CodeGenFunction.h:2344
clang::ASTContext::getCanonicalParamType
CanQualType getCanonicalParamType(QualType T) const
Return the canonical parameter type corresponding to the specific potentially non-canonical one...
Definition: ASTContext.cpp:5023
clang::CodeGen::CodeGenFunction::GenerateOpenMPCapturedStmtFunction
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
Definition: CGStmtOpenMP.cpp:550
clang::VarDecl
Represents a variable declaration or definition.
Definition: Decl.h:813
clang::OMPPrivateClause
This represents clause &#39;private&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:1441
clang::CodeGen::CodeGenFunction::EmitOMPCriticalDirective
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
Definition: CGStmtOpenMP.cpp:2730
clang::OMPTargetParallelForDirective::hasCancel
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:2646
clang::OMPNumThreadsClause
This represents &#39;num_threads&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:382
clang::SC_Static
Definition: Specifiers.h:210
clang::SC_None
Definition: Specifiers.h:208
clang::OMPLoopDirective::getBody
const Stmt * getBody() const
Definition: StmtOpenMP.h:959
clang::FunctionProtoType::ExtProtoInfo
Extra information about a function prototype.
Definition: Type.h:3767
clang::CodeGen::CodeGenFunction::getProfileCount
uint64_t getProfileCount(const Stmt *S)
Get the profiler&#39;s count for the given statement.
Definition: CodeGenFunction.h:1341
clang::OMPLoopDirective::getCombinedUpperBoundVariable
Expr * getCombinedUpperBoundVariable() const
Definition: StmtOpenMP.h:911
clang::OMPLoopDirective::getCalcLastIteration
Expr * getCalcLastIteration() const
Definition: StmtOpenMP.h:793
clang::CodeGen::Address::getPointer
llvm::Value * getPointer() const
Definition: Address.h:38
clang::CapturedDecl::getContextParamPosition
unsigned getContextParamPosition() const
Definition: Decl.h:4118
clang::CodeGen::CodeGenFunction::EmitOMPInnerLoop
void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
Definition: CGStmtOpenMP.cpp:1324
OMPFlushClause
This represents implicit clause &#39;flush&#39; for the &#39;#pragma omp flush&#39; directive.
Definition: OpenMPClause.h:3251
clang::CodeGen::CodeGenFunction::EmitOMPLastprivateClauseFinal
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
Definition: CGStmtOpenMP.cpp:984
clang::CodeGen::CodeGenFunction::EmitOMPTeamsDistributeDirective
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
Definition: CGStmtOpenMP.cpp:4301
emitTargetTeamsDistributeRegion
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
Definition: CGStmtOpenMP.cpp:4210
clang::OMPAtomicDirective::isXLHSInRHSPart
bool isXLHSInRHSPart() const
Return true if helper update expression has form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; and...
Definition: StmtOpenMP.h:2260
clang::CodeGen::CodeGenFunction::EmitOMPTargetParallelDeviceFunction
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
Definition: CGStmtOpenMP.cpp:4749
OMPNogroupClause
This represents &#39;nogroup&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:4513
clang::CodeGen::CodeGenFunction::JumpDest
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
Definition: CodeGenFunction.h:235
clang::CodeGen::CodeGenFunction::EmitOMPWorksharingLoop
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
Definition: CGStmtOpenMP.cpp:2221
clang::OMPSafelenClause
This represents &#39;safelen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:447
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Definition: CGStmtOpenMP.cpp:4487
clang::CodeGen::TEK_Aggregate
Definition: CodeGenFunction.h:108
clang::NamedDecl::getIdentifier
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
Definition: Decl.h:270
clang::Expr::IgnoreImpCasts
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
Definition: Expr.h:3162
clang::CodeGen::CodeGenFunction::EmitOMPTaskLoopSimdDirective
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
Definition: CGStmtOpenMP.cpp:5004
clang::CodeGen::OMPTaskDataTy::PrivateVars
SmallVector< const Expr *, 4 > PrivateVars
Definition: CGOpenMPRuntime.h:93
clang::RecordDecl
Represents a struct/union/class.
Definition: Decl.h:3593
clang::CodeGen::CodeGenFunction::LambdaCaptureFields
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
Definition: CodeGenFunction.h:483
clang::IdentifierInfo
One of these records is kept for each identifier that is lexed.
Definition: IdentifierTable.h:58
clang::CodeGen::CodeGenFunction::EmitOMPSimdDirective
void EmitOMPSimdDirective(const OMPSimdDirective &S)
Definition: CGStmtOpenMP.cpp:1766
clang::OMPParallelDirective
This represents &#39;#pragma omp parallel&#39; directive.
Definition: StmtOpenMP.h:276
clang::CodeGen::LValue::getAddress
Address getAddress() const
Definition: CGValue.h:327
clang::CodeGen::CodeGenFunction
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Definition: CodeGenFunction.h:227
clang::ASTContext
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:155
clang::NestedNameSpecifierLoc
A C++ nested-name-specifier augmented with source location information.
Definition: NestedNameSpecifier.h:237
llvm::Optional
Definition: LLVM.h:38
OMPSIMDClause
This represents &#39;simd&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:3520
clang::CodeGen::CodeGenTypeCache::Int64Ty
llvm::IntegerType * Int64Ty
Definition: CodeGenTypeCache.h:38
clang::CodeGen::CodeGenFunction::EmitOMPTaskLoopBasedDirective
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
Definition: CGStmtOpenMP.cpp:4859
clang::CodeGen::TEK_Complex
Definition: CodeGenFunction.h:107
clang::CodeGen::CodeGenFunction::OMPPrivateScope
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
Definition: CodeGenFunction.h:894
clang::RecordDecl::fields
field_range fields() const
Definition: Decl.h:3784
emitTargetRegion
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:4088
clang::OMPLoopDirective::getEnsureUpperBound
Expr * getEnsureUpperBound() const
Definition: StmtOpenMP.h:849
clang::CodeGen::OMPTaskDataTy::LastprivateVars
SmallVector< const Expr *, 4 > LastprivateVars
Definition: CGOpenMPRuntime.h:98
clang::Stmt::getBeginLoc
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Stmt.cpp:288
clang::CodeGen::CodeGenFunction::EmitLoadOfReference
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
Definition: CGExpr.cpp:2300
clang::FieldDecl
Represents a member of a struct/union/class.
Definition: Decl.h:2579
clang::OMPLastprivateClause
This represents clause &#39;lastprivate&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:1662
clang::CodeGen::CodeGenFunction::EmitOMPTaskBasedDirective
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
Definition: CGStmtOpenMP.cpp:2783
clang::CodeGen::CodeGenFunction::EmitOMPDistributeLoop
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
Definition: CGStmtOpenMP.cpp:3276
clang::CodeGen::LValue::getAlignment
CharUnits getAlignment() const
Definition: CGValue.h:316
clang::CodeGen::CodeGenTypeCache::SizeTy
llvm::IntegerType * SizeTy
Definition: CodeGenTypeCache.h:48
clang::OMPExecutableDirective::getCapturedStmt
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive...
Definition: StmtOpenMP.h:209
clang::CodeGen::CodeGenFunction::OpaqueValueMapping
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Definition: CodeGenFunction.h:1141
clang::OMPLoopDirective::getInc
Expr * getInc() const
Definition: StmtOpenMP.h:809
emitSimdlenSafelenClause
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, bool IsMonotonic)
Definition: CGStmtOpenMP.cpp:1584
clang::OMPLoopDirective::updates
ArrayRef< Expr * > updates()
Definition: StmtOpenMP.h:989
clang::Type::isReferenceType
bool isReferenceType() const
Definition: Type.h:6308
clang::CodeGen::CodeGenFunction::EmitOMPAggregateAssign
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
Definition: CGStmtOpenMP.cpp:628
clang::CodeGen::CodeGenFunction::EmitOMPPrivateLoopCounters
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
Definition: CGStmtOpenMP.cpp:1482
clang::DeclRefExpr::Create
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr)
Definition: Expr.cpp:406
clang::OMPTargetSimdDirective
This represents &#39;#pragma omp target simd&#39; directive.
Definition: StmtOpenMP.h:3384
clang::CodeGen::CodeGenFunction::Builder
CGBuilderTy Builder
Definition: CodeGenFunction.h:263
clang::CodeGen::CodeGenFunction::AutoVarEmission::getAllocatedAddress
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself...
Definition: CodeGenFunction.h:2724
clang::OMPExecutableDirective::getDirectiveKind
OpenMPDirectiveKind getDirectiveKind() const
Definition: StmtOpenMP.h:244
clang::DeclaratorDecl::getBeginLoc
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Decl.h:739
clang::OMPBarrierDirective
This represents &#39;#pragma omp barrier&#39; directive.
Definition: StmtOpenMP.h:1883
clang::OMPLoopDirective
This is a common base class for loop directives (&#39;omp simd&#39;, &#39;omp for&#39;, &#39;omp for simd&#39; etc...
Definition: StmtOpenMP.h:338
clang::OMPCriticalDirective
This represents &#39;#pragma omp critical&#39; directive.
Definition: StmtOpenMP.h:1478
emitOMPAtomicRMW
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
Definition: CGStmtOpenMP.cpp:3650
clang::OMPCancellationPointDirective::getCancelRegion
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:2762
llvm::SmallVectorImpl
Definition: LLVM.h:37
clang::ASTContext::Idents
IdentifierTable & Idents
Definition: ASTContext.h:566
emitOMPAtomicCaptureExpr
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3823
clang::VK_RValue
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
Definition: Specifiers.h:110
clang::CodeGen::CodeGenFunction::EmitStoreOfScalar
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
Definition: CodeGenFunction.h:3424
clang::CodeGen::CodeGenFunction::EmitOMPDistributeParallelForSimdDirective
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
Definition: CGStmtOpenMP.cpp:2168
clang::CodeGen::CodeGenFunction::EmitOMPLastprivateClauseInit
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
Definition: CGStmtOpenMP.cpp:925
clang::CodeGen::CodeGenFunction::EmitAnyExpr
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
Definition: CGExpr.cpp:194
OMPCopyprivateClause
This represents clause &#39;copyprivate&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:3096
clang::OpenMPDistScheduleClauseKind
OpenMPDistScheduleClauseKind
OpenMP attributes for &#39;dist_schedule&#39; clause.
Definition: OpenMPKinds.h:109
clang::OMPDistributeParallelForDirective
This represents &#39;#pragma omp distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3099
clang::CapturedDecl::getBody
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
Definition: Decl.cpp:4466
clang::OMPTeamsDistributeParallelForSimdDirective
This represents &#39;#pragma omp teams distribute parallel for simd&#39; composite directive.
Definition: StmtOpenMP.h:3588
clang::BinaryOperatorKind
BinaryOperatorKind
Definition: OperationKinds.h:26
clang::OMPC_uniform
Definition: OpenMPKinds.h:38
clang::CodeGen::CodeGenFunction::hasScalarEvaluationKind
static bool hasScalarEvaluationKind(QualType T)
Definition: CodeGenFunction.h:2028
clang::CapturedStmt::const_capture_init_iterator
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
Definition: Stmt.h:3266
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
Definition: CGStmtOpenMP.cpp:4293
clang::OMPLoopDirective::finals
ArrayRef< Expr * > finals()
Definition: StmtOpenMP.h:995
clang::OMPLoopDirective::getIsLastIterVariable
Expr * getIsLastIterVariable() const
Definition: StmtOpenMP.h:817
clang::CodeGen::CodeGenFunction::emitOMPSimpleStore
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3594
clang::OpenMPScheduleTy::Schedule
OpenMPScheduleClauseKind Schedule
Definition: OpenMPKinds.h:142
clang::CharUnits
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
emitEmptyOrdered
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
Definition: CGStmtOpenMP.cpp:1986
clang::isOpenMPTeamsDirective
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
Definition: OpenMPKinds.cpp:857
emitDistributeParallelForDistributeInnerBoundParams
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value *> &CapturedVars)
Definition: CGStmtOpenMP.cpp:2110
clang::TargetInfo::hasBuiltinAtomic
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
Definition: TargetInfo.h:614
clang::CodeGen::CodeGenFunction::CodeGenLoopBoundsTy
llvm::function_ref< std::pair< LValue, LValue > CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
Definition: CodeGenFunction.h:280
clang::OMPAtomicDirective::getX
Expr * getX()
Get &#39;x&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2244
clang::Type::getAsArrayTypeUnsafe
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
Definition: Type.h:6797
clang::CodeGen::CodeGenFunction::EmitOMPCancellationPointDirective
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
Definition: CGStmtOpenMP.cpp:4495
clang::CapturedDecl::param_end
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
Definition: Decl.h:4126
clang::CodeGen::Address::getAlignment
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:67
clang::CodeGen::CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
Definition: CGStmtOpenMP.cpp:4369
clang::CodeGen::CodeGenFunction::EmitOMPTargetDirective
void EmitOMPTargetDirective(const OMPTargetDirective &S)
Definition: CGStmtOpenMP.cpp:4115
clang::OMPLoopDirective::getIterationVariable
Expr * getIterationVariable() const
Definition: StmtOpenMP.h:785
clang::CodeGen::RValue::isComplex
bool isComplex() const
Definition: CGValue.h:53
clang::BinaryOperator
A builtin binary operation expression such as "x + y" or "x <= y".
Definition: Expr.h:3287
clang::QualType::isVolatileQualified
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
Definition: Type.h:6142
clang::OpenMPScheduleTy::M2
OpenMPScheduleClauseModifier M2
Definition: OpenMPKinds.h:144
clang::CodeGen::LoopInfoStack::setVectorizeWidth
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
Definition: CGLoopInfo.h:169
clang::CodeGen::OMPTaskDataTy::PrivateCopies
SmallVector< const Expr *, 4 > PrivateCopies
Definition: CGOpenMPRuntime.h:94
clang::CodeGen::CodeGenFunction::OMPMapVars::restore
void restore(CodeGenFunction &CGF)
Restores original addresses of the variables.
Definition: CodeGenFunction.h:865
emitOutlinedOrderedFunction
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S)
Definition: CGStmtOpenMP.cpp:3506
clang::CodeGen::CodeGenFunction::EmitOMPTargetUpdateDirective
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
Definition: CGStmtOpenMP.cpp:5010
clang::OMPCancellationPointDirective
This represents &#39;#pragma omp cancellation point&#39; directive.
Definition: StmtOpenMP.h:2718
clang::ASTContext::getDeclAlign
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
Definition: ASTContext.cpp:1556
EmitOMPHelperVar
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
Definition: CGStmtOpenMP.cpp:1672
clang::CodeGen::ReductionCodeGen::adjustPrivateAddress
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations...
Definition: CGOpenMPRuntime.cpp:1132
clang::VariableArrayType::getSizeExpr
Expr * getSizeExpr() const
Definition: Type.h:2991
clang::Scope
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:41
clang::RecordDecl::field_begin
field_iterator field_begin() const
Definition: Decl.cpp:4145
clang::OMPFinalClause
This represents &#39;final&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:330
clang::isOpenMPTargetExecutionDirective
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
Definition: OpenMPKinds.cpp:835
clang::CodeGen::CodeGenFunction::EmitOMPParallelDirective
void EmitOMPParallelDirective(const OMPParallelDirective &S)
Definition: CGStmtOpenMP.cpp:1271
clang::CodeGen::CodeGenFunction::createBasicBlock
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
Definition: CodeGenFunction.h:2037
clang::OMPTeamsDirective
This represents &#39;#pragma omp teams&#39; directive.
Definition: StmtOpenMP.h:2661
clang::CodeGen::CodeGenFunction::EmitIgnoredExpr
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition: CGExpr.cpp:182
clang::CodeGen::OMPTaskDataTy::FirstprivateCopies
SmallVector< const Expr *, 4 > FirstprivateCopies
Definition: CGOpenMPRuntime.h:96
clang::OMPLoopDirective::getInit
Expr * getInit() const
Definition: StmtOpenMP.h:805
emitPostUpdateForReductionClause
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Definition: CGStmtOpenMP.cpp:1190
clang::OMPReductionClause
This represents clause &#39;reduction&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:1921
clang::CodeGen::ReductionCodeGen::getBaseDecl
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
Definition: CGOpenMPRuntime.h:194
clang::OMPTeamsDistributeSimdDirective
This represents &#39;#pragma omp teams distribute simd&#39; combined directive.
Definition: StmtOpenMP.h:3518
clang::CodeGen::CodeGenFunction::RunCleanupsScope::ForceCleanup
void ForceCleanup(std::initializer_list< llvm::Value **> ValuesToReload={})
Force the emission of cleanups now, instead of waiting until this object is destroyed.
Definition: CodeGenFunction.h:745
clang::CodeGen::LValue::isSimple
bool isSimple() const
Definition: CGValue.h:252
clang::isOpenMPWorksharingDirective
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
Definition: OpenMPKinds.cpp:803
clang::CodeGen::CodeGenTypeCache::getPointerAlign
CharUnits getPointerAlign() const
Definition: CodeGenTypeCache.h:109
clang::CodeGen::CodeGenFunction::EmitOMPSimdFinal
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Definition: CGStmtOpenMP.cpp:1619
clang::OK_Ordinary
An ordinary object is located at an address in memory.
Definition: Specifiers.h:126
clang::CodeGen::OMPTaskDataTy::ReductionOps
SmallVector< const Expr *, 4 > ReductionOps
Definition: CGOpenMPRuntime.h:102
clang::CodeGen::CodeGenFunction::OMPCancelStackRAII
Controls insertion of cancellation exit blocks in worksharing constructs.
Definition: CodeGenFunction.h:2944
clang::CodeGen::CodeGenFunction::incrementProfileCounter
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler&#39;s counter for the given statement by StepV.
Definition: CodeGenFunction.h:1334
clang::CodeGen::CodeGenFunction::CodeGenDispatchBoundsTy
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * > CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
Definition: CodeGenFunction.h:286
clang::CapturedDecl
Represents the body of a CapturedStmt, and serves as its DeclContext.
Definition: Decl.h:4044
llvm::ArrayRef
Definition: LLVM.h:32
llvm::SmallString
Definition: LLVM.h:35
clang::CodeGen::OMPTaskDataTy::ReductionVars
SmallVector< const Expr *, 4 > ReductionVars
Definition: CGOpenMPRuntime.h:100
convertToScalarValue
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3543
clang::FunctionDecl::Create
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, bool isConstexprSpecified=false)
Definition: Decl.h:1875
clang::CodeGen::OMPTaskDataTy::Reductions
llvm::Value * Reductions
Definition: CGOpenMPRuntime.h:107
emitOutlinedFunctionPrologue
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address >> &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value *>> &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
Definition: CGStmtOpenMP.cpp:363
clang::CodeGen::CodeGenFunction::EmitLoadOfScalar
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
Definition: CodeGenFunction.h:3402
convertToComplexValue
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3556
clang::ImplicitParamDecl::Create
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
Definition: Decl.cpp:4404
clang::OMPTargetParallelForSimdDirective
This represents &#39;#pragma omp target parallel for simd&#39; directive.
Definition: StmtOpenMP.h:3316
clang::OMPLoopDirective::private_counters
ArrayRef< Expr * > private_counters()
Definition: StmtOpenMP.h:977
clang::CompoundStmt
CompoundStmt - This represents a group of statements like { stmt stmt }.
Definition: Stmt.h:1241
clang::CodeGen::CodeGenFunction::ComplexPairTy
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Definition: CodeGenFunction.h:261
clang::CodeGen::CodeGenFunction::EmitAutoVarAlloca
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
Definition: CGDecl.cpp:1298
clang::CodeGen::CodeGenFunction::getContext
ASTContext & getContext() const
Definition: CodeGenFunction.h:1596
clang::OMPSectionDirective::hasCancel
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1361
clang::CodeGen::CodeGenFunction::EmitOMPParallelSectionsDirective
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
Definition: CGStmtOpenMP.cpp:2771
clang::OMPTaskgroupDirective
This represents &#39;#pragma omp taskgroup&#39; directive.
Definition: StmtOpenMP.h:1971
clang::CodeGen::RValue
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
Definition: CGValue.h:39
OMPAlignedClause
This represents clause &#39;aligned&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:2845
clang::ImplicitCastExpr::Create
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat)
Definition: Expr.cpp:1844
clang::CodeGen::ReductionCodeGen
Class intended to support codegen of all kind of the reduction clauses.
Definition: CGOpenMPRuntime.h:114
clang::CodeGen::CodeGenFunction::OMPPrivateScope::isGlobalVarCaptured
bool isGlobalVarCaptured(const VarDecl *VD) const
Checks if the global variable is captured in current function.
Definition: CodeGenFunction.h:936
clang::OMPLoopDirective::getCombinedLowerBoundVariable
Expr * getCombinedLowerBoundVariable() const
Definition: StmtOpenMP.h:905
clang::CodeGen::CodeGenFunction::OMPMapVars
The class used to assign some variables some temporarily addresses.
Definition: CodeGenFunction.h:815
clang::OMPTaskReductionClause
This represents clause &#39;task_reduction&#39; in the &#39;#pragma omp taskgroup&#39; directives.
Definition: OpenMPClause.h:2143
clang::CodeGen::CodeGenFunction::EmitOMPLoopBody
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
Definition: CGStmtOpenMP.cpp:1298
clang::CharUnits::getQuantity
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:179
clang::ASTContext::getTrivialTypeSourceInfo
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
Definition: ASTContext.cpp:2600
createImplicitFirstprivateForType
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3039
clang::ArrayType::Normal
Definition: Type.h:2819
clang::OMPDistributeDirective
This represents &#39;#pragma omp distribute&#39; directive.
Definition: StmtOpenMP.h:2972
OMPDependClause
This represents implicit clause &#39;depend&#39; for the &#39;#pragma omp task&#39; directive.
Definition: OpenMPClause.h:3312
clang::CodeGen::CodeGenFunction::getEvaluationKind
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
Definition: CodeGenFunction.cpp:192
clang::CodeGen::CodeGenFunction::OMPTargetDataInfo::NumberOfTargetItems
unsigned NumberOfTargetItems
Definition: CodeGenFunction.h:3098
clang::OMPExecutableDirective::getAssociatedStmt
const Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
Definition: StmtOpenMP.h:196
clang::CodeGen::CodeGenFunction::OMPPrivateScope::addPrivate
bool addPrivate(const VarDecl *LocalVD, const llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
Definition: CodeGenFunction.h:908
clang::OMPParallelForDirective::hasCancel
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1615
clang::OMPProcBindClause
This represents &#39;proc_bind&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:675
clang::ValueDecl
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Definition: Decl.h:637
clang::Expr
This represents one expression.
Definition: Expr.h:106
emitTargetParallelForRegion
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:4771
clang::codegenoptions::LimitedDebugInfo
Emit only debug info necessary for generating line number tables (-gline-tables-only).
Definition: DebugInfoOptions.h:32
clang::CodeGen::CodeGenFunction::EmitAutoVarInit
void EmitAutoVarInit(const AutoVarEmission &emission)
Definition: CGDecl.cpp:1578
clang::CodeGen::Address::invalid
static Address invalid()
Definition: Address.h:35
CodeGenModule.h
clang::CodeGen::CodeGenFunction::LoopStack
LoopInfoStack LoopStack
Definition: CodeGenFunction.h:262
clang::VarDecl::hasLocalStorage
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
Definition: Decl.h:1036
clang::CodeGen::CodeGenFunction::RunCleanupsScope
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
Definition: CodeGenFunction.h:700
clang::OMPSimdlenClause
This represents &#39;simdlen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:501
clang::Stmt::IgnoreContainers
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top, if IgnoreCaptured is true.
Definition: Stmt.cpp:147
clang::CodeGen::CodeGenFunction::EmitComplexToScalarConversion
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type, where the destination type is an LLVM scalar type.
Definition: CGExprScalar.cpp:4310
clang::CodeGen::CodeGenFunction::CodeGenOrderedTy
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
Definition: CodeGenFunction.h:275
clang::CodeGen::RValue::getComplexVal
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:66
clang::isOpenMPParallelDirective
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
Definition: OpenMPKinds.cpp:822
clang::OMPC_threadprivate
Definition: OpenMPKinds.h:37
clang::CodeGen::CodeGenFunction::EmitOMPTeamsDirective
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
Definition: CGStmtOpenMP.cpp:4148
clang::CodeGen::CodeGenFunction::VlaSizePair::NumElts
llvm::Value * NumElts
Definition: CodeGenFunction.h:2408
clang::Type::castAs
const T * castAs() const
Member-template castAs<specific type>.
Definition: Type.h:6811
clang::CodeGen::CodeGenFunction::JumpDest::getBlock
llvm::BasicBlock * getBlock() const
Definition: CodeGenFunction.h:243
emitTargetTeamsDistributeParallelForRegion
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:4392
clang::OMPForDirective::hasCancel
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1165
emitDispatchForLoopBounds
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
Definition: CGStmtOpenMP.cpp:2459
clang::OMPTargetTeamsDistributeParallelForSimdDirective
This represents &#39;#pragma omp target teams distribute parallel for simd&#39; combined directive.
Definition: StmtOpenMP.h:3948
emitPreCond
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
Definition: CGStmtOpenMP.cpp:1531
clang::OpenMPClauseKind
OpenMPClauseKind
OpenMP clauses.
Definition: OpenMPKinds.h:33
clang::CodeGen::Address::getType
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:44
clang::OMPExecutableDirective::getClausesOfKind
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause *> Clauses)
Definition: StmtOpenMP.h:130
clang::OMPTargetTeamsDistributeSimdDirective
This represents &#39;#pragma omp target teams distribute simd&#39; combined directive.
Definition: StmtOpenMP.h:4021
clang::CodeGen::CodeGenFunction::EmitOMPSimdInit
void EmitOMPSimdInit(const OMPLoopDirective &D, bool IsMonotonic=false)
Helpers for the OpenMP loop directives.
Definition: CGStmtOpenMP.cpp:1611
clang::OMPOrderedClause
This represents &#39;ordered&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:1129
clang::CapturedStmt::children
child_range children()
Definition: Stmt.cpp:1260
clang::OMPLoopDirective::getDistInc
Expr * getDistInc() const
Definition: StmtOpenMP.h:893
clang::OMPLoopDirective::getNextLowerBound
Expr * getNextLowerBound() const
Definition: StmtOpenMP.h:857
clang::Expr::getType
QualType getType() const
Definition: Expr.h:128
clang::OMPLoopDirective::getPrevEnsureUpperBound
Expr * getPrevEnsureUpperBound() const
Definition: StmtOpenMP.h:899
clang::OMPForDirective
This represents &#39;#pragma omp for&#39; directive.
Definition: StmtOpenMP.h:1104
emitTargetTeamsRegion
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
Definition: CGStmtOpenMP.cpp:4165
Stmt.h
clang::CodeGen::OMPTaskDataTy::FirstprivateVars
SmallVector< const Expr *, 4 > FirstprivateVars
Definition: CGOpenMPRuntime.h:95
clang::CapturedDecl::Create
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
Definition: Decl.cpp:4454
clang::CodeGen::CodeGenFunction::EmitOMPMasterDirective
void EmitOMPMasterDirective(const OMPMasterDirective &S)
Definition: CGStmtOpenMP.cpp:2721
emitDistributeParallelForDispatchBounds
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the &#39;for&#39; loop has a dispatch schedule (e.g.
Definition: CGStmtOpenMP.cpp:2093
clang::CharUnits::alignmentOfArrayElement
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
Definition: CharUnits.h:197
clang::CodeGen::LoopInfoStack::setParallel
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
Definition: CGLoopInfo.h:144
clang::OMPTargetTeamsDirective
This represents &#39;#pragma omp target teams&#39; directive.
Definition: StmtOpenMP.h:3737
clang::CodeGen::CodeGenFunction::MakeNaturalAlignAddrLValue
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
Definition: CodeGenFunction.cpp:164
clang::SourceRange::getEnd
SourceLocation getEnd() const
Definition: SourceLocation.h:208
clang::UnaryOperator
UnaryOperator - This represents the unary-expression&#39;s (except sizeof and alignof), the postinc/postdec operators from postfix-expression, and various extensions.
Definition: Expr.h:1896
clang::isOpenMPTaskLoopDirective
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
Definition: OpenMPKinds.cpp:818
clang::ASTContext::getFunctionType
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
Definition: ASTContext.h:1380
clang::CodeGen::CodeGenFunction::EmitOMPBarrierDirective
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
Definition: CGStmtOpenMP.cpp:3217
clang::CodeGen::CodeGenFunction::EmitOMPLinearClauseInit
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
Definition: CGStmtOpenMP.cpp:1373
clang::OMPCancelDirective
This represents &#39;#pragma omp cancel&#39; directive.
Definition: StmtOpenMP.h:2776
clang::OMPFirstprivateClause
This represents clause &#39;firstprivate&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:1535
clang::DeclRefExpr::getDecl
ValueDecl * getDecl()
Definition: Expr.h:1114
clang::CodeGen::CodeGenModule::getLangOpts
const LangOptions & getLangOpts() const
Definition: CodeGenModule.h:653
clang::CodeGen::CodeGenModule::getContext
ASTContext & getContext() const
Definition: CodeGenModule.h:652
emitCommonOMPTargetDirective
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
Definition: CGStmtOpenMP.cpp:4013
clang::CodeGen::CodeGenFunction::EmitAtomicLoad
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
Definition: CGAtomic.cpp:1471
CodeGenFunction.h
emitForLoopBounds
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
Definition: CGStmtOpenMP.cpp:2445
clang::VarDecl::getCanonicalDecl
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
Definition: Decl.cpp:2026
clang::GlobalDecl
GlobalDecl - represents a global declaration.
Definition: GlobalDecl.h:35
clang::CodeGen::CodeGenFunction::EmitSimpleOMPExecutableDirective
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
Definition: CGStmtOpenMP.cpp:5031
clang::OMPFlushDirective
This represents &#39;#pragma omp flush&#39; directive.
Definition: StmtOpenMP.h:2044
clang::OMPExecutableDirective::hasClausesOfKind
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind. ...
Definition: StmtOpenMP.h:162
clang::OMPParallelForSimdDirective
This represents &#39;#pragma omp parallel for simd&#39; directive.
Definition: StmtOpenMP.h:1632
clang::CodeGen::CGOpenMPRuntime
Definition: CGOpenMPRuntime.h:202
clang::OMPSeqCstClause
This represents &#39;seq_cst&#39; clause in the &#39;#pragma omp atomic&#39; directive.
Definition: OpenMPClause.h:1412
clang::OMPUntiedClause
This represents &#39;untied&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:1241
clang::Decl::hasBody
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
Definition: DeclBase.h:985
castValueFromUintptr
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV, bool isReferenceType=false)
Definition: CGStmtOpenMP.cpp:299
emitAlignedClause
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Definition: CGStmtOpenMP.cpp:1448
clang::CodeGen::AlignmentSource::Type
The l-value was considered opaque, so the alignment was determined from a type.
clang::CodeGen::CodeGenFunction::CodeGenLoopTy
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
Definition: CodeGenFunction.h:272
clang::OMPLoopDirective::getLastIteration
Expr * getLastIteration() const
Definition: StmtOpenMP.h:789
clang::OMPExecutableDirective::getSingleClause
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
Definition: StmtOpenMP.h:148
clang::OMPAtomicDirective::isPostfixUpdate
bool isPostfixUpdate() const
Return true if &#39;v&#39; expression must be updated to original value of &#39;x&#39;, false if &#39;v&#39; must be updated ...
Definition: StmtOpenMP.h:2263
clang::OMPTargetEnterDataDirective
This represents &#39;#pragma omp target enter data&#39; directive.
Definition: StmtOpenMP.h:2404
clang::CodeGen::CodeGenFunction::EmitOMPFlushDirective
void EmitOMPFlushDirective(const OMPFlushDirective &S)
Definition: CGStmtOpenMP.cpp:3264
CGCleanup.h
clang::OMPLoopDirective::getStrideVariable
Expr * getStrideVariable() const
Definition: StmtOpenMP.h:841
OMPNumTeamsClause
This represents &#39;num_teams&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:4278
clang::OpaqueValueExpr
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
Definition: Expr.h:945
clang::OMPLoopDirective::getCombinedDistCond
Expr * getCombinedDistCond() const
Definition: StmtOpenMP.h:947
clang::OMPLoopDirective::getPreInits
const Stmt * getPreInits() const
Definition: StmtOpenMP.h:813
Kind
Kind
Definition: ChrootChecker.cpp:29
clang::CapturedStmt
This captures a statement into a function.
Definition: Stmt.h:3105
clang::CodeGen::OMPTaskDataTy::Tied
bool Tied
Definition: CGOpenMPRuntime.h:109
emitTargetParallelRegion
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:4724
emitOMPSimdRegion
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:1679
clang::OMPSingleDirective
This represents &#39;#pragma omp single&#39; directive.
Definition: StmtOpenMP.h:1376
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:88
clang::IdentifierTable::get
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
Definition: IdentifierTable.h:519
OMPHintClause
This represents &#39;hint&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:4593
clang::CodeGen::CodeGenFunction::EmitOMPForDirective
void EmitOMPForDirective(const OMPForDirective &S)
Definition: CGStmtOpenMP.cpp:2469
TargetInfo.h
clang::ASTContext::getUIntPtrType
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
Definition: ASTContext.cpp:4996
clang::ASTContext::getOpenMPDefaultSimdAlign
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
Definition: ASTContext.cpp:2144
clang::OMPCapturedExprDecl
Pseudo declaration for capturing expressions.
Definition: DeclOpenMP.h:217
clang::CodeGen::OMPTaskDataTy::Final
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
Definition: CGOpenMPRuntime.h:104
clang::Type::isVariablyModifiedType
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Definition: Type.h:2095
emitInnerParallelForWhenCombined
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
Definition: CGStmtOpenMP.cpp:2128
emitTargetParallelForSimdRegion
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:4812
clang::OMPExecutableDirective
This is a basic class for representing single OpenMP executable directive.
Definition: StmtOpenMP.h:33
clang::OMPScheduleClause
This represents &#39;schedule&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:948
clang::OMPAtomicDirective::getExpr
Expr * getExpr()
Get &#39;expr&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2270
clang::OpenMPDirectiveKind
OpenMPDirectiveKind
OpenMP directives.
Definition: OpenMPKinds.h:23
clang::CodeGen::CodeGenFunction::OMPTargetDataInfo::PointersArray
Address PointersArray
Definition: CodeGenFunction.h:3096
clang::ASTContext::getBaseElementType
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
Definition: ASTContext.cpp:5519
clang::OMPCriticalDirective::getDirectiveName
DeclarationNameInfo getDirectiveName() const
Return name of the directive.
Definition: StmtOpenMP.h:1536
clang::CodeGen::CodeGenFunction::EmitStmt
void EmitStmt(const Stmt *S, ArrayRef< const Attr *> Attrs=None)
EmitStmt - Emit the code for the statement.
Definition: CGStmt.cpp:46
clang::VarDecl::hasGlobalStorage
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
Definition: Decl.h:1078
clang::VarDecl::CInit
C-style initialization with assignment.
Definition: Decl.h:818
clang::OMPTaskwaitDirective
This represents &#39;#pragma omp taskwait&#39; directive.
Definition: StmtOpenMP.h:1927
clang::CodeGen::CodeGenFunction::EmitOMPTaskgroupDirective
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
Definition: CGStmtOpenMP.cpp:3225
DeclOpenMP.h
This file defines OpenMP nodes for declarative directives.
clang::CodeGen::CodeGenFunction::EmitOMPTargetParallelForSimdDirective
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
Definition: CGStmtOpenMP.cpp:4842
clang::CodeGen::CodeGenFunction::EmitOMPTargetSimdDeviceFunction
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
Definition: CGStmtOpenMP.cpp:2187
clang::OMPClause
This is a basic class for representing single OpenMP clause.
Definition: OpenMPClause.h:51
clang::CodeGen::CodeGenFunction::EmitOMPTeamsDistributeSimdDirective
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
Definition: CGStmtOpenMP.cpp:4324
clang::ASTContext::VoidTy
CanQualType VoidTy
Definition: ASTContext.h:1016
clang::isOpenMPLoopBoundSharingDirective
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
Definition: OpenMPKinds.cpp:910
clang::CodeGen::ReductionCodeGen::emitAggregateType
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
Definition: CGOpenMPRuntime.cpp:955
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
Definition: CGStmtOpenMP.cpp:4419
llvm::SmallVector< llvm::Value *, 4 >
clang::Type::isAnyPointerType
bool isAnyPointerType() const
Definition: Type.h:6300
clang::CodeGen::CodeGenFunction::VlaSizePair::Type
QualType Type
Definition: CodeGenFunction.h:2409
clang::CodeGen::PrePostActionTy::Enter
virtual void Enter(CodeGenFunction &CGF)
Definition: CGOpenMPRuntime.h:58
clang::CodeGen::CodeGenFunction::EmitOMPSingleDirective
void EmitOMPSingleDirective(const OMPSingleDirective &S)
Definition: CGStmtOpenMP.cpp:2680
clang::CodeGen::Address
An aligned address.
Definition: Address.h:25
clang::CodeGen::CodeGenFunction::StartFunction
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
Definition: CodeGenFunction.cpp:826
clang::CapturedStmt::getCapturedStmt
Stmt * getCapturedStmt()
Retrieve the statement being captured.
Definition: Stmt.h:3206
clang::Expr::isLValue
bool isLValue() const
isLValue - True if this expression is an "l-value" according to the rules of the current language...
Definition: Expr.h:249
clang::OMPTargetDirective
This represents &#39;#pragma omp target&#39; directive.
Definition: StmtOpenMP.h:2288
clang::CodeGen::CodeGenFunction::EmitOMPForSimdDirective
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
Definition: CGStmtOpenMP.cpp:2489
clang::OMPAtomicDirective::getV
Expr * getV()
Get &#39;v&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2265
clang::Dtor_Complete
Complete object dtor.
Definition: ABI.h:36
clang::XRayInstrKind::None
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
clang::CodeGen::CodeGenFunction::EmitOMPAtomicDirective
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
Definition: CGStmtOpenMP.cpp:3980
clang::CodeGen::CodeGenFunction::EmitOMPSectionDirective
void EmitOMPSectionDirective(const OMPSectionDirective &S)
Definition: CGStmtOpenMP.cpp:2671
clang::CodeGen::CodeGenFunction::EmitOMPSectionsDirective
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
Definition: CGStmtOpenMP.cpp:2659
clang::OMPOrderedDirective
This represents &#39;#pragma omp ordered&#39; directive.
Definition: StmtOpenMP.h:2099
clang::CodeGen::LValue::getType
QualType getType() const
Definition: CGValue.h:264
clang::OMPTargetUpdateDirective
This represents &#39;#pragma omp target update&#39; directive.
Definition: StmtOpenMP.h:3040
clang::CodeGen::CodeGenFunction::EmitOMPTaskLoopDirective
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
Definition: CGStmtOpenMP.cpp:5000
clang::MultiVersionKind::None
clang::Expr::getExprLoc
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:215
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
Definition: CGStmtOpenMP.cpp:4233
clang::ImplicitParamDecl
Definition: Decl.h:1486
clang::CodeGen::CodeGenFunction::getOMPCancelDestination
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
Definition: CGStmtOpenMP.cpp:4515
clang::CodeGen::CodeGenFunction::EmitScalarExpr
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
Definition: CGExprScalar.cpp:4290
clang::CodeGen::FunctionArgList
FunctionArgList - Type for representing both the decl and type of parameters to a function...
Definition: CGCall.h:356
clang::CodeGen::RValue::getScalarVal
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:59
CGOpenMPRuntime.h
clang::CodeGen::RegionCodeGenTy::setAction
void setAction(PrePostActionTy &Action) const
Definition: CGOpenMPRuntime.h:88
clang::CodeGen::CGFunctionInfo
CGFunctionInfo - Class to encapsulate the information about a function definition.
Definition: CGFunctionInfo.h:493
clang::CodeGen::CodeGenModule
This class organizes the cross-function state that is used while generating LLVM code.
Definition: CodeGenModule.h:274
clang::CodeGen::CodeGenModule::getOpenMPRuntime
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
Definition: CodeGenModule.h:571
clang::OMPC_SCHEDULE_unknown
Definition: OpenMPKinds.h:63
clang::OpenMPScheduleClauseModifier
OpenMPScheduleClauseModifier
OpenMP modifiers for &#39;schedule&#39; clause.
Definition: OpenMPKinds.h:67
clang::CodeGen::CodeGenFunction::EmitOMPSharedLValue
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
Definition: CGStmtOpenMP.cpp:217
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDirective
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
Definition: CGStmtOpenMP.cpp:4201
clang::ParmVarDecl::Create
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
Definition: Decl.cpp:2535
clang
Dataflow Directional Tag Classes.
Definition: CFGReachabilityAnalysis.h:22
clang::DeclarationNameInfo::getAsString
std::string getAsString() const
getAsString - Retrieve the human-readable string for this name.
Definition: DeclarationName.cpp:454
clang::OMPLoopDirective::getPrevUpperBoundVariable
Expr * getPrevUpperBoundVariable() const
Definition: StmtOpenMP.h:887
clang::CodeGen::RegionCodeGenTy
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
Definition: CGOpenMPRuntime.h:65
OMPDeviceClause
This represents &#39;device&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:3431
emitEmptyBoundParameters
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value *> &)
Definition: CGStmtOpenMP.cpp:1267
clang::CodeGen::CodeGenFunction::EmitOMPTaskyieldDirective
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
Definition: CGStmtOpenMP.cpp:3212
clang::CodeGen::RValue::getComplex
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:93
clang::Expr::EvalResult
EvalResult is a struct with detailed info about an evaluated expression.
Definition: Expr.h:571
clang::CodeGen::CodeGenFunction::LexicalScope
Definition: CodeGenFunction.h:759
clang::prec::And
Definition: OperatorPrecedence.h:36
clang::CodeGen::CodeGenFunction::EmitOMPTargetParallelDirective
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
Definition: CGStmtOpenMP.cpp:4763
clang::CodeGen::AggValueSlot::ignored
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
Definition: CGValue.h:499
clang::CodeGen::CodeGenFunction::EmitOMPParallelForDirective
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
Definition: CGStmtOpenMP.cpp:2744
clang::tooling::fixit::internal::getSourceRange
SourceRange getSourceRange(const SourceRange &Range)
Returns the SourceRange of a SourceRange.
Definition: FixIt.h:34
clang::OMPLoopDirective::getCond
Expr * getCond() const
Definition: StmtOpenMP.h:801
clang::CodeGen::OMPTaskDataTy::Nogroup
bool Nogroup
Definition: CGOpenMPRuntime.h:110
clang::OMPSectionDirective
This represents &#39;#pragma omp section&#39; directive.
Definition: StmtOpenMP.h:1314
clang::LangOptions::OMPTargetTriples
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Definition: LangOptions.h:243
clang::OpenMPScheduleTy::M1
OpenMPScheduleClauseModifier M1
Definition: OpenMPKinds.h:143
clang::OMPTeamsDistributeDirective
This represents &#39;#pragma omp teams distribute&#39; directive.
Definition: StmtOpenMP.h:3450
clang::CodeGen::PrePostActionTy
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
Definition: CGOpenMPRuntime.h:55
emitOMPLoopBodyWithStopPoint
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
Definition: CGStmtOpenMP.cpp:1664
clang::CodeGen::CodeGenFunction::EmitOMPPrivateClause
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Definition: CGStmtOpenMP.cpp:830
clang::CodeGen::ReductionCodeGen::getSharedLValue
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
Definition: CGOpenMPRuntime.h:187
clang::CodeGen::CodeGenFunction::EmitOMPTargetParallelForDirective
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
Definition: CGStmtOpenMP.cpp:4803
clang::VarDecl::getInit
const Expr * getInit() const
Definition: Decl.h:1220
clang::OMPSimdDirective
This represents &#39;#pragma omp simd&#39; directive.
Definition: StmtOpenMP.h:1039
clang::CodeGen::CGBuilderTy::CreateLoad
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:70
emitCommonOMPTeamsDirective
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Definition: CGStmtOpenMP.cpp:4122
clang::DeclarationName
The name of a declaration.
Definition: DeclarationName.h:146
clang::isOpenMPSimdDirective
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
Definition: OpenMPKinds.cpp:865
clang::OMPTaskgroupDirective::getReductionRef
const Expr * getReductionRef() const
Returns reference to the task_reduction return variable.
Definition: StmtOpenMP.h:2022
clang::CodeGen::OMPTaskDataTy::Priority
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
Definition: CGOpenMPRuntime.h:106
convertToType
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3806
clang::CodeGen::CodeGenFunction::EmitOMPLinearClause
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
Definition: CGStmtOpenMP.cpp:1549
clang::CodeGen::CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
Definition: CGStmtOpenMP.cpp:4346
clang::CapturedStmt::getEndLoc
SourceLocation getEndLoc() const LLVM_READONLY
Definition: Stmt.h:3301
clang::OMPLinearClause
This represents clause &#39;linear&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:2606
clang::CodeGen::CodeGenFunction::EmitOMPFirstprivateClause
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Definition: CGStmtOpenMP.cpp:726
clang::CodeGen::CodeGenFunction::EmitOMPCancelDirective
void EmitOMPCancelDirective(const OMPCancelDirective &S)
Definition: CGStmtOpenMP.cpp:4501
clang::OMPAtomicDirective::getUpdateExpr
Expr * getUpdateExpr()
Get helper expression of the form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; or &#39;OpaqueValueExp...
Definition: StmtOpenMP.h:2251
clang::CapturedStmt::getBeginLoc
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Stmt.h:3297
clang::CodeGen::CodeGenModule::getModule
llvm::Module & getModule() const
Definition: CodeGenModule.h:659
clang::OMPAtomicDirective
This represents &#39;#pragma omp atomic&#39; directive.
Definition: StmtOpenMP.h:2154
clang::CodeGen::CodeGenFunction::MakeAddrLValue
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
Definition: CodeGenFunction.h:2099
clang::OMPLoopDirective::getCombinedInit
Expr * getCombinedInit() const
Definition: StmtOpenMP.h:923
clang::OMPLoopDirective::getCollapsedNumber
unsigned getCollapsedNumber() const
Get number of collapsed loops.
Definition: StmtOpenMP.h:783
clang::OMPLoopDirective::getCombinedNextLowerBound
Expr * getCombinedNextLowerBound() const
Definition: StmtOpenMP.h:935
clang::OMPLoopDirective::counters
ArrayRef< Expr * > counters()
Definition: StmtOpenMP.h:971
emitTargetTeamsDistributeSimdRegion
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
Definition: CGStmtOpenMP.cpp:4255
clang::CodeGen::ReductionCodeGen::emitInitialization
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
Definition: CGOpenMPRuntime.cpp:1012
clang::CodeGen::CodeGenFunction::EmitOMPReductionClauseFinal
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
Definition: CGStmtOpenMP.cpp:1161
clang::SwitchStmt
SwitchStmt - This represents a &#39;switch&#39; stmt.
Definition: Stmt.h:1886
clang::OMPLoopDirective::getCombinedNextUpperBound
Expr * getCombinedNextUpperBound() const
Definition: StmtOpenMP.h:941
clang::CodeGen::CodeGenFunction::EmitAutoVarCleanups
void EmitAutoVarCleanups(const AutoVarEmission &emission)
Definition: CGDecl.cpp:1873
clang::CodeGen::CodeGenFunction::CGCapturedStmtInfo
API for captured statement code generation.
Definition: CodeGenFunction.h:343
clang::getOpenMPCaptureRegions
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
Definition: OpenMPKinds.cpp:919
clang::ComplexType
Complex values, per C99 6.2.5p11.
Definition: Type.h:2477
clang::isOpenMPDistributeDirective
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
Definition: OpenMPKinds.cpp:884
emitSimpleAtomicStore
static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, LValue LVal, RValue RVal)
Definition: CGStmtOpenMP.cpp:3582
StmtOpenMP.h
This file defines OpenMP AST classes for executable directives and clauses.
clang::OMPLoopDirective::inits
ArrayRef< Expr * > inits()
Definition: StmtOpenMP.h:983
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDeviceFunction
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
Definition: CGStmtOpenMP.cpp:4187
clang::CodeGen::Address::getElementType
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:52
clang::VarDecl::isStaticLocal
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
Definition: Decl.h:1060
clang::OpenMPScheduleClauseKind
OpenMPScheduleClauseKind
OpenMP attributes for &#39;schedule&#39; clause.
Definition: OpenMPKinds.h:59
clang::OMPLoopDirective::getNumIterations
Expr * getNumIterations() const
Definition: StmtOpenMP.h:873
clang::ReferenceType
Base for LValueReferenceType and RValueReferenceType.
Definition: Type.h:2673
clang::CodeGen::CodeGenModule::SetInternalFunctionAttributes
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
Definition: CodeGenModule.cpp:1482
clang::CodeGen::CodeGenModule::getMangledName
StringRef getMangledName(GlobalDecl GD)
Definition: CodeGenModule.cpp:1030
clang::CodeGen::CodeGenFunction::OMPTargetDataInfo::BasePointersArray
Address BasePointersArray
Definition: CodeGenFunction.h:3095
clang::InternalLinkage
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
Definition: Linkage.h:32
clang::CodeGen::CodeGenFunction::OMPMapVars::setVarAddr
bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD, Address TempAddr)
Sets the address of the variable LocalVD to be TempAddr in function CGF.
Definition: CodeGenFunction.h:830
clang::CodeGen::CodeGenFunction::EmitBlock
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition: CGStmt.cpp:443
clang::CodeGen::CodeGenFunction::EmitOMPTargetEnterDataDirective
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
Definition: CGStmtOpenMP.cpp:4682
clang::OMPExecutableDirective::hasAssociatedStmt
bool hasAssociatedStmt() const
Returns true if directive has associated statement.
Definition: StmtOpenMP.h:193
clang::OMPLoopDirective::getPrevLowerBoundVariable
Expr * getPrevLowerBoundVariable() const
Definition: StmtOpenMP.h:881
clang::CodeGen::CodeGenFunction::AutoVarEmission
Definition: CodeGenFunction.h:2670
clang::ASTContext::getTypeSize
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
Definition: ASTContext.h:2070
clang::CodeGen::CodeGenFunction::EmitOMPDistributeSimdDirective
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
Definition: CGStmtOpenMP.cpp:2178
clang::CodeGen::CodeGenFunction::CurFuncDecl
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
Definition: CodeGenFunction.h:296
clang::CodeGen::CodeGenFunction::EmitStopPoint
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
Definition: CGStmt.cpp:36
clang::ASTContext::getCanonicalType
CanQualType getCanonicalType(QualType T) const
Return the canonical (structural) type corresponding to the specified potentially non-canonical type ...
Definition: ASTContext.h:2253
clang::Type::isLValueReferenceType
bool isLValueReferenceType() const
Definition: Type.h:6312
X
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
Definition: SemaDecl.cpp:13954
clang::ASTContext::toCharUnitsFromBits
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
Definition: ASTContext.cpp:2156
clang::CapturedStmt::getCapturedDecl
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
Definition: Stmt.cpp:1265
clang::ASTContext::toBits
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
Definition: ASTContext.cpp:2161
clang::ASTContext::getTranslationUnitDecl
TranslationUnitDecl * getTranslationUnitDecl() const
Definition: ASTContext.h:1009
clang::CodeGen::CodeGenFunction::EmitOMPReductionClauseInit
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for reduction variables.
Definition: CGStmtOpenMP.cpp:1057
clang::OMPTargetParallelDirective
This represents &#39;#pragma omp target parallel&#39; directive.
Definition: StmtOpenMP.h:2521
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeDirective
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
Definition: CGStmtOpenMP.cpp:4247
getCanonicalParamType
static QualType getCanonicalParamType(ASTContext &C, QualType T)
Definition: CGStmtOpenMP.cpp:325
clang::OMPNowaitClause
This represents &#39;nowait&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:1213
clang::CodeGen::OMPTaskDataTy::Schedule
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
Definition: CGOpenMPRuntime.h:105
clang::Type::hasSignedIntegerRepresentation
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g., it is an signed integer type or a vector.
Definition: Type.cpp:1874
clang::CodeGen::CodeGenFunction::EmitBranch
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
Definition: CGStmt.cpp:463
OMPNumTasksClause
This represents &#39;num_tasks&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:4543
clang::CodeGen::OMPTaskDataTy::NumberOfParts
unsigned NumberOfParts
Definition: CGOpenMPRuntime.h:108
Privates
Privates[]
Gets the list of initial values for linear variables.
Definition: OpenMPClause.h:151
clang::QualType::getQualifiers
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition: Type.h:6099
clang::OMPLoopDirective::getCombinedCond
Expr * getCombinedCond() const
Definition: StmtOpenMP.h:929
clang::CodeGen::CodeGenFunction::EmitOMPLinearClauseFinal
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
Definition: CGStmtOpenMP.cpp:1410
clang::CodeGen::CodeGenFunction::EmitOMPUseDevicePtrClause
void EmitOMPUseDevicePtrClause(const OMPClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, Address > &CaptureDeviceAddrMap)
Definition: CGStmtOpenMP.cpp:4528
clang::CodeGen::CodeGenFunction::EmitLValue
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
Definition: CGExpr.cpp:1236
emitTargetTeamsDistributeParallelForSimdRegion
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:4443
clang::ASTContext::getPointerType
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
Definition: ASTContext.cpp:2847
clang::OMPC_unknown
Definition: OpenMPKinds.h:39
clang::CodeGen::CallArg
Definition: CGCall.h:215
clang::Stmt::getSourceRange
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Definition: Stmt.cpp:276
clang::CodeGen::CodeGenFunction::EmitLoadOfLValue
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition: CGExpr.cpp:1760
clang::OMPTaskLoopSimdDirective
This represents &#39;#pragma omp taskloop simd&#39; directive.
Definition: StmtOpenMP.h:2906
clang::CodeGen::CodeGenFunction::EmitOMPCopyinClause
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
Definition: CGStmtOpenMP.cpp:856
clang::OMPLoopDirective::getPreCond
Expr * getPreCond() const
Definition: StmtOpenMP.h:797
OMPDistScheduleClause
This represents &#39;dist_schedule&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:4644
clang::CodeGen::CodeGenFunction::CapturedStmtInfo
CGCapturedStmtInfo * CapturedStmtInfo
Definition: CodeGenFunction.h:408
clang::CodeGen::LValue::isGlobalReg
bool isGlobalReg() const
Definition: CGValue.h:256
clang::CodeGen::CodeGenFunction::EmitScalarConversion
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
Definition: CGExprScalar.cpp:4300
clang::ASTContext::getAsVariableArrayType
const VariableArrayType * getAsVariableArrayType(QualType T) const
Definition: ASTContext.h:2416
clang::CodeGen::CodeGenFunction::TaskGenTy
const llvm::function_ref< void(CodeGenFunction &, llvm::Value *, const OMPTaskDataTy &)> TaskGenTy
Definition: CodeGenFunction.h:3089
clang::OMPSectionsDirective
This represents &#39;#pragma omp sections&#39; directive.
Definition: StmtOpenMP.h:1246
clang::OMPTargetDataDirective
This represents &#39;#pragma omp target data&#39; directive.
Definition: StmtOpenMP.h:2346
clang::CodeGen::CodeGenFunction::EmitOMPTargetExitDataDirective
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
Definition: CGStmtOpenMP.cpp:4703
clang::OMPCancelDirective::getCancelRegion
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:2825
clang::CapturedStmt::captures
capture_range captures()
Definition: Stmt.h:3240
clang::DeclRefExpr
A reference to a declared variable, function, enum, etc.
Definition: Expr.h:1041
clang::CodeGen::RValue::get
static RValue get(llvm::Value *V)
Definition: CGValue.h:86
clang::Type::isPointerType
bool isPointerType() const
Definition: Type.h:6296
AttributeLangSupport::C
Definition: SemaDeclAttr.cpp:45
clang::CodeGen::CodeGenFunction::EmitOMPTargetDeviceFunction
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
Definition: CGStmtOpenMP.cpp:4101
clang::CodeGen::CodeGenFunction::EmitOMPTargetDataDirective
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
Definition: CGStmtOpenMP.cpp:4593
clang::CodeGen::TEK_Scalar
Definition: CodeGenFunction.h:106
clang::ASTContext::getTypeSizeInChars
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
Definition: ASTContext.cpp:2167
clang::CodeGen::CodeGenFunction::EmitOMPTargetParallelForDeviceFunction
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
Definition: CGStmtOpenMP.cpp:4788
clang::CodeGen::CodeGenFunction::EmitOMPDistributeParallelForDirective
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
Definition: CGStmtOpenMP.cpp:2158
clang::CodeGen::CodeGenFunction::EmitAlignmentAssumption
void EmitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
Definition: CodeGenFunction.cpp:2210
clang::ValueDecl::getType
QualType getType() const
Definition: Decl.h:648
clang::VK_LValue
An l-value expression is a reference to an object with independent storage.
Definition: Specifiers.h:114
clang::CodeGen::CodeGenFunction::OMPTargetDataInfo::SizesArray
Address SizesArray
Definition: CodeGenFunction.h:3097
clang::SourceRange
A trivial tuple used to represent a source range.
Definition: SourceLocation.h:198
clang::CodeGen::LValue
LValue - This represents an lvalue references.
Definition: CGValue.h:167
clang::OMPTaskyieldDirective
This represents &#39;#pragma omp taskyield&#39; directive.
Definition: StmtOpenMP.h:1839
clang::CodeGen::CodeGenFunction::CGM
CodeGenModule & CGM
Definition: CodeGenFunction.h:258
clang::OMPDistributeParallelForSimdDirective
This represents &#39;#pragma omp distribute parallel for simd&#39; composite directive.
Definition: StmtOpenMP.h:3179
clang::ASTContext::BoolTy
CanQualType BoolTy
Definition: ASTContext.h:1017
clang::VariableArrayType
Represents a C array with a specified size that is not an integer-constant-expression.
Definition: Type.h:2971
clang::OMPParallelSectionsDirective
This represents &#39;#pragma omp parallel sections&#39; directive.
Definition: StmtOpenMP.h:1700
clang::CodeGen::CodeGenFunction::OMPMapVars::apply
bool apply(CodeGenFunction &CGF)
Applies new addresses to the list of the variables.
Definition: CodeGenFunction.h:858
clang::SourceRange::getBegin
SourceLocation getBegin() const
Definition: SourceLocation.h:207
clang::CapturedStmt::capture_init_end
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
Definition: Stmt.h:3289
clang::CodeGen::LValue::getPointer
llvm::Value * getPointer() const
Definition: CGValue.h:323
clang::CodeGen::CodeGenFunction::EmitOMPTaskDirective
void EmitOMPTaskDirective(const OMPTaskDirective &S)
Definition: CGStmtOpenMP.cpp:3182
clang::Decl::getBodyRBrace
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
Definition: DeclBase.cpp:906
clang::CodeGen::RValue::isScalar
bool isScalar() const
Definition: CGValue.h:52
clang::OMPTargetParallelForDirective
This represents &#39;#pragma omp target parallel for&#39; directive.
Definition: StmtOpenMP.h:2581
clang::CodeGen::OMPTaskDataTy::FirstprivateInits
SmallVector< const Expr *, 4 > FirstprivateInits
Definition: CGOpenMPRuntime.h:97
clang::Decl::getLocation
SourceLocation getLocation() const
Definition: DeclBase.h:418
OMPUseDevicePtrClause
This represents clause &#39;use_device_ptr&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:5049
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
Definition: CGStmtOpenMP.cpp:4279
clang::ASTContext::getIntTypeForBitwidth
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
Definition: ASTContext.cpp:10021
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
Definition: CGStmtOpenMP.cpp:4435
clang::Create
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
clang::CodeGen::CodeGenFunction::OMPPrivateScope::Privatize
bool Privatize()
Privatizes local variables previously registered as private.
Definition: CodeGenFunction.h:922
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
Definition: CGStmtOpenMP.cpp:4471
clang::CodeGen::CodeGenTypes::GetFunctionType
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1550
clang::OMPTaskLoopDirective
This represents &#39;#pragma omp taskloop&#39; directive.
Definition: StmtOpenMP.h:2841