clang 10.0.0git

CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit OpenMP nodes as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "CGCleanup.h"
14 #include "CGOpenMPRuntime.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/AST/Attr.h"
20 #include "clang/AST/DeclOpenMP.h"
21 #include "clang/AST/Stmt.h"
22 #include "clang/AST/StmtOpenMP.h"
23 #include "clang/Basic/PrettyStackTrace.h"
24 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
25 using namespace clang;
26 using namespace CodeGen;
27 using namespace llvm::omp;
28
29 namespace {
30 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
31 /// for captured expressions.
32 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
33 void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
34 for (const auto *C : S.clauses()) {
35 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
36 if (const auto *PreInit =
37 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
38 for (const auto *I : PreInit->decls()) {
39 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
40 CGF.EmitVarDecl(cast<VarDecl>(*I));
41 } else {
42 CodeGenFunction::AutoVarEmission Emission =
43 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
44 CGF.EmitAutoVarCleanups(Emission);
45 }
46 }
47 }
48 }
49 }
50 }
51 CodeGenFunction::OMPPrivateScope InlinedShareds;
52
53 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
54 return CGF.LambdaCaptureFields.lookup(VD) ||
55 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
56 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
57 }
58
59 public:
60 OMPLexicalScope(
61 CodeGenFunction &CGF, const OMPExecutableDirective &S,
62 const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
63 const bool EmitPreInitStmt = true)
64 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
65 InlinedShareds(CGF) {
66 if (EmitPreInitStmt)
67 emitPreInitStmt(CGF, S);
68 if (!CapturedRegion.hasValue())
69 return;
70 assert(S.hasAssociatedStmt() &&
71 "Expected associated statement for inlined directive.");
72 const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
73 for (const auto &C : CS->captures()) {
74 if (C.capturesVariable() || C.capturesVariableByCopy()) {
75 auto *VD = C.getCapturedVar();
76 assert(VD == VD->getCanonicalDecl() &&
77 "Canonical decl must be captured.");
78 DeclRefExpr DRE(
79 CGF.getContext(), const_cast<VarDecl *>(VD),
80 isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
81 InlinedShareds.isGlobalVarCaptured(VD)),
82 VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
83 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
84 return CGF.EmitLValue(&DRE).getAddress(CGF);
85 });
86 }
87 }
88 (void)InlinedShareds.Privatize();
89 }
90 };
91
92 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
93 /// for captured expressions.
94 class OMPParallelScope final : public OMPLexicalScope {
95 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
96 OpenMPDirectiveKind Kind = S.getDirectiveKind();
97 return !(isOpenMPTargetExecutionDirective(Kind) ||
98 isOpenMPLoopBoundSharingDirective(Kind)) &&
99 isOpenMPParallelDirective(Kind);
100 }
101
102 public:
103 OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
104 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
105 EmitPreInitStmt(S)) {}
106 };
107
108 /// Lexical scope for OpenMP teams construct, that handles correct codegen
109 /// for captured expressions.
110 class OMPTeamsScope final : public OMPLexicalScope {
111 bool EmitPreInitStmt(const OMPExecutableDirective &S) {
112 OpenMPDirectiveKind Kind = S.getDirectiveKind();
113 return !isOpenMPTargetExecutionDirective(Kind) &&
114 isOpenMPTeamsDirective(Kind);
115 }
116
117 public:
118 OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
119 : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
120 EmitPreInitStmt(S)) {}
121 };
122
123 /// Private scope for OpenMP loop-based directives, that supports capturing
124 /// of used expression from loop statement.
125 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
126 void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
127 CodeGenFunction::OMPMapVars PreCondVars;
128 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
129 for (const auto *E : S.counters()) {
130 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
131 EmittedAsPrivate.insert(VD->getCanonicalDecl());
132 (void)PreCondVars.setVarAddr(
133 CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
134 }
135 // Mark private vars as undefs.
136 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
137 for (const Expr *IRef : C->varlists()) {
138 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
139 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
140 (void)PreCondVars.setVarAddr(
141 CGF, OrigVD,
142 Address(llvm::UndefValue::get(
143 CGF.ConvertTypeForMem(CGF.getContext().getPointerType(
144 OrigVD->getType().getNonReferenceType()))),
145 CGF.getContext().getDeclAlign(OrigVD)));
146 }
147 }
148 }
149 (void)PreCondVars.apply(CGF);
150 // Emit init, __range and __end variables for C++ range loops.
151 const Stmt *Body =
152 S.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
153 for (unsigned Cnt = 0; Cnt < S.getCollapsedNumber(); ++Cnt) {
154 Body = OMPLoopDirective::tryToFindNextInnerLoop(
155 Body, /*TryImperfectlyNestedLoops=*/true);
156 if (auto *For = dyn_cast<ForStmt>(Body)) {
157 Body = For->getBody();
158 } else {
159 assert(isa<CXXForRangeStmt>(Body) &&
160 "Expected canonical for loop or range-based for loop.");
161 auto *CXXFor = cast<CXXForRangeStmt>(Body);
162 if (const Stmt *Init = CXXFor->getInit())
163 CGF.EmitStmt(Init);
164 CGF.EmitStmt(CXXFor->getRangeStmt());
165 CGF.EmitStmt(CXXFor->getEndStmt());
166 Body = CXXFor->getBody();
167 }
168 }
169 if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
170 for (const auto *I : PreInits->decls())
171 CGF.EmitVarDecl(cast<VarDecl>(*I));
172 }
173 PreCondVars.restore(CGF);
174 }
175
176 public:
177 OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
178 : CodeGenFunction::RunCleanupsScope(CGF) {
179 emitPreInitStmt(CGF, S);
180 }
181 };
182
183 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
184 CodeGenFunction::OMPPrivateScope InlinedShareds;
185
186 static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
187 return CGF.LambdaCaptureFields.lookup(VD) ||
188 (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
189 (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
190 cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
191 }
192
193 public:
194 OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
195 : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
196 InlinedShareds(CGF) {
197 for (const auto *C : S.clauses()) {
198 if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
199 if (const auto *PreInit =
200 cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
201 for (const auto *I : PreInit->decls()) {
202 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
203 CGF.EmitVarDecl(cast<VarDecl>(*I));
204 } else {
205 CodeGenFunction::AutoVarEmission Emission =
206 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
207 CGF.EmitAutoVarCleanups(Emission);
208 }
209 }
210 }
211 } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
212 for (const Expr *E : UDP->varlists()) {
213 const Decl *D = cast<DeclRefExpr>(E)->getDecl();
214 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
215 CGF.EmitVarDecl(*OED);
216 }
217 }
218 }
219 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
220 CGF.EmitOMPPrivateClause(S, InlinedShareds);
221 if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
222 if (const Expr *E = TG->getReductionRef())
223 CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
224 }
225 const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
226 while (CS) {
227 for (auto &C : CS->captures()) {
228 if (C.capturesVariable() || C.capturesVariableByCopy()) {
229 auto *VD = C.getCapturedVar();
230 assert(VD == VD->getCanonicalDecl() &&
231 "Canonical decl must be captured.");
232 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
233 isCapturedVar(CGF, VD) ||
234 (CGF.CapturedStmtInfo &&
235 InlinedShareds.isGlobalVarCaptured(VD)),
236 VD->getType().getNonReferenceType(), VK_LValue,
237 C.getLocation());
238 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
239 return CGF.EmitLValue(&DRE).getAddress(CGF);
240 });
241 }
242 }
243 CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
244 }
245 (void)InlinedShareds.Privatize();
246 }
247 };
248
249 } // namespace
250
251 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
252 const OMPExecutableDirective &S,
253 const RegionCodeGenTy &CodeGen);
254
255 LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
256 if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
257 if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
258 OrigVD = OrigVD->getCanonicalDecl();
259 bool IsCaptured =
260 LambdaCaptureFields.lookup(OrigVD) ||
261 (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
262 (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
263 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
264 OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
265 return EmitLValue(&DRE);
266 }
267 }
268 return EmitLValue(E);
269 }
270
271 llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
272 ASTContext &C = getContext();
273 llvm::Value *Size = nullptr;
274 auto SizeInChars = C.getTypeSizeInChars(Ty);
275 if (SizeInChars.isZero()) {
276 // getTypeSizeInChars() returns 0 for a VLA.
277 while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
278 VlaSizePair VlaSize = getVLASize(VAT);
279 Ty = VlaSize.Type;
280 Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
281 : VlaSize.NumElts;
282 }
283 SizeInChars = C.getTypeSizeInChars(Ty);
284 if (SizeInChars.isZero())
285 return llvm::ConstantInt::get(SizeTy, /*V=*/0);
286 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
287 }
288 return CGM.getSize(SizeInChars);
289 }
290
291 void CodeGenFunction::GenerateOpenMPCapturedVars(
292 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
293 const RecordDecl *RD = S.getCapturedRecordDecl();
294 auto CurField = RD->field_begin();
295 auto CurCap = S.captures().begin();
296 for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
297 E = S.capture_init_end();
298 I != E; ++I, ++CurField, ++CurCap) {
299 if (CurField->hasCapturedVLAType()) {
300 const VariableArrayType *VAT = CurField->getCapturedVLAType();
301 llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
302 CapturedVars.push_back(Val);
303 } else if (CurCap->capturesThis()) {
304 CapturedVars.push_back(CXXThisValue);
305 } else if (CurCap->capturesVariableByCopy()) {
306 llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
307
308 // If the field is not a pointer, we need to save the actual value
309 // and load it as a void pointer.
310 if (!CurField->getType()->isAnyPointerType()) {
311 ASTContext &Ctx = getContext();
312 Address DstAddr = CreateMemTemp(
313 Ctx.getUIntPtrType(),
314 Twine(CurCap->getCapturedVar()->getName(), ".casted"));
315 LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
316
317 llvm::Value *SrcAddrVal = EmitScalarConversion(
318 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
319 Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
320 LValue SrcLV =
321 MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
322
323 // Store the value using the source type pointer.
324 EmitStoreThroughLValue(RValue::get(CV), SrcLV);
325
326 // Load the value using the destination type pointer.
327 CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
328 }
329 CapturedVars.push_back(CV);
330 } else {
331 assert(CurCap->capturesVariable() && "Expected capture by reference.");
332 CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer());
333 }
334 }
335 }
336
337 static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
338 QualType DstType, StringRef Name,
339 LValue AddrLV) {
340 ASTContext &Ctx = CGF.getContext();
341
342 llvm::Value *CastedPtr = CGF.EmitScalarConversion(
343 AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(),
344 Ctx.getPointerType(DstType), Loc);
345 Address TmpAddr =
346 CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
347 .getAddress(CGF);
348 return TmpAddr;
349 }
350
351 static QualType getCanonicalParamType(ASTContext &C, QualType T) {
352 if (T->isLValueReferenceType())
353 return C.getLValueReferenceType(
354 getCanonicalParamType(C, T.getNonReferenceType()),
355 /*SpelledAsLValue=*/false);
356 if (T->isPointerType())
357 return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
358 if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
359 if (const auto *VLA = dyn_cast<VariableArrayType>(A))
360 return getCanonicalParamType(C, VLA->getElementType());
361 if (!A->isVariablyModifiedType())
362 return C.getCanonicalType(T);
363 }
364 return C.getCanonicalParamType(T);
365 }
366
367 namespace {
368 /// Contains required data for proper outlined function codegen.
369 struct FunctionOptions {
370 /// Captured statement for which the function is generated.
371 const CapturedStmt *S = nullptr;
372 /// true if cast to/from UIntPtr is required for variables captured by
373 /// value.
374 const bool UIntPtrCastRequired = true;
375 /// true if only casted arguments must be registered as local args or VLA
376 /// sizes.
377 const bool RegisterCastedArgsOnly = false;
378 /// Name of the generated function.
379 const StringRef FunctionName;
380 explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
381 bool RegisterCastedArgsOnly,
382 StringRef FunctionName)
383 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
384 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
385 FunctionName(FunctionName) {}
386 };
387 }
388
389 static llvm::Function *emitOutlinedFunctionPrologue(
390 CodeGenFunction &CGF, FunctionArgList &Args,
391 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
392 &LocalAddrs,
393 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
394 &VLASizes,
395 llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
396 const CapturedDecl *CD = FO.S->getCapturedDecl();
397 const RecordDecl *RD = FO.S->getCapturedRecordDecl();
398 assert(CD->hasBody() && "missing CapturedDecl body");
399
400 CXXThisValue = nullptr;
401 // Build the argument list.
402 CodeGenModule &CGM = CGF.CGM;
403 ASTContext &Ctx = CGM.getContext();
404 FunctionArgList TargetArgs;
405 Args.append(CD->param_begin(),
406 std::next(CD->param_begin(), CD->getContextParamPosition()));
407 TargetArgs.append(
408 CD->param_begin(),
409 std::next(CD->param_begin(), CD->getContextParamPosition()));
410 auto I = FO.S->captures().begin();
411 FunctionDecl *DebugFunctionDecl = nullptr;
412 if (!FO.UIntPtrCastRequired) {
413 FunctionProtoType::ExtProtoInfo EPI;
414 QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI);
415 DebugFunctionDecl = FunctionDecl::Create(
416 Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
417 SourceLocation(), DeclarationName(), FunctionTy,
418 Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
419 /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
420 }
421 for (const FieldDecl *FD : RD->fields()) {
422 QualType ArgType = FD->getType();
423 IdentifierInfo *II = nullptr;
424 VarDecl *CapVar = nullptr;
425
426 // If this is a capture by copy and the type is not a pointer, the outlined
427 // function argument type should be uintptr and the value properly casted to
428 // uintptr. This is necessary given that the runtime library is only able to
429 // deal with pointers. We can pass in the same way the VLA type sizes to the
430 // outlined function.
431 if (FO.UIntPtrCastRequired &&
432 ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
433 I->capturesVariableArrayType()))
434 ArgType = Ctx.getUIntPtrType();
435
436 if (I->capturesVariable() || I->capturesVariableByCopy()) {
437 CapVar = I->getCapturedVar();
438 II = CapVar->getIdentifier();
439 } else if (I->capturesThis()) {
440 II = &Ctx.Idents.get("this");
441 } else {
442 assert(I->capturesVariableArrayType());
443 II = &Ctx.Idents.get("vla");
444 }
445 if (ArgType->isVariablyModifiedType())
446 ArgType = getCanonicalParamType(Ctx, ArgType);
447 VarDecl *Arg;
448 if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
449 Arg = ParmVarDecl::Create(
450 Ctx, DebugFunctionDecl,
451 CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
452 CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
453 /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
454 } else {
455 Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
456 II, ArgType, ImplicitParamDecl::Other);
457 }
458 Args.emplace_back(Arg);
459 // Do not cast arguments if we emit function with non-original types.
460 TargetArgs.emplace_back(
461 FO.UIntPtrCastRequired
462 ? Arg
463 : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
464 ++I;
465 }
466 Args.append(
467 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
468 CD->param_end());
469 TargetArgs.append(
470 std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
471 CD->param_end());
472
473 // Create the function declaration.
474 const CGFunctionInfo &FuncInfo =
475 CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
476 llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
477
478 auto *F =
479 llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
480 FO.FunctionName, &CGM.getModule());
481 CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
482 if (CD->isNothrow())
483 F->setDoesNotThrow();
484 F->setDoesNotRecurse();
485
486 // Generate the function.
487 CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
488 FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
489 unsigned Cnt = CD->getContextParamPosition();
490 I = FO.S->captures().begin();
491 for (const FieldDecl *FD : RD->fields()) {
492 // Do not map arguments if we emit function with non-original types.
493 Address LocalAddr(Address::invalid());
494 if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
495 LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
496 TargetArgs[Cnt]);
497 } else {
498 LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
499 }
500 // If we are capturing a pointer by copy we don't need to do anything, just
501 // use the value that we get from the arguments.
502 if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
503 const VarDecl *CurVD = I->getCapturedVar();
504 if (!FO.RegisterCastedArgsOnly)
505 LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
506 ++Cnt;
507 ++I;
508 continue;
509 }
510
511 LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
512 AlignmentSource::Decl);
513 if (FD->hasCapturedVLAType()) {
514 if (FO.UIntPtrCastRequired) {
515 ArgLVal = CGF.MakeAddrLValue(
516 castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
517 Args[Cnt]->getName(), ArgLVal),
518 FD->getType(), AlignmentSource::Decl);
519 }
520 llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
521 const VariableArrayType *VAT = FD->getCapturedVLAType();
522 VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
523 } else if (I->capturesVariable()) {
524 const VarDecl *Var = I->getCapturedVar();
525 QualType VarTy = Var->getType();
526 Address ArgAddr = ArgLVal.getAddress(CGF);
527 if (ArgLVal.getType()->isLValueReferenceType()) {
528 ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
529 } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
530 assert(ArgLVal.getType()->isPointerType());
531 ArgAddr = CGF.EmitLoadOfPointer(
532 ArgAddr, ArgLVal.getType()->castAs<PointerType>());
533 }
534 if (!FO.RegisterCastedArgsOnly) {
535 LocalAddrs.insert(
536 {Args[Cnt],
537 {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
538 }
539 } else if (I->capturesVariableByCopy()) {
540 assert(!FD->getType()->isAnyPointerType() &&
541 "Not expecting a captured pointer.");
542 const VarDecl *Var = I->getCapturedVar();
543 LocalAddrs.insert({Args[Cnt],
544 {Var, FO.UIntPtrCastRequired
545 ? castValueFromUintptr(
546 CGF, I->getLocation(), FD->getType(),
547 Args[Cnt]->getName(), ArgLVal)
548 : ArgLVal.getAddress(CGF)}});
549 } else {
550 // If 'this' is captured, load it into CXXThisValue.
551 assert(I->capturesThis());
552 CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
553 LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}});
554 }
555 ++Cnt;
556 ++I;
557 }
558
559 return F;
560 }
561
562 llvm::Function *
563 CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) {
564 assert(
565 CapturedStmtInfo &&
566 "CapturedStmtInfo should be set when generating the captured function");
567 const CapturedDecl *CD = S.getCapturedDecl();
568 // Build the argument list.
569 bool NeedWrapperFunction =
570 getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
571 FunctionArgList Args;
572 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
573 llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
574 SmallString<256> Buffer;
575 llvm::raw_svector_ostream Out(Buffer);
576 Out << CapturedStmtInfo->getHelperName();
577 if (NeedWrapperFunction)
578 Out << "_debug__";
579 FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
580 Out.str());
581 llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
582 VLASizes, CXXThisValue, FO);
583 CodeGenFunction::OMPPrivateScope LocalScope(*this);
584 for (const auto &LocalAddrPair : LocalAddrs) {
585 if (LocalAddrPair.second.first) {
586 LocalScope.addPrivate(LocalAddrPair.second.first, [&LocalAddrPair]() {
587 return LocalAddrPair.second.second;
588 });
589 }
590 }
591 (void)LocalScope.Privatize();
592 for (const auto &VLASizePair : VLASizes)
593 VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
594 PGO.assignRegionCounters(GlobalDecl(CD), F);
595 CapturedStmtInfo->EmitBody(*this, CD->getBody());
596 (void)LocalScope.ForceCleanup();
597 FinishFunction(CD->getBodyRBrace());
598 if (!NeedWrapperFunction)
599 return F;
600
601 FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
602 /*RegisterCastedArgsOnly=*/true,
603 CapturedStmtInfo->getHelperName());
604 CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
605 WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
606 Args.clear();
607 LocalAddrs.clear();
608 VLASizes.clear();
609 llvm::Function *WrapperF =
610 emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
611 WrapperCGF.CXXThisValue, WrapperFO);
612 llvm::SmallVector<llvm::Value *, 4> CallArgs;
613 for (const auto *Arg : Args) {
614 llvm::Value *CallArg;
615 auto I = LocalAddrs.find(Arg);
616 if (I != LocalAddrs.end()) {
617 LValue LV = WrapperCGF.MakeAddrLValue(
618 I->second.second,
619 I->second.first ? I->second.first->getType() : Arg->getType(),
620 AlignmentSource::Decl);
621 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
622 } else {
623 auto EI = VLASizes.find(Arg);
624 if (EI != VLASizes.end()) {
625 CallArg = EI->second.second;
626 } else {
627 LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
628 Arg->getType(),
629 AlignmentSource::Decl);
630 CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
631 }
632 }
633 CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
634 }
635 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
636 F, CallArgs);
637 WrapperCGF.FinishFunction();
638 return WrapperF;
639 }
640
641 //===----------------------------------------------------------------------===//
642 // OpenMP Directive Emission
643 //===----------------------------------------------------------------------===//
644 void CodeGenFunction::EmitOMPAggregateAssign(
645 Address DestAddr, Address SrcAddr, QualType OriginalType,
646 const llvm::function_ref<void(Address, Address)> CopyGen) {
647 // Perform element-by-element initialization.
648 QualType ElementTy;
649
650 // Drill down to the base element type on both arrays.
651 const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
652 llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
653 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
654
655 llvm::Value *SrcBegin = SrcAddr.getPointer();
656 llvm::Value *DestBegin = DestAddr.getPointer();
657 // Cast from pointer to array type to pointer to single element.
658 llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
659 // The basic structure here is a while-do loop.
660 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
661 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
662 llvm::Value *IsEmpty =
663 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
664 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
665
666 // Enter the loop body, making that address the current address.
667 llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
668 EmitBlock(BodyBB);
669
670 CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
671
672 llvm::PHINode *SrcElementPHI =
673 Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
674 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
675 Address SrcElementCurrent =
676 Address(SrcElementPHI,
677 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
678
679 llvm::PHINode *DestElementPHI =
680 Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
681 DestElementPHI->addIncoming(DestBegin, EntryBB);
682 Address DestElementCurrent =
683 Address(DestElementPHI,
684 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
685
686 // Emit copy.
687 CopyGen(DestElementCurrent, SrcElementCurrent);
688
689 // Shift the address forward by one element.
690 llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
691 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
692 llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
693 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
694 // Check whether we've reached the end.
695 llvm::Value *Done =
696 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
697 Builder.CreateCondBr(Done, DoneBB, BodyBB);
698 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
699 SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
700
701 // Done.
702 EmitBlock(DoneBB, /*IsFinished=*/true);
703 }
704
705 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
706 Address SrcAddr, const VarDecl *DestVD,
707 const VarDecl *SrcVD, const Expr *Copy) {
708 if (OriginalType->isArrayType()) {
709 const auto *BO = dyn_cast<BinaryOperator>(Copy);
710 if (BO && BO->getOpcode() == BO_Assign) {
711 // Perform simple memcpy for simple copying.
712 LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
713 LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
714 EmitAggregateAssign(Dest, Src, OriginalType);
715 } else {
716 // For arrays with complex element types perform element by element
717 // copying.
718 EmitOMPAggregateAssign(
719 DestAddr, SrcAddr, OriginalType,
720 [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
721 // Working with the single array element, so have to remap
722 // destination and source variables to corresponding array
723 // elements.
724 CodeGenFunction::OMPPrivateScope Remap(*this);
725 Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
726 Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
727 (void)Remap.Privatize();
728 EmitIgnoredExpr(Copy);
729 });
730 }
731 } else {
732 // Remap pseudo source variable to private copy.
733 CodeGenFunction::OMPPrivateScope Remap(*this);
734 Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
735 Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
736 (void)Remap.Privatize();
737 // Emit copying of the whole variable.
738 EmitIgnoredExpr(Copy);
739 }
740 }
741
742 bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
743 OMPPrivateScope &PrivateScope) {
744 if (!HaveInsertPoint())
745 return false;
746 bool DeviceConstTarget =
747 getLangOpts().OpenMPIsDevice &&
748 isOpenMPTargetExecutionDirective(D.getDirectiveKind());
749 bool FirstprivateIsLastprivate = false;
750 llvm::DenseSet<const VarDecl *> Lastprivates;
751 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
752 for (const auto *D : C->varlists())
753 Lastprivates.insert(
754 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
755 }
756 llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
757 llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
758 getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
759 // Force emission of the firstprivate copy if the directive does not emit
760 // outlined function, like omp for, omp simd, omp distribute etc.
761 bool MustEmitFirstprivateCopy =
762 CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
763 for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
764 auto IRef = C->varlist_begin();
765 auto InitsRef = C->inits().begin();
766 for (const Expr *IInit : C->private_copies()) {
767 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
768 bool ThisFirstprivateIsLastprivate =
769 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
770 const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
771 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
772 if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
773 !FD->getType()->isReferenceType() &&
774 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
775 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
776 ++IRef;
777 ++InitsRef;
778 continue;
779 }
780 // Do not emit copy for firstprivate constant variables in target regions,
781 // captured by reference.
782 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
783 FD && FD->getType()->isReferenceType() &&
784 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
785 (void)CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(*this,
786 OrigVD);
787 ++IRef;
788 ++InitsRef;
789 continue;
790 }
791 FirstprivateIsLastprivate =
792 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
793 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
794 const auto *VDInit =
795 cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
796 bool IsRegistered;
797 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
798 /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
799 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
800 LValue OriginalLVal;
801 if (!FD) {
802 // Check if the firstprivate variable is just a constant value.
803 ConstantEmission CE = tryEmitAsConstant(&DRE);
804 if (CE && !CE.isReference()) {
805 // Constant value, no need to create a copy.
806 ++IRef;
807 ++InitsRef;
808 continue;
809 }
810 if (CE && CE.isReference()) {
811 OriginalLVal = CE.getReferenceLValue(*this, &DRE);
812 } else {
813 assert(!CE && "Expected non-constant firstprivate.");
814 OriginalLVal = EmitLValue(&DRE);
815 }
816 } else {
817 OriginalLVal = EmitLValue(&DRE);
818 }
819 QualType Type = VD->getType();
820 if (Type->isArrayType()) {
821 // Emit VarDecl with copy init for arrays.
822 // Get the address of the original variable captured in current
823 // captured region.
824 IsRegistered = PrivateScope.addPrivate(
825 OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
826 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
827 const Expr *Init = VD->getInit();
828 if (!isa<CXXConstructExpr>(Init) ||
829 isTrivialInitializer(Init)) {
830 // Perform simple memcpy.
831 LValue Dest =
832 MakeAddrLValue(Emission.getAllocatedAddress(), Type);
833 EmitAggregateAssign(Dest, OriginalLVal, Type);
834 } else {
835 EmitOMPAggregateAssign(
836 Emission.getAllocatedAddress(),
837 OriginalLVal.getAddress(*this), Type,
838 [this, VDInit, Init](Address DestElement,
839 Address SrcElement) {
840 // Clean up any temporaries needed by the
841 // initialization.
842 RunCleanupsScope InitScope(*this);
843 // Emit initialization for single element.
844 setAddrOfLocalVar(VDInit, SrcElement);
845 EmitAnyExprToMem(Init, DestElement,
846 Init->getType().getQualifiers(),
847 /*IsInitializer*/ false);
848 LocalDeclMap.erase(VDInit);
849 });
850 }
851 EmitAutoVarCleanups(Emission);
852 return Emission.getAllocatedAddress();
853 });
854 } else {
855 Address OriginalAddr = OriginalLVal.getAddress(*this);
856 IsRegistered = PrivateScope.addPrivate(
857 OrigVD, [this, VDInit, OriginalAddr, VD]() {
858 // Emit private VarDecl with copy init.
859 // Remap temp VDInit variable to the address of the original
860 // variable (for proper handling of captured global variables).
861 setAddrOfLocalVar(VDInit, OriginalAddr);
862 EmitDecl(*VD);
863 LocalDeclMap.erase(VDInit);
864 return GetAddrOfLocalVar(VD);
865 });
866 }
867 assert(IsRegistered &&
868 "firstprivate var already registered as private");
869 // Silence the warning about unused variable.
870 (void)IsRegistered;
871 }
872 ++IRef;
873 ++InitsRef;
874 }
875 }
876 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
877 }
878
879 void CodeGenFunction::EmitOMPPrivateClause(
880 const OMPExecutableDirective &D,
881 CodeGenFunction::OMPPrivateScope &PrivateScope) {
882 if (!HaveInsertPoint())
883 return;
884 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
885 for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
886 auto IRef = C->varlist_begin();
887 for (const Expr *IInit : C->private_copies()) {
888 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
889 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
890 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
891 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
892 // Emit private VarDecl with copy init.
893 EmitDecl(*VD);
894 return GetAddrOfLocalVar(VD);
895 });
896 assert(IsRegistered && "private var already registered as private");
897 // Silence the warning about unused variable.
898 (void)IsRegistered;
899 }
900 ++IRef;
901 }
902 }
903 }
904
905 bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
906 if (!HaveInsertPoint())
907 return false;
908 // threadprivate_var1 = master_threadprivate_var1;
909 // operator=(threadprivate_var2, master_threadprivate_var2);
910 // ...
911 // __kmpc_barrier(&loc, global_tid);
912 llvm::DenseSet<const VarDecl *> CopiedVars;
913 llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
914 for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
915 auto IRef = C->varlist_begin();
916 auto ISrcRef = C->source_exprs().begin();
917 auto IDestRef = C->destination_exprs().begin();
918 for (const Expr *AssignOp : C->assignment_ops()) {
919 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
920 QualType Type = VD->getType();
921 if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
922 // Get the address of the master variable. If we are emitting code with
923 // TLS support, the address is passed from the master as field in the
924 // captured declaration.
925 Address MasterAddr = Address::invalid();
926 if (getLangOpts().OpenMPUseTLS &&
927 getContext().getTargetInfo().isTLSSupported()) {
928 assert(CapturedStmtInfo->lookup(VD) &&
929 "Copyin threadprivates should have been captured!");
930 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
931 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
932 MasterAddr = EmitLValue(&DRE).getAddress(*this);
933 LocalDeclMap.erase(VD);
934 } else {
935 MasterAddr =
936 Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
937 : CGM.GetAddrOfGlobal(VD),
938 getContext().getDeclAlign(VD));
939 }
940 // Get the address of the threadprivate variable.
941 Address PrivateAddr = EmitLValue(*IRef).getAddress(*this);
942 if (CopiedVars.size() == 1) {
943 // At first check if current thread is a master thread. If it is, no
944 // need to copy data.
945 CopyBegin = createBasicBlock("copyin.not.master");
946 CopyEnd = createBasicBlock("copyin.not.master.end");
947 Builder.CreateCondBr(
948 Builder.CreateICmpNE(
949 Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
950 Builder.CreatePtrToInt(PrivateAddr.getPointer(),
951 CGM.IntPtrTy)),
952 CopyBegin, CopyEnd);
953 EmitBlock(CopyBegin);
954 }
955 const auto *SrcVD =
956 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
957 const auto *DestVD =
958 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
959 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
960 }
961 ++IRef;
962 ++ISrcRef;
963 ++IDestRef;
964 }
965 }
966 if (CopyEnd) {
967 // Exit out of copying procedure for non-master thread.
968 EmitBlock(CopyEnd, /*IsFinished=*/true);
969 return true;
970 }
971 return false;
972 }
973
974 bool CodeGenFunction::EmitOMPLastprivateClauseInit(
975 const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
976 if (!HaveInsertPoint())
977 return false;
978 bool HasAtLeastOneLastprivate = false;
979 llvm::DenseSet<const VarDecl *> SIMDLCVs;
980 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
981 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
982 for (const Expr *C : LoopDirective->counters()) {
983 SIMDLCVs.insert(
984 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
985 }
986 }
987 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
988 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
989 HasAtLeastOneLastprivate = true;
990 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
991 !getLangOpts().OpenMPSimd)
992 break;
993 auto IRef = C->varlist_begin();
994 auto IDestRef = C->destination_exprs().begin();
995 for (const Expr *IInit : C->private_copies()) {
996 // Keep the address of the original variable for future update at the end
997 // of the loop.
998 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
999 // Taskloops do not require additional initialization, it is done in
1000 // runtime support library.
1001 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
1002 const auto *DestVD =
1003 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1004 PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
1005 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1006 /*RefersToEnclosingVariableOrCapture=*/
1007 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1008 (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
1009 return EmitLValue(&DRE).getAddress(*this);
1010 });
1011 // Check if the variable is also a firstprivate: in this case IInit is
1012 // not generated. Initialization of this variable will happen in codegen
1013 // for 'firstprivate' clause.
1014 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
1015 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
1016 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
1017 // Emit private VarDecl with copy init.
1018 EmitDecl(*VD);
1019 return GetAddrOfLocalVar(VD);
1020 });
1021 assert(IsRegistered &&
1022 "lastprivate var already registered as private");
1023 (void)IsRegistered;
1024 }
1025 }
1026 ++IRef;
1027 ++IDestRef;
1028 }
1029 }
1030 return HasAtLeastOneLastprivate;
1031 }
1032
1033 void CodeGenFunction::EmitOMPLastprivateClauseFinal(
1034 const OMPExecutableDirective &D, bool NoFinals,
1035 llvm::Value *IsLastIterCond) {
1036 if (!HaveInsertPoint())
1037 return;
1038 // Emit following code:
1039 // if (<IsLastIterCond>) {
1040 // orig_var1 = private_orig_var1;
1041 // ...
1042 // orig_varn = private_orig_varn;
1043 // }
1044 llvm::BasicBlock *ThenBB = nullptr;
1045 llvm::BasicBlock *DoneBB = nullptr;
1046 if (IsLastIterCond) {
1047 // Emit implicit barrier if at least one lastprivate conditional is found
1048 // and this is not a simd mode.
1049 if (!getLangOpts().OpenMPSimd &&
1050 llvm::any_of(D.getClausesOfKind<OMPLastprivateClause>(),
1051 [](const OMPLastprivateClause *C) {
1052 return C->getKind() == OMPC_LASTPRIVATE_conditional;
1053 })) {
1054 CGM.getOpenMPRuntime().emitBarrierCall(*this, D.getBeginLoc(),
1055 OMPD_unknown,
1056 /*EmitChecks=*/false,
1057 /*ForceSimpleCall=*/true);
1058 }
1059 ThenBB = createBasicBlock(".omp.lastprivate.then");
1060 DoneBB = createBasicBlock(".omp.lastprivate.done");
1061 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1062 EmitBlock(ThenBB);
1063 }
1064 llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1065 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1066 if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1067 auto IC = LoopDirective->counters().begin();
1068 for (const Expr *F : LoopDirective->finals()) {
1069 const auto *D =
1070 cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1071 if (NoFinals)
1072 AlreadyEmittedVars.insert(D);
1073 else
1074 LoopCountersAndUpdates[D] = F;
1075 ++IC;
1076 }
1077 }
1078 for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1079 auto IRef = C->varlist_begin();
1080 auto ISrcRef = C->source_exprs().begin();
1081 auto IDestRef = C->destination_exprs().begin();
1082 for (const Expr *AssignOp : C->assignment_ops()) {
1083 const auto *PrivateVD =
1084 cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1085 QualType Type = PrivateVD->getType();
1086 const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1087 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1088 // If lastprivate variable is a loop control variable for loop-based
1089 // directive, update its value before copyin back to original
1090 // variable.
1091 if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1092 EmitIgnoredExpr(FinalExpr);
1093 const auto *SrcVD =
1094 cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1095 const auto *DestVD =
1096 cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1097 // Get the address of the private variable.
1098 Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1099 if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1100 PrivateAddr =
1101 Address(Builder.CreateLoad(PrivateAddr),
1102 getNaturalTypeAlignment(RefTy->getPointeeType()));
1103 // Store the last value to the private copy in the last iteration.
1104 if (C->getKind() == OMPC_LASTPRIVATE_conditional)
1105 CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
1106 *this, MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
1107 (*IRef)->getExprLoc());
1108 // Get the address of the original variable.
1109 Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1110 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1111 }
1112 ++IRef;
1113 ++ISrcRef;
1114 ++IDestRef;
1115 }
1116 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1117 EmitIgnoredExpr(PostUpdate);
1118 }
1119 if (IsLastIterCond)
1120 EmitBlock(DoneBB, /*IsFinished=*/true);
1121 }
1122
1123 void CodeGenFunction::EmitOMPReductionClauseInit(
1124 const OMPExecutableDirective &D,
1125 CodeGenFunction::OMPPrivateScope &PrivateScope) {
1126 if (!HaveInsertPoint())
1127 return;
1128 SmallVector<const Expr *, 4> Shareds;
1129 SmallVector<const Expr *, 4> Privates;
1130 SmallVector<const Expr *, 4> ReductionOps;
1131 SmallVector<const Expr *, 4> LHSs;
1132 SmallVector<const Expr *, 4> RHSs;
1133 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1134 auto IPriv = C->privates().begin();
1135 auto IRed = C->reduction_ops().begin();
1136 auto ILHS = C->lhs_exprs().begin();
1137 auto IRHS = C->rhs_exprs().begin();
1138 for (const Expr *Ref : C->varlists()) {
1139 Shareds.emplace_back(Ref);
1140 Privates.emplace_back(*IPriv);
1141 ReductionOps.emplace_back(*IRed);
1142 LHSs.emplace_back(*ILHS);
1143 RHSs.emplace_back(*IRHS);
1144 std::advance(IPriv, 1);
1145 std::advance(IRed, 1);
1146 std::advance(ILHS, 1);
1147 std::advance(IRHS, 1);
1148 }
1149 }
1150 ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1151 unsigned Count = 0;
1152 auto ILHS = LHSs.begin();
1153 auto IRHS = RHSs.begin();
1154 auto IPriv = Privates.begin();
1155 for (const Expr *IRef : Shareds) {
1156 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1157 // Emit private VarDecl with reduction init.
1158 RedCG.emitSharedLValue(*this, Count);
1159 RedCG.emitAggregateType(*this, Count);
1160 AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1161 RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1162 RedCG.getSharedLValue(Count),
1163 [&Emission](CodeGenFunction &CGF) {
1164 CGF.EmitAutoVarInit(Emission);
1165 return true;
1166 });
1167 EmitAutoVarCleanups(Emission);
1168 Address BaseAddr = RedCG.adjustPrivateAddress(
1169 *this, Count, Emission.getAllocatedAddress());
1170 bool IsRegistered = PrivateScope.addPrivate(
1171 RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1172 assert(IsRegistered && "private var already registered as private");
1173 // Silence the warning about unused variable.
1174 (void)IsRegistered;
1175
1176 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1177 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1178 QualType Type = PrivateVD->getType();
1179 bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1180 if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1181 // Store the address of the original variable associated with the LHS
1182 // implicit variable.
1183 PrivateScope.addPrivate(LHSVD, [&RedCG, Count, this]() {
1184 return RedCG.getSharedLValue(Count).getAddress(*this);
1185 });
1186 PrivateScope.addPrivate(
1187 RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1188 } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1189 isa<ArraySubscriptExpr>(IRef)) {
1190 // Store the address of the original variable associated with the LHS
1191 // implicit variable.
1192 PrivateScope.addPrivate(LHSVD, [&RedCG, Count, this]() {
1193 return RedCG.getSharedLValue(Count).getAddress(*this);
1194 });
1195 PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1196 return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1197 ConvertTypeForMem(RHSVD->getType()),
1198 "rhs.begin");
1199 });
1200 } else {
1201 QualType Type = PrivateVD->getType();
1202 bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1203 Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*this);
1204 // Store the address of the original variable associated with the LHS
1205 // implicit variable.
1206 if (IsArray) {
1207 OriginalAddr = Builder.CreateElementBitCast(
1208 OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1209 }
1210 PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1211 PrivateScope.addPrivate(
1212 RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1213 return IsArray
1214 ? Builder.CreateElementBitCast(
1215 GetAddrOfLocalVar(PrivateVD),
1216 ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1217 : GetAddrOfLocalVar(PrivateVD);
1218 });
1219 }
1220 ++ILHS;
1221 ++IRHS;
1222 ++IPriv;
1223 ++Count;
1224 }
1225 }
1226
1227 void CodeGenFunction::EmitOMPReductionClauseFinal(
1228 const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1229 if (!HaveInsertPoint())
1230 return;
1231 llvm::SmallVector<const Expr *, 8> Privates;
1232 llvm::SmallVector<const Expr *, 8> LHSExprs;
1233 llvm::SmallVector<const Expr *, 8> RHSExprs;
1234 llvm::SmallVector<const Expr *, 8> ReductionOps;
1235 bool HasAtLeastOneReduction = false;
1236 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1237 HasAtLeastOneReduction = true;
1238 Privates.append(C->privates().begin(), C->privates().end());
1239 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1240 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1241 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1242 }
1243 if (HasAtLeastOneReduction) {
1244 bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1245 isOpenMPParallelDirective(D.getDirectiveKind()) ||
1246 ReductionKind == OMPD_simd;
1247 bool SimpleReduction = ReductionKind == OMPD_simd;
1248 // Emit nowait reduction if nowait clause is present or directive is a
1249 // parallel directive (it always has implicit barrier).
1250 CGM.getOpenMPRuntime().emitReduction(
1251 *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1252 {WithNowait, SimpleReduction, ReductionKind});
1253 }
1254 }
1255
1256 static void emitPostUpdateForReductionClause(
1257 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1258 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1259 if (!CGF.HaveInsertPoint())
1260 return;
1261 llvm::BasicBlock *DoneBB = nullptr;
1262 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1263 if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1264 if (!DoneBB) {
1265 if (llvm::Value *Cond = CondGen(CGF)) {
1266 // If the first post-update expression is found, emit conditional
1267 // block if it was requested.
1268 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1269 DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1270 CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1271 CGF.EmitBlock(ThenBB);
1272 }
1273 }
1274 CGF.EmitIgnoredExpr(PostUpdate);
1275 }
1276 }
1277 if (DoneBB)
1278 CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1279 }
1280
1281 namespace {
1282 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1283 /// parallel function. This is necessary for combined constructs such as
1284 /// 'distribute parallel for'
1285 typedef llvm::function_ref<void(CodeGenFunction &,
1286 const OMPExecutableDirective &,
1287 llvm::SmallVectorImpl<llvm::Value *> &)>
1288 CodeGenBoundParametersTy;
1289 } // anonymous namespace
1290
1291 static void emitCommonOMPParallelDirective(
1292 CodeGenFunction &CGF, const OMPExecutableDirective &S,
1293 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1294 const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1295 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1296 llvm::Function *OutlinedFn =
1297 CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1298 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1299 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1300 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1301 llvm::Value *NumThreads =
1302 CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1303 /*IgnoreResultAssign=*/true);
1304 CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1305 CGF, NumThreads, NumThreadsClause->getBeginLoc());
1306 }
1307 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1308 CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1309 CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1310 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1311 }
1312 const Expr *IfCond = nullptr;
1313 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1314 if (C->getNameModifier() == OMPD_unknown ||
1315 C->getNameModifier() == OMPD_parallel) {
1316 IfCond = C->getCondition();
1317 break;
1318 }
1319 }
1320
1321 OMPParallelScope Scope(CGF, S);
1322 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
1323 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1324 // lower and upper bounds with the pragma 'for' chunking mechanism.
1325 // The following lambda takes care of appending the lower and upper bound
1326 // parameters when necessary
1327 CodeGenBoundParameters(CGF, S, CapturedVars);
1328 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1329 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1330 CapturedVars, IfCond);
1331 }
1332
1333 static void emitEmptyBoundParameters(CodeGenFunction &,
1334 const OMPExecutableDirective &,
1335 llvm::SmallVectorImpl<llvm::Value *> &) {}
1336
1337 void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
1338
1339 if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder()) {
1340 // Check if we have any if clause associated with the directive.
1341 llvm::Value *IfCond = nullptr;
1342 if (const auto *C = S.getSingleClause<OMPIfClause>())
1343 IfCond = EmitScalarExpr(C->getCondition(),
1344 /*IgnoreResultAssign=*/true);
1345
1346 llvm::Value *NumThreads = nullptr;
1347 if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())
1348 NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(),
1349 /*IgnoreResultAssign=*/true);
1350
1351 ProcBindKind ProcBind = OMP_PROC_BIND_default;
1352 if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
1353 ProcBind = ProcBindClause->getProcBindKind();
1354
1355 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1356
1357 // The cleanup callback that finalizes all variabels at the given location,
1358 // thus calls destructors etc.
1359 auto FiniCB = [this](InsertPointTy IP) {
1360 CGBuilderTy::InsertPointGuard IPG(Builder);
1361 assert(IP.getBlock()->end() != IP.getPoint() &&
1362 "OpenMP IR Builder should cause terminated block!");
1363 llvm::BasicBlock *IPBB = IP.getBlock();
1364 llvm::BasicBlock *DestBB = IPBB->splitBasicBlock(IP.getPoint());
1365 IPBB->getTerminator()->eraseFromParent();
1366 Builder.SetInsertPoint(IPBB);
1367 CodeGenFunction::JumpDest Dest = getJumpDestInCurrentScope(DestBB);
1368 EmitBranchThroughCleanup(Dest);
1369 };
1370
1371 // Privatization callback that performs appropriate action for
1372 // shared/private/firstprivate/lastprivate/copyin/... variables.
1373 //
1374 // TODO: This defaults to shared right now.
1375 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1376 llvm::Value &Val, llvm::Value *&ReplVal) {
1377 // The next line is appropriate only for variables (Val) with the
1378 // data-sharing attribute "shared".
1379 ReplVal = &Val;
1380
1381 return CodeGenIP;
1382 };
1383
1384 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1385 const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
1386
1387 auto BodyGenCB = [ParallelRegionBodyStmt,
1388 this](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
1389 llvm::BasicBlock &ContinuationBB) {
1390 auto OldAllocaIP = AllocaInsertPt;
1391 AllocaInsertPt = &*AllocaIP.getPoint();
1392
1393 auto OldReturnBlock = ReturnBlock;
1394 ReturnBlock = getJumpDestInCurrentScope(&ContinuationBB);
1395
1396 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
1397 CodeGenIPBB->splitBasicBlock(CodeGenIP.getPoint());
1398 llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator();
1399 CodeGenIPBBTI->removeFromParent();
1400
1401 Builder.SetInsertPoint(CodeGenIPBB);
1402
1403 EmitStmt(ParallelRegionBodyStmt);
1404
1405 Builder.Insert(CodeGenIPBBTI);
1406
1407 AllocaInsertPt = OldAllocaIP;
1408 ReturnBlock = OldReturnBlock;
1409 };
1410
1411 CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
1412 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
1413 Builder.restoreIP(OMPBuilder->CreateParallel(Builder, BodyGenCB, PrivCB,
1414 FiniCB, IfCond, NumThreads,
1415 ProcBind, S.hasCancel()));
1416 return;
1417 }
1418
1419 // Emit parallel region as a standalone region.
1420 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1421 Action.Enter(CGF);
1422 OMPPrivateScope PrivateScope(CGF);
1423 bool Copyins = CGF.EmitOMPCopyinClause(S);
1424 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1425 if (Copyins) {
1426 // Emit implicit barrier to synchronize threads and avoid data races on
1427 // propagation master's thread values of threadprivate variables to local
1428 // instances of that variables of all other implicit threads.
1429 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1430 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1431 /*ForceSimpleCall=*/true);
1432 }
1433 CGF.EmitOMPPrivateClause(S, PrivateScope);
1434 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1435 (void)PrivateScope.Privatize();
1436 CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1437 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1438 };
1439 emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1440 emitEmptyBoundParameters);
1441 emitPostUpdateForReductionClause(*this, S,
1442 [](CodeGenFunction &) { return nullptr; });
1443 }
1444
1445 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
1446 int MaxLevel, int Level = 0) {
1447 assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
1448 const Stmt *SimplifiedS = S->IgnoreContainers();
1449 if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
1450 PrettyStackTraceLoc CrashInfo(
1451 CGF.getContext().getSourceManager(), CS->getLBracLoc(),
1452 "LLVM IR generation of compound statement ('{}')");
1453
1454 // Keep track of the current cleanup stack depth, including debug scopes.
1455 CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
1456 for (const Stmt *CurStmt : CS->body())
1457 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
1458 return;
1459 }
1460 if (SimplifiedS == NextLoop) {
1461 if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
1462 S = For->getBody();
1463 } else {
1464 assert(isa<CXXForRangeStmt>(SimplifiedS) &&
1465 "Expected canonical for loop or range-based for loop.");
1466 const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
1467 CGF.EmitStmt(CXXFor->getLoopVarStmt());
1468 S = CXXFor->getBody();
1469 }
1470 if (Level + 1 < MaxLevel) {
1471 NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(
1472 S, /*TryImperfectlyNestedLoops=*/true);
1473 emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
1474 return;
1475 }
1476 }
1477 CGF.EmitStmt(S);
1478 }
1479
1480 void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
1481 JumpDest LoopExit) {
1482 RunCleanupsScope BodyScope(*this);
1483 // Update counters values on current iteration.
1484 for (const Expr *UE : D.updates())
1485 EmitIgnoredExpr(UE);
1486 // Update the linear variables.
1487 // In distribute directives only loop counters may be marked as linear, no
1488 // need to generate the code for them.
1489 if (!isOpenMPDistributeDirective(D.getDirectiveKind())) {
1490 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1491 for (const Expr *UE : C->updates())
1492 EmitIgnoredExpr(UE);
1493 }
1494 }
1495
1496 // On a continue in the body, jump to the end.
1497 JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1498 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1499 for (const Expr *E : D.finals_conditions()) {
1500 if (!E)
1501 continue;
1502 // Check that loop counter in non-rectangular nest fits into the iteration
1503 // space.
1504 llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");
1505 EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(),
1506 getProfileCount(D.getBody()));
1507 EmitBlock(NextBB);
1508 }
1509 // Emit loop variables for C++ range loops.
1510 const Stmt *Body =
1511 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
1512 // Emit loop body.
1513 emitBody(*this, Body,
1514 OMPLoopDirective::tryToFindNextInnerLoop(
1515 Body, /*TryImperfectlyNestedLoops=*/true),
1516 D.getCollapsedNumber());
1517
1518 // The end (updates/cleanups).
1519 EmitBlock(Continue.getBlock());
1520 BreakContinueStack.pop_back();
1521 }
1522
1523 void CodeGenFunction::EmitOMPInnerLoop(
1524 const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1525 const Expr *IncExpr,
1526 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1527 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1528 auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1529
1530 // Start the loop with a block that tests the condition.
1531 auto CondBlock = createBasicBlock("omp.inner.for.cond");
1532 EmitBlock(CondBlock);
1533 const SourceRange R = S.getSourceRange();
1534 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1535 SourceLocToDebugLoc(R.getEnd()));
1536
1537 // If there are any cleanups between here and the loop-exit scope,
1538 // create a block to stage a loop exit along.
1539 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1540 if (RequiresCleanup)
1541 ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1542
1543 llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1544
1545 // Emit condition.
1546 EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1547 if (ExitBlock != LoopExit.getBlock()) {
1548 EmitBlock(ExitBlock);
1549 EmitBranchThroughCleanup(LoopExit);
1550 }
1551
1552 EmitBlock(LoopBody);
1553 incrementProfileCounter(&S);
1554
1555 // Create a block for the increment.
1556 JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1557 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1558
1559 BodyGen(*this);
1560
1561 // Emit "IV = IV + 1" and a back-edge to the condition block.
1562 EmitBlock(Continue.getBlock());
1563 EmitIgnoredExpr(IncExpr);
1564 PostIncGen(*this);
1565 BreakContinueStack.pop_back();
1566 EmitBranch(CondBlock);
1567 LoopStack.pop();
1568 // Emit the fall-through block.
1569 EmitBlock(LoopExit.getBlock());
1570 }
1571
1572 bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
1573 if (!HaveInsertPoint())
1574 return false;
1575 // Emit inits for the linear variables.
1576 bool HasLinears = false;
1577 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1578 for (const Expr *Init : C->inits()) {
1579 HasLinears = true;
1580 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1581 if (const auto *Ref =
1582 dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1583 AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1584 const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1585 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1586 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1587 VD->getInit()->getType(), VK_LValue,
1588 VD->getInit()->getExprLoc());
1589 EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1590 VD->getType()),
1591 /*capturedByInit=*/false);
1592 EmitAutoVarCleanups(Emission);
1593 } else {
1594 EmitVarDecl(*VD);
1595 }
1596 }
1597 // Emit the linear steps for the linear clauses.
1598 // If a step is not constant, it is pre-calculated before the loop.
1599 if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1600 if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1601 EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1602 // Emit calculation of the linear step.
1603 EmitIgnoredExpr(CS);
1604 }
1605 }
1606 return HasLinears;
1607 }
1608
1609 void CodeGenFunction::EmitOMPLinearClauseFinal(
1610 const OMPLoopDirective &D,
1611 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1612 if (!HaveInsertPoint())
1613 return;
1614 llvm::BasicBlock *DoneBB = nullptr;
1615 // Emit the final values of the linear variables.
1616 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1617 auto IC = C->varlist_begin();
1618 for (const Expr *F : C->finals()) {
1619 if (!DoneBB) {
1620 if (llvm::Value *Cond = CondGen(*this)) {
1621 // If the first post-update expression is found, emit conditional
1622 // block if it was requested.
1623 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1624 DoneBB = createBasicBlock(".omp.linear.pu.done");
1625 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1626 EmitBlock(ThenBB);
1627 }
1628 }
1629 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1630 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1631 CapturedStmtInfo->lookup(OrigVD) != nullptr,
1632 (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1633 Address OrigAddr = EmitLValue(&DRE).getAddress(*this);
1634 CodeGenFunction::OMPPrivateScope VarScope(*this);
1635 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1636 (void)VarScope.Privatize();
1637 EmitIgnoredExpr(F);
1638 ++IC;
1639 }
1640 if (const Expr *PostUpdate = C->getPostUpdateExpr())
1641 EmitIgnoredExpr(PostUpdate);
1642 }
1643 if (DoneBB)
1644 EmitBlock(DoneBB, /*IsFinished=*/true);
1645 }
1646
1647 static void emitAlignedClause(CodeGenFunction &CGF,
1648 const OMPExecutableDirective &D) {
1649 if (!CGF.HaveInsertPoint())
1650 return;
1651 for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1652 llvm::APInt ClauseAlignment(64, 0);
1653 if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1654 auto *AlignmentCI =
1655 cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1656 ClauseAlignment = AlignmentCI->getValue();
1657 }
1658 for (const Expr *E : Clause->varlists()) {
1659 llvm::APInt Alignment(ClauseAlignment);
1660 if (Alignment == 0) {
1661 // OpenMP [2.8.1, Description]
1662 // If no optional parameter is specified, implementation-defined default
1663 // alignments for SIMD instructions on the target platforms are assumed.
1664 Alignment =
1665 CGF.getContext()
1666 .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
1667 E->getType()->getPointeeType()))
1668 .getQuantity();
1669 }
1670 assert((Alignment == 0 || Alignment.isPowerOf2()) &&
1671 "alignment is not power of 2");
1672 if (Alignment != 0) {
1673 llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1674 CGF.EmitAlignmentAssumption(
1675 PtrValue, E, /*No second loc needed*/ SourceLocation(),
1676 llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
1677 }
1678 }
1679 }
1680 }
1681
1682 void CodeGenFunction::EmitOMPPrivateLoopCounters(
1683 const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1684 if (!HaveInsertPoint())
1685 return;
1686 auto I = S.private_counters().begin();
1687 for (const Expr *E : S.counters()) {
1688 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1689 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1690 // Emit var without initialization.
1691 AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1692 EmitAutoVarCleanups(VarEmission);
1693 LocalDeclMap.erase(PrivateVD);
1694 (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1695 return VarEmission.getAllocatedAddress();
1696 });
1697 if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1698 VD->hasGlobalStorage()) {
1699 (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1700 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
1701 LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1702 E->getType(), VK_LValue, E->getExprLoc());
1703 return EmitLValue(&DRE).getAddress(*this);
1704 });
1705 } else {
1706 (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1707 return VarEmission.getAllocatedAddress();
1708 });
1709 }
1710 ++I;
1711 }
1712 // Privatize extra loop counters used in loops for ordered(n) clauses.
1713 for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1714 if (!C->getNumForLoops())
1715 continue;
1716 for (unsigned I = S.getCollapsedNumber(),
1717 E = C->getLoopNumIterations().size();
1718 I < E; ++I) {
1719 const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
1720 const auto *VD = cast<VarDecl>(DRE->getDecl());
1721 // Override only those variables that can be captured to avoid re-emission
1722 // of the variables declared within the loops.
1723 if (DRE->refersToEnclosingVariableOrCapture()) {
1724 (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1725 return CreateMemTemp(DRE->getType(), VD->getName());
1726 });
1727 }
1728 }
1729 }
1730 }
1731
1732 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1733 const Expr *Cond, llvm::BasicBlock *TrueBlock,
1734 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1735 if (!CGF.HaveInsertPoint())
1736 return;
1737 {
1738 CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1739 CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1740 (void)PreCondScope.Privatize();
1741 // Get initial values of real counters.
1742 for (const Expr *I : S.inits()) {
1743 CGF.EmitIgnoredExpr(I);
1744 }
1745 }
1746 // Create temp loop control variables with their init values to support
1747 // non-rectangular loops.
1748 CodeGenFunction::OMPMapVars PreCondVars;
1749 for (const Expr * E: S.dependent_counters()) {
1750 if (!E)
1751 continue;
1752 assert(!E->getType().getNonReferenceType()->isRecordType() &&
1753 "dependent counter must not be an iterator.");
1754 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1755 Address CounterAddr =
1756 CGF.CreateMemTemp(VD->getType().getNonReferenceType());
1757 (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
1758 }
1759 (void)PreCondVars.apply(CGF);
1760 for (const Expr *E : S.dependent_inits()) {
1761 if (!E)
1762 continue;
1763 CGF.EmitIgnoredExpr(E);
1764 }
1765 // Check that loop is executed at least one time.
1766 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1767 PreCondVars.restore(CGF);
1768 }
1769
1770 void CodeGenFunction::EmitOMPLinearClause(
1771 const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1772 if (!HaveInsertPoint())
1773 return;
1774 llvm::DenseSet<const VarDecl *> SIMDLCVs;
1775 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
1776 const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1777 for (const Expr *C : LoopDirective->counters()) {
1778 SIMDLCVs.insert(
1779 cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1780 }
1781 }
1782 for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1783 auto CurPrivate = C->privates().begin();
1784 for (const Expr *E : C->varlists()) {
1785 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1786 const auto *PrivateVD =
1787 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1788 if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1789 bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1790 // Emit private VarDecl with copy init.
1791 EmitVarDecl(*PrivateVD);
1792 return GetAddrOfLocalVar(PrivateVD);
1793 });
1794 assert(IsRegistered && "linear var already registered as private");
1795 // Silence the warning about unused variable.
1796 (void)IsRegistered;
1797 } else {
1798 EmitVarDecl(*PrivateVD);
1799 }
1800 ++CurPrivate;
1801 }
1802 }
1803 }
1804
1805 static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
1806 const OMPExecutableDirective &D,
1807 bool IsMonotonic) {
1808 if (!CGF.HaveInsertPoint())
1809 return;
1810 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1811 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1812 /*ignoreResult=*/true);
1813 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1814 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1815 // In presence of finite 'safelen', it may be unsafe to mark all
1816 // the memory instructions parallel, because loop-carried
1817 // dependences of 'safelen' iterations are possible.
1818 if (!IsMonotonic)
1819 CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
1820 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1821 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1822 /*ignoreResult=*/true);
1823 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1824 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1825 // In presence of finite 'safelen', it may be unsafe to mark all
1826 // the memory instructions parallel, because loop-carried
1827 // dependences of 'safelen' iterations are possible.
1828 CGF.LoopStack.setParallel(/*Enable=*/false);
1829 }
1830 }
1831
1832 void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D,
1833 bool IsMonotonic) {
1834 // Walk clauses and process safelen/lastprivate.
1835 LoopStack.setParallel(!IsMonotonic);
1836 LoopStack.setVectorizeEnable();
1837 emitSimdlenSafelenClause(*this, D, IsMonotonic);
1838 }
1839
1840 void CodeGenFunction::EmitOMPSimdFinal(
1841 const OMPLoopDirective &D,
1842 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1843 if (!HaveInsertPoint())
1844 return;
1845 llvm::BasicBlock *DoneBB = nullptr;
1846 auto IC = D.counters().begin();
1847 auto IPC = D.private_counters().begin();
1848 for (const Expr *F : D.finals()) {
1849 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1850 const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1851 const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1852 if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1853 OrigVD->hasGlobalStorage() || CED) {
1854 if (!DoneBB) {
1855 if (llvm::Value *Cond = CondGen(*this)) {
1856 // If the first post-update expression is found, emit conditional
1857 // block if it was requested.
1858 llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1859 DoneBB = createBasicBlock(".omp.final.done");
1860 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1861 EmitBlock(ThenBB);
1862 }
1863 }
1864 Address OrigAddr = Address::invalid();
1865 if (CED) {
1866 OrigAddr =
1867 EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(*this);
1868 } else {
1869 DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
1870 /*RefersToEnclosingVariableOrCapture=*/false,
1871 (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1872 OrigAddr = EmitLValue(&DRE).getAddress(*this);
1873 }
1874 OMPPrivateScope VarScope(*this);
1875 VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1876 (void)VarScope.Privatize();
1877 EmitIgnoredExpr(F);
1878 }
1879 ++IC;
1880 ++IPC;
1881 }
1882 if (DoneBB)
1883 EmitBlock(DoneBB, /*IsFinished=*/true);
1884 }
1885
1886 static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
1887 const OMPLoopDirective &S,
1888 CodeGenFunction::JumpDest LoopExit) {
1889 CGF.CGM.getOpenMPRuntime().initLastprivateConditionalCounter(CGF, S);
1890 CGF.EmitOMPLoopBody(S, LoopExit);
1891 CGF.EmitStopPoint(&S);
1892 }
1893
1894 /// Emit a helper variable and return corresponding lvalue.
1895 static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
1896 const DeclRefExpr *Helper) {
1897 auto VDecl = cast<VarDecl>(Helper->getDecl());
1898 CGF.EmitVarDecl(*VDecl);
1899 return CGF.EmitLValue(Helper);
1900 }
1901
1902 static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S,
1903 const RegionCodeGenTy &SimdInitGen,
1904 const RegionCodeGenTy &BodyCodeGen) {
1905 auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
1906 PrePostActionTy &) {
1907 CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
1908 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
1909 SimdInitGen(CGF);
1910
1911 BodyCodeGen(CGF);
1912 };
1913 auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
1914 CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
1915 CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
1916
1917 BodyCodeGen(CGF);
1918 };
1919 const Expr *IfCond = nullptr;
1920 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1921 if (CGF.getLangOpts().OpenMP >= 50 &&
1922 (C->getNameModifier() == OMPD_unknown ||
1923 C->getNameModifier() == OMPD_simd)) {
1924 IfCond = C->getCondition();
1925 break;
1926 }
1927 }
1928 if (IfCond) {
1929 CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen);
1930 } else {
1931 RegionCodeGenTy ThenRCG(ThenGen);
1932 ThenRCG(CGF);
1933 }
1934 }
1935
1936 static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
1937 PrePostActionTy &Action) {
1938 Action.Enter(CGF);
1939 assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&
1940 "Expected simd directive");
1941 OMPLoopScope PreInitScope(CGF, S);
1942 // if (PreCond) {
1943 // for (IV in 0..LastIteration) BODY;
1944 // <Final counter/linear vars updates>;
1945 // }
1946 //
1947 if (isOpenMPDistributeDirective(S.getDirectiveKind()) ||
1948 isOpenMPWorksharingDirective(S.getDirectiveKind()) ||
1949 isOpenMPTaskLoopDirective(S.getDirectiveKind())) {
1950 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1951 (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1952 }
1953
1954 // Emit: if (PreCond) - begin.
1955 // If the condition constant folds and can be elided, avoid emitting the
1956 // whole loop.
1957 bool CondConstant;
1958 llvm::BasicBlock *ContBlock = nullptr;
1959 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1960 if (!CondConstant)
1961 return;
1962 } else {
1963 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1964 ContBlock = CGF.createBasicBlock("simd.if.end");
1965 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1966 CGF.getProfileCount(&S));
1967 CGF.EmitBlock(ThenBlock);
1968 CGF.incrementProfileCounter(&S);
1969 }
1970
1971 // Emit the loop iteration variable.
1972 const Expr *IVExpr = S.getIterationVariable();
1973 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1974 CGF.EmitVarDecl(*IVDecl);
1975 CGF.EmitIgnoredExpr(S.getInit());
1976
1977 // Emit the iterations count variable.
1978 // If it is not a variable, Sema decided to calculate iterations count on
1979 // each iteration (e.g., it is foldable into a constant).
1980 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1981 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1982 // Emit calculation of the iterations count.
1983 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
1984 }
1985
1986 emitAlignedClause(CGF, S);
1987 (void)CGF.EmitOMPLinearClauseInit(S);
1988 {
1989 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1990 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1991 CGF.EmitOMPLinearClause(S, LoopScope);
1992 CGF.EmitOMPPrivateClause(S, LoopScope);
1993 CGF.EmitOMPReductionClauseInit(S, LoopScope);
1994 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(
1995 CGF, S, CGF.EmitLValue(S.getIterationVariable()));
1996 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1997 (void)LoopScope.Privatize();
1998 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
1999 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2000
2001 emitCommonSimdLoop(
2002 CGF, S,
2003 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2004 CGF.EmitOMPSimdInit(S);
2005 },
2006 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2007 CGF.EmitOMPInnerLoop(
2008 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
2009 [&S](CodeGenFunction &CGF) {
2010 CGF.CGM.getOpenMPRuntime().initLastprivateConditionalCounter(
2011 CGF, S);
2012 CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest());
2013 CGF.EmitStopPoint(&S);
2014 },
2015 [](CodeGenFunction &) {});
2016 });
2017 CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
2018 // Emit final copy of the lastprivate variables at the end of loops.
2019 if (HasLastprivateClause)
2020 CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
2021 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
2022 emitPostUpdateForReductionClause(CGF, S,
2023 [](CodeGenFunction &) { return nullptr; });
2024 }
2025 CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
2026 // Emit: if (PreCond) - end.
2027 if (ContBlock) {
2028 CGF.EmitBranch(ContBlock);
2029 CGF.EmitBlock(ContBlock, true);
2030 }
2031 }
2032
2033 void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
2034 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2035 emitOMPSimdRegion(CGF, S, Action);
2036 };
2037 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2038 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2039 }
2040
2041 void CodeGenFunction::EmitOMPOuterLoop(
2042 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
2043 CodeGenFunction::OMPPrivateScope &LoopScope,
2044 const CodeGenFunction::OMPLoopArguments &LoopArgs,
2045 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
2046 const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
2047 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2048
2049 const Expr *IVExpr = S.getIterationVariable();
2050 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2051 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2052
2053 JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
2054
2055 // Start the loop with a block that tests the condition.
2056 llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
2057 EmitBlock(CondBlock);
2058 const SourceRange R = S.getSourceRange();
2059 LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
2060 SourceLocToDebugLoc(R.getEnd()));
2061
2062 llvm::Value *BoolCondVal = nullptr;
2063 if (!DynamicOrOrdered) {
2064 // UB = min(UB, GlobalUB) or
2065 // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
2066 // 'distribute parallel for')
2067 EmitIgnoredExpr(LoopArgs.EUB);
2068 // IV = LB
2069 EmitIgnoredExpr(LoopArgs.Init);
2070 // IV < UB
2071 BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
2072 } else {
2073 BoolCondVal =
2074 RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
2075 LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
2076 }
2077
2078 // If there are any cleanups between here and the loop-exit scope,
2079 // create a block to stage a loop exit along.
2080 llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
2081 if (LoopScope.requiresCleanups())
2082 ExitBlock = createBasicBlock("omp.dispatch.cleanup");
2083
2084 llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
2085 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
2086 if (ExitBlock != LoopExit.getBlock()) {
2087 EmitBlock(ExitBlock);
2088 EmitBranchThroughCleanup(LoopExit);
2089 }
2090 EmitBlock(LoopBody);
2091
2092 // Emit "IV = LB" (in case of static schedule, we have already calculated new
2093 // LB for loop condition and emitted it above).
2094 if (DynamicOrOrdered)
2095 EmitIgnoredExpr(LoopArgs.Init);
2096
2097 // Create a block for the increment.
2098 JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
2099 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
2100
2101 emitCommonSimdLoop(
2102 *this, S,
2103 [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) {
2104 // Generate !llvm.loop.parallel metadata for loads and stores for loops
2105 // with dynamic/guided scheduling and without ordered clause.
2106 if (!isOpenMPSimdDirective(S.getDirectiveKind()))
2107 CGF.LoopStack.setParallel(!IsMonotonic);
2108 else
2109 CGF.EmitOMPSimdInit(S, IsMonotonic);
2110 },
2111 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
2112 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2113 SourceLocation Loc = S.getBeginLoc();
2114 // when 'distribute' is not combined with a 'for':
2115 // while (idx <= UB) { BODY; ++idx; }
2116 // when 'distribute' is combined with a 'for'
2117 // (e.g. 'distribute parallel for')
2118 // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
2119 CGF.EmitOMPInnerLoop(
2120 S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
2121 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
2122 CodeGenLoop(CGF, S, LoopExit);
2123 },
2124 [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
2125 CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
2126 });
2127 });
2128
2129 EmitBlock(Continue.getBlock());
2130 BreakContinueStack.pop_back();
2131 if (!DynamicOrOrdered) {
2132 // Emit "LB = LB + Stride", "UB = UB + Stride".
2133 EmitIgnoredExpr(LoopArgs.NextLB);
2134 EmitIgnoredExpr(LoopArgs.NextUB);
2135 }
2136
2137 EmitBranch(CondBlock);
2138 LoopStack.pop();
2139 // Emit the fall-through block.
2140 EmitBlock(LoopExit.getBlock());
2141
2142 // Tell the runtime we are done.
2143 auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
2144 if (!DynamicOrOrdered)
2145 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2146 S.getDirectiveKind());
2147 };
2148 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2149 }
2150
2151 void CodeGenFunction::EmitOMPForOuterLoop(
2152 const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
2153 const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
2154 const OMPLoopArguments &LoopArgs,
2155 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2156 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2157
2158 // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
2159 const bool DynamicOrOrdered =
2160 Ordered || RT.isDynamic(ScheduleKind.Schedule);
2161
2162 assert((Ordered ||
2163 !RT.isStaticNonchunked(ScheduleKind.Schedule,
2164 LoopArgs.Chunk != nullptr)) &&
2165 "static non-chunked schedule does not need outer loop");
2166
2167 // Emit outer loop.
2168 //
2169 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2170 // When schedule(dynamic,chunk_size) is specified, the iterations are
2171 // distributed to threads in the team in chunks as the threads request them.
2172 // Each thread executes a chunk of iterations, then requests another chunk,
2173 // until no chunks remain to be distributed. Each chunk contains chunk_size
2174 // iterations, except for the last chunk to be distributed, which may have
2175 // fewer iterations. When no chunk_size is specified, it defaults to 1.
2176 //
2177 // When schedule(guided,chunk_size) is specified, the iterations are assigned
2178 // to threads in the team in chunks as the executing threads request them.
2179 // Each thread executes a chunk of iterations, then requests another chunk,
2180 // until no chunks remain to be assigned. For a chunk_size of 1, the size of
2181 // each chunk is proportional to the number of unassigned iterations divided
2182 // by the number of threads in the team, decreasing to 1. For a chunk_size
2183 // with value k (greater than 1), the size of each chunk is determined in the
2184 // same way, with the restriction that the chunks do not contain fewer than k
2185 // iterations (except for the last chunk to be assigned, which may have fewer
2186 // than k iterations).
2187 //
2188 // When schedule(auto) is specified, the decision regarding scheduling is
2189 // delegated to the compiler and/or runtime system. The programmer gives the
2190 // implementation the freedom to choose any possible mapping of iterations to
2191 // threads in the team.
2192 //
2193 // When schedule(runtime) is specified, the decision regarding scheduling is
2194 // deferred until run time, and the schedule and chunk size are taken from the
2195 // run-sched-var ICV. If the ICV is set to auto, the schedule is
2196 // implementation defined
2197 //
2198 // while(__kmpc_dispatch_next(&LB, &UB)) {
2199 // idx = LB;
2200 // while (idx <= UB) { BODY; ++idx;
2201 // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
2202 // } // inner loop
2203 // }
2204 //
2205 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2206 // When schedule(static, chunk_size) is specified, iterations are divided into
2207 // chunks of size chunk_size, and the chunks are assigned to the threads in
2208 // the team in a round-robin fashion in the order of the thread number.
2209 //
2210 // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
2211 // while (idx <= UB) { BODY; ++idx; } // inner loop
2212 // LB = LB + ST;
2213 // UB = UB + ST;
2214 // }
2215 //
2216
2217 const Expr *IVExpr = S.getIterationVariable();
2218 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2219 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2220
2221 if (DynamicOrOrdered) {
2222 const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
2223 CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
2224 llvm::Value *LBVal = DispatchBounds.first;
2225 llvm::Value *UBVal = DispatchBounds.second;
2226 CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
2227 LoopArgs.Chunk};
2228 RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
2229 IVSigned, Ordered, DipatchRTInputValues);
2230 } else {
2231 CGOpenMPRuntime::StaticRTInput StaticInit(
2232 IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
2233 LoopArgs.ST, LoopArgs.Chunk);
2234 RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2235 ScheduleKind, StaticInit);
2236 }
2237
2238 auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
2239 const unsigned IVSize,
2240 const bool IVSigned) {
2241 if (Ordered) {
2242 CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
2243 IVSigned);
2244 }
2245 };
2246
2247 OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
2248 LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
2249 OuterLoopArgs.IncExpr = S.getInc();
2250 OuterLoopArgs.Init = S.getInit();
2251 OuterLoopArgs.Cond = S.getCond();
2252 OuterLoopArgs.NextLB = S.getNextLowerBound();
2253 OuterLoopArgs.NextUB = S.getNextUpperBound();
2254 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
2255 emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
2256 }
2257
2258 static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
2259 const unsigned IVSize, const bool IVSigned) {}
2260
2261 void CodeGenFunction::EmitOMPDistributeOuterLoop(
2262 OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
2263 OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
2264 const CodeGenLoopTy &CodeGenLoopContent) {
2265
2266 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2267
2268 // Emit outer loop.
2269 // Same behavior as a OMPForOuterLoop, except that schedule cannot be
2270 // dynamic
2271 //
2272
2273 const Expr *IVExpr = S.getIterationVariable();
2274 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2275 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2276
2277 CGOpenMPRuntime::StaticRTInput StaticInit(
2278 IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2279 LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2280 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
2281
2282 // for combined 'distribute' and 'for' the increment expression of distribute
2283 // is stored in DistInc. For 'distribute' alone, it is in Inc.
2284 Expr *IncExpr;
2285 if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
2286 IncExpr = S.getDistInc();
2287 else
2288 IncExpr = S.getInc();
2289
2290 // this routine is shared by 'omp distribute parallel for' and
2291 // 'omp distribute': select the right EUB expression depending on the
2292 // directive
2293 OMPLoopArguments OuterLoopArgs;
2294 OuterLoopArgs.LB = LoopArgs.LB;
2295 OuterLoopArgs.UB = LoopArgs.UB;
2296 OuterLoopArgs.ST = LoopArgs.ST;
2297 OuterLoopArgs.IL = LoopArgs.IL;
2298 OuterLoopArgs.Chunk = LoopArgs.Chunk;
2299 OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2300 ? S.getCombinedEnsureUpperBound()
2301 : S.getEnsureUpperBound();
2302 OuterLoopArgs.IncExpr = IncExpr;
2303 OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2304 ? S.getCombinedInit()
2305 : S.getInit();
2306 OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2307 ? S.getCombinedCond()
2308 : S.getCond();
2309 OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2310 ? S.getCombinedNextLowerBound()
2311 : S.getNextLowerBound();
2312 OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2313 ? S.getCombinedNextUpperBound()
2314 : S.getNextUpperBound();
2315
2316 EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2317 LoopScope, OuterLoopArgs, CodeGenLoopContent,
2318 emitEmptyOrdered);
2319 }
2320
2321 static std::pair<LValue, LValue>
2322 emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
2323 const OMPExecutableDirective &S) {
2324 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2325 LValue LB =
2326 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2327 LValue UB =
2328 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2329
2330 // When composing 'distribute' with 'for' (e.g. as in 'distribute
2331 // parallel for') we need to use the 'distribute'
2332 // chunk lower and upper bounds rather than the whole loop iteration
2333 // space. These are parameters to the outlined function for 'parallel'
2334 // and we copy the bounds of the previous schedule into the
2335 // the current ones.
2336 LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2337 LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2338 llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2339 PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2340 PrevLBVal = CGF.EmitScalarConversion(
2341 PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2342 LS.getIterationVariable()->getType(),
2343 LS.getPrevLowerBoundVariable()->getExprLoc());
2344 llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2345 PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2346 PrevUBVal = CGF.EmitScalarConversion(
2347 PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2348 LS.getIterationVariable()->getType(),
2349 LS.getPrevUpperBoundVariable()->getExprLoc());
2350
2351 CGF.EmitStoreOfScalar(PrevLBVal, LB);
2352 CGF.EmitStoreOfScalar(PrevUBVal, UB);
2353
2354 return {LB, UB};
2355 }
2356
2357 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2358 /// we need to use the LB and UB expressions generated by the worksharing
2359 /// code generation support, whereas in non combined situations we would
2360 /// just emit 0 and the LastIteration expression
2361 /// This function is necessary due to the difference of the LB and UB
2362 /// types for the RT emission routines for 'for_static_init' and
2363 /// 'for_dispatch_init'
2364 static std::pair<llvm::Value *, llvm::Value *>
2365 emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
2366 const OMPExecutableDirective &S,
2367 Address LB, Address UB) {
2368 const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2369 const Expr *IVExpr = LS.getIterationVariable();
2370 // when implementing a dynamic schedule for a 'for' combined with a
2371 // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2372 // is not normalized as each team only executes its own assigned
2373 // distribute chunk
2374 QualType IteratorTy = IVExpr->getType();
2375 llvm::Value *LBVal =
2376 CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2377 llvm::Value *UBVal =
2378 CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2379 return {LBVal, UBVal};
2380 }
2381
2382 static void emitDistributeParallelForDistributeInnerBoundParams(
2383 CodeGenFunction &CGF, const OMPExecutableDirective &S,
2384 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2385 const auto &Dir = cast<OMPLoopDirective>(S);
2386 LValue LB =
2387 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2388 llvm::Value *LBCast =
2389 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)),
2390 CGF.SizeTy, /*isSigned=*/false);
2391 CapturedVars.push_back(LBCast);
2392 LValue UB =
2393 CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2394
2395 llvm::Value *UBCast =
2396 CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)),
2397 CGF.SizeTy, /*isSigned=*/false);
2398 CapturedVars.push_back(UBCast);
2399 }
2400
2401 static void
2402 emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
2403 const OMPLoopDirective &S,
2404 CodeGenFunction::JumpDest LoopExit) {
2405 auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2406 PrePostActionTy &Action) {
2407 Action.Enter(CGF);
2408 bool HasCancel = false;
2409 if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
2410 if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2411 HasCancel = D->hasCancel();
2412 else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2413 HasCancel = D->hasCancel();
2414 else if (const auto *D =
2415 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2416 HasCancel = D->hasCancel();
2417 }
2418 CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
2419 HasCancel);
2420 CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
2421 emitDistributeParallelForInnerBounds,
2422 emitDistributeParallelForDispatchBounds);
2423 };
2424
2425 emitCommonOMPParallelDirective(
2426 CGF, S,
2427 isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2428 CGInlinedWorksharingLoop,
2429 emitDistributeParallelForDistributeInnerBoundParams);
2430 }
2431
2432 void CodeGenFunction::EmitOMPDistributeParallelForDirective(
2433 const OMPDistributeParallelForDirective &S) {
2434 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2435 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2436 S.getDistInc());
2437 };
2438 OMPLexicalScope Scope(*this, S, OMPD_parallel);
2439 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2440 }
2441
2442 void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
2443 const OMPDistributeParallelForSimdDirective &S) {
2444 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2445 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
2446 S.getDistInc());
2447 };
2448 OMPLexicalScope Scope(*this, S, OMPD_parallel);
2449 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2450 }
2451
2452 void CodeGenFunction::EmitOMPDistributeSimdDirective(
2453 const OMPDistributeSimdDirective &S) {
2454 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2455 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
2456 };
2457 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2458 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2459 }
2460
2461 void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
2462 CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2463 // Emit SPMD target parallel for region as a standalone region.
2464 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2465 emitOMPSimdRegion(CGF, S, Action);
2466 };
2467 llvm::Function *Fn;
2468 llvm::Constant *Addr;
2469 // Emit target region as a standalone region.
2470 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2471 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2472 assert(Fn && Addr && "Target device function emission failed.");
2473 }
2474
2475 void CodeGenFunction::EmitOMPTargetSimdDirective(
2476 const OMPTargetSimdDirective &S) {
2477 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2478 emitOMPSimdRegion(CGF, S, Action);
2479 };
2480 emitCommonOMPTargetDirective(*this, S, CodeGen);
2481 }
2482
2483 namespace {
2484 struct ScheduleKindModifiersTy {
2485 OpenMPScheduleClauseKind Kind;
2486 OpenMPScheduleClauseModifier M1;
2487 OpenMPScheduleClauseModifier M2;
2488 ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2489 OpenMPScheduleClauseModifier M1,
2490 OpenMPScheduleClauseModifier M2)
2491 : Kind(Kind), M1(M1), M2(M2) {}
2492 };
2493 } // namespace
2494
2495 bool CodeGenFunction::EmitOMPWorksharingLoop(
2496 const OMPLoopDirective &S, Expr *EUB,
2497 const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2498 const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2499 // Emit the loop iteration variable.
2500 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2501 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2502 EmitVarDecl(*IVDecl);
2503
2504 // Emit the iterations count variable.
2505 // If it is not a variable, Sema decided to calculate iterations count on each
2506 // iteration (e.g., it is foldable into a constant).
2507 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2508 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2509 // Emit calculation of the iterations count.
2510 EmitIgnoredExpr(S.getCalcLastIteration());
2511 }
2512
2513 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2514
2515 bool HasLastprivateClause;
2516 // Check pre-condition.
2517 {
2518 OMPLoopScope PreInitScope(*this, S);
2519 // Skip the entire loop if we don't meet the precondition.
2520 // If the condition constant folds and can be elided, avoid emitting the
2521 // whole loop.
2522 bool CondConstant;
2523 llvm::BasicBlock *ContBlock = nullptr;
2524 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2525 if (!CondConstant)
2526 return false;
2527 } else {
2528 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2529 ContBlock = createBasicBlock("omp.precond.end");
2530 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2531 getProfileCount(&S));
2532 EmitBlock(ThenBlock);
2533 incrementProfileCounter(&S);
2534 }
2535
2536 RunCleanupsScope DoacrossCleanupScope(*this);
2537 bool Ordered = false;
2538 if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2539 if (OrderedClause->getNumForLoops())
2540 RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
2541 else
2542 Ordered = true;
2543 }
2544
2545 llvm::DenseSet<const Expr *> EmittedFinals;
2546 emitAlignedClause(*this, S);
2547 bool HasLinears = EmitOMPLinearClauseInit(S);
2548 // Emit helper vars inits.
2549
2550 std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2551 LValue LB = Bounds.first;
2552 LValue UB = Bounds.second;
2553 LValue ST =
2554 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2555 LValue IL =
2556 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2557
2558 // Emit 'then' code.
2559 {
2560 OMPPrivateScope LoopScope(*this);
2561 if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2562 // Emit implicit barrier to synchronize threads and avoid data races on
2563 // initialization of firstprivate variables and post-update of
2564 // lastprivate variables.
2565 CGM.getOpenMPRuntime().emitBarrierCall(
2566 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2567 /*ForceSimpleCall=*/true);
2568 }
2569 EmitOMPPrivateClause(S, LoopScope);
2570 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(
2571 *this, S, EmitLValue(S.getIterationVariable()));
2572 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2573 EmitOMPReductionClauseInit(S, LoopScope);
2574 EmitOMPPrivateLoopCounters(S, LoopScope);
2575 EmitOMPLinearClause(S, LoopScope);
2576 (void)LoopScope.Privatize();
2577 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2578 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
2579
2580 // Detect the loop schedule kind and chunk.
2581 const Expr *ChunkExpr = nullptr;
2582 OpenMPScheduleTy ScheduleKind;
2583 if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2584 ScheduleKind.Schedule = C->getScheduleKind();
2585 ScheduleKind.M1 = C->getFirstScheduleModifier();
2586 ScheduleKind.M2 = C->getSecondScheduleModifier();
2587 ChunkExpr = C->getChunkSize();
2588 } else {
2589 // Default behaviour for schedule clause.
2590 CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
2591 *this, S, ScheduleKind.Schedule, ChunkExpr);
2592 }
2593 bool HasChunkSizeOne = false;
2594 llvm::Value *Chunk = nullptr;
2595 if (ChunkExpr) {
2596 Chunk = EmitScalarExpr(ChunkExpr);
2597 Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
2598 S.getIterationVariable()->getType(),
2599 S.getBeginLoc());
2600 Expr::EvalResult Result;
2601 if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
2602 llvm::APSInt EvaluatedChunk = Result.Val.getInt();
2603 HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
2604 }
2605 }
2606 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2607 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2608 // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2609 // If the static schedule kind is specified or if the ordered clause is
2610 // specified, and if no monotonic modifier is specified, the effect will
2611 // be as if the monotonic modifier was specified.
2612 bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule,
2613 /* Chunked */ Chunk != nullptr) && HasChunkSizeOne &&
2614 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
2615 if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
2616 /* Chunked */ Chunk != nullptr) ||
2617 StaticChunkedOne) &&
2618 !Ordered) {
2619 JumpDest LoopExit =
2620 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2621 emitCommonSimdLoop(
2622 *this, S,
2623 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2624 if (isOpenMPSimdDirective(S.getDirectiveKind()))
2625 CGF.EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2626 },
2627 [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
2628 &S, ScheduleKind, LoopExit,
2629 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
2630 // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2631 // When no chunk_size is specified, the iteration space is divided
2632 // into chunks that are approximately equal in size, and at most
2633 // one chunk is distributed to each thread. Note that the size of
2634 // the chunks is unspecified in this case.
2635 CGOpenMPRuntime::StaticRTInput StaticInit(
2636 IVSize, IVSigned, Ordered, IL.getAddress(CGF),
2637 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF),
2638 StaticChunkedOne ? Chunk : nullptr);
2639 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2640 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
2641 StaticInit);
2642 // UB = min(UB, GlobalUB);
2643 if (!StaticChunkedOne)
2644 CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
2645 // IV = LB;
2646 CGF.EmitIgnoredExpr(S.getInit());
2647 // For unchunked static schedule generate:
2648 //
2649 // while (idx <= UB) {
2650 // BODY;
2651 // ++idx;
2652 // }
2653 //
2654 // For static schedule with chunk one:
2655 //
2656 // while (IV <= PrevUB) {
2657 // BODY;
2658 // IV += ST;
2659 // }
2660 CGF.EmitOMPInnerLoop(
2661 S, LoopScope.requiresCleanups(),
2662 StaticChunkedOne ? S.getCombinedParForInDistCond()
2663 : S.getCond(),
2664 StaticChunkedOne ? S.getDistInc() : S.getInc(),
2665 [&S, LoopExit](CodeGenFunction &CGF) {
2666 CGF.CGM.getOpenMPRuntime()
2667 .initLastprivateConditionalCounter(CGF, S);
2668 CGF.EmitOMPLoopBody(S, LoopExit);
2669 CGF.EmitStopPoint(&S);
2670 },
2671 [](CodeGenFunction &) {});
2672 });
2673 EmitBlock(LoopExit.getBlock());
2674 // Tell the runtime we are done.
2675 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2676 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2677 S.getDirectiveKind());
2678 };
2679 OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2680 } else {
2681 const bool IsMonotonic =
2682 Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2683 ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2684 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2685 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2686 // Emit the outer loop, which requests its work chunk [LB..UB] from
2687 // runtime and runs the inner loop to process it.
2688 const OMPLoopArguments LoopArguments(
2689 LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
2690 IL.getAddress(*this), Chunk, EUB);
2691 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2692 LoopArguments, CGDispatchBounds);
2693 }
2694 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
2695 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
2696 return CGF.Builder.CreateIsNotNull(
2697 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2698 });
2699 }
2700 EmitOMPReductionClauseFinal(
2701 S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2702 ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2703 : /*Parallel only*/ OMPD_parallel);
2704 // Emit post-update of the reduction variables if IsLastIter != 0.
2705 emitPostUpdateForReductionClause(
2706 *this, S, [IL, &S](CodeGenFunction &CGF) {
2707 return CGF.Builder.CreateIsNotNull(
2708 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2709 });
2710 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2711 if (HasLastprivateClause)
2712 EmitOMPLastprivateClauseFinal(
2713 S, isOpenMPSimdDirective(S.getDirectiveKind()),
2714 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
2715 }
2716 EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2717 return CGF.Builder.CreateIsNotNull(
2718 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2719 });
2720 DoacrossCleanupScope.ForceCleanup();
2721 // We're now done with the loop, so jump to the continuation block.
2722 if (ContBlock) {
2723 EmitBranch(ContBlock);
2724 EmitBlock(ContBlock, /*IsFinished=*/true);
2725 }
2726 }
2727 return HasLastprivateClause;
2728 }
2729
2730 /// The following two functions generate expressions for the loop lower
2731 /// and upper bounds in case of static and dynamic (dispatch) schedule
2732 /// of the associated 'for' or 'distribute' loop.
2733 static std::pair<LValue, LValue>
2734 emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
2735 const auto &LS = cast<OMPLoopDirective>(S);
2736 LValue LB =
2737 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2738 LValue UB =
2739 EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2740 return {LB, UB};
2741 }
2742
2743 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2744 /// consider the lower and upper bound expressions generated by the
2745 /// worksharing loop support, but we use 0 and the iteration space size as
2746 /// constants
2747 static std::pair<llvm::Value *, llvm::Value *>
2748 emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
2749 Address LB, Address UB) {
2750 const auto &LS = cast<OMPLoopDirective>(S);
2751 const Expr *IVExpr = LS.getIterationVariable();
2752 const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2753 llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2754 llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2755 return {LBVal, UBVal};
2756 }
2757
2758 void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
2759 bool HasLastprivates = false;
2760 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2761 PrePostActionTy &) {
2762 OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2763 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2764 emitForLoopBounds,
2765 emitDispatchForLoopBounds);
2766 };
2767 {
2768 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2769 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2770 S.hasCancel());
2771 }
2772
2773 // Emit an implicit barrier at the end.
2774 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2775 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2776 }
2777
2778 void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
2779 bool HasLastprivates = false;
2780 auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2781 PrePostActionTy &) {
2782 HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2783 emitForLoopBounds,
2784 emitDispatchForLoopBounds);
2785 };
2786 {
2787 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2788 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2789 }
2790
2791 // Emit an implicit barrier at the end.
2792 if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2793 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2794 }
2795
2796 static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
2797 const Twine &Name,
2798 llvm::Value *Init = nullptr) {
2799 LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2800 if (Init)
2801 CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2802 return LVal;
2803 }
2804
2805 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2806 const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
2807 const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2808 bool HasLastprivates = false;
2809 auto &&CodeGen = [&S, CapturedStmt, CS,
2810 &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2811 ASTContext &C = CGF.getContext();
2812 QualType KmpInt32Ty =
2813 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2814 // Emit helper vars inits.
2815 LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2816 CGF.Builder.getInt32(0));
2817 llvm::ConstantInt *GlobalUBVal = CS != nullptr
2818 ? CGF.Builder.getInt32(CS->size() - 1)
2819 : CGF.Builder.getInt32(0);
2820 LValue UB =
2821 createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2822 LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2823 CGF.Builder.getInt32(1));
2824 LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2825 CGF.Builder.getInt32(0));
2826 // Loop counter.
2827 LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2828 OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2829 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2830 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2831 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2832 // Generate condition for loop.
2833 BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2834 OK_Ordinary, S.getBeginLoc(), FPOptions());
2835 // Increment for loop counter.
2836 UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2837 S.getBeginLoc(), true);
2838 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2839 // Iterate through all sections and emit a switch construct:
2840 // switch (IV) {
2841 // case 0:
2842 // <SectionStmt[0]>;
2843 // break;
2844 // ...
2845 // case <NumSection> - 1:
2846 // <SectionStmt[<NumSection> - 1]>;
2847 // break;
2848 // }
2849 // .omp.sections.exit:
2850 CGF.CGM.getOpenMPRuntime().initLastprivateConditionalCounter(CGF, S);
2851 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2852 llvm::SwitchInst *SwitchStmt =
2853 CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
2854 ExitBB, CS == nullptr ? 1 : CS->size());
2855 if (CS) {
2856 unsigned CaseNumber = 0;
2857 for (const Stmt *SubStmt : CS->children()) {
2858 auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2859 CGF.EmitBlock(CaseBB);
2860 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2861 CGF.EmitStmt(SubStmt);
2862 CGF.EmitBranch(ExitBB);
2863 ++CaseNumber;
2864 }
2865 } else {
2866 llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2867 CGF.EmitBlock(CaseBB);
2868 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2869 CGF.EmitStmt(CapturedStmt);
2870 CGF.EmitBranch(ExitBB);
2871 }
2872 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2873 };
2874
2875 CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2876 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2877 // Emit implicit barrier to synchronize threads and avoid data races on
2878 // initialization of firstprivate variables and post-update of lastprivate
2879 // variables.
2880 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2881 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2882 /*ForceSimpleCall=*/true);
2883 }
2884 CGF.EmitOMPPrivateClause(S, LoopScope);
2885 CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
2886 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2887 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2888 (void)LoopScope.Privatize();
2889 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2890 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2891
2892 // Emit static non-chunked loop.
2893 OpenMPScheduleTy ScheduleKind;
2894 ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2895 CGOpenMPRuntime::StaticRTInput StaticInit(
2896 /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF),
2897 LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF));
2898 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2899 CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2900 // UB = min(UB, GlobalUB);
2901 llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
2902 llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2903 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2904 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2905 // IV = LB;
2906 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
2907 // while (idx <= UB) { BODY; ++idx; }
2908 CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2909 [](CodeGenFunction &) {});
2910 // Tell the runtime we are done.
2911 auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2912 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2913 S.getDirectiveKind());
2914 };
2915 CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2916 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2917 // Emit post-update of the reduction variables if IsLastIter != 0.
2918 emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2919 return CGF.Builder.CreateIsNotNull(
2920 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2921 });
2922
2923 // Emit final copy of the lastprivate variables if IsLastIter != 0.
2924 if (HasLastprivates)
2925 CGF.EmitOMPLastprivateClauseFinal(
2926 S, /*NoFinals=*/false,
2927 CGF.Builder.CreateIsNotNull(
2928 CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
2929 };
2930
2931 bool HasCancel = false;
2932 if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2933 HasCancel = OSD->hasCancel();
2934 else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2935 HasCancel = OPSD->hasCancel();
2936 OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2937 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2938 HasCancel);
2939 // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2940 // clause. Otherwise the barrier will be generated by the codegen for the
2941 // directive.
2942 if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2943 // Emit implicit barrier to synchronize threads and avoid data races on
2944 // initialization of firstprivate variables.
2945 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2946 OMPD_unknown);
2947 }
2948 }
2949
2950 void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
2951 {
2952 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2953 EmitSections(S);
2954 }
2955 // Emit an implicit barrier at the end.
2956 if (!S.getSingleClause<OMPNowaitClause>()) {
2957 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2958 OMPD_sections);
2959 }
2960 }
2961
2962 void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
2963 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2964 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2965 };
2966 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2967 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2968 S.hasCancel());
2969 }
2970
2971 void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
2972 llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2973 llvm::SmallVector<const Expr *, 8> DestExprs;
2974 llvm::SmallVector<const Expr *, 8> SrcExprs;
2975 llvm::SmallVector<const Expr *, 8> AssignmentOps;
2976 // Check if there are any 'copyprivate' clauses associated with this
2977 // 'single' construct.
2978 // Build a list of copyprivate variables along with helper expressions
2979 // (<source>, <destination>, <destination>=<source> expressions)
2980 for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2981 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2982 DestExprs.append(C->destination_exprs().begin(),
2983 C->destination_exprs().end());
2984 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2985 AssignmentOps.append(C->assignment_ops().begin(),
2986 C->assignment_ops().end());
2987 }
2988 // Emit code for 'single' region along with 'copyprivate' clauses
2989 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2990 Action.Enter(CGF);
2991 OMPPrivateScope SingleScope(CGF);
2992 (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2993 CGF.EmitOMPPrivateClause(S, SingleScope);
2994 (void)SingleScope.Privatize();
2995 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
2996 };
2997 {
2998 OMPLexicalScope Scope(*this, S, OMPD_unknown);
2999 CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
3000 CopyprivateVars, DestExprs,
3001 SrcExprs, AssignmentOps);
3002 }
3003 // Emit an implicit barrier at the end (to avoid data race on firstprivate
3004 // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
3005 if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
3006 CGM.getOpenMPRuntime().emitBarrierCall(
3007 *this, S.getBeginLoc(),
3008 S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
3009 }
3010 }
3011
3012 static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
3013 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3014 Action.Enter(CGF);
3015 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
3016 };
3017 CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
3018 }
3019
3020 void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
3021 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3022 emitMaster(*this, S);
3023 }
3024
3025 void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
3026 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3027 Action.Enter(CGF);
3028 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
3029 };
3030 const Expr *Hint = nullptr;
3031 if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
3032 Hint = HintClause->getHint();
3033 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3034 CGM.getOpenMPRuntime().emitCriticalRegion(*this,
3035 S.getDirectiveName().getAsString(),
3036 CodeGen, S.getBeginLoc(), Hint);
3037 }
3038
3039 void CodeGenFunction::EmitOMPParallelForDirective(
3040 const OMPParallelForDirective &S) {
3041 // Emit directive as a combined directive that consists of two implicit
3042 // directives: 'parallel' with 'for' directive.
3043 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3044 Action.Enter(CGF);
3045 OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
3046 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
3047 emitDispatchForLoopBounds);
3048 };
3049 emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
3050 emitEmptyBoundParameters);
3051 }
3052
3053 void CodeGenFunction::EmitOMPParallelForSimdDirective(
3054 const OMPParallelForSimdDirective &S) {
3055 // Emit directive as a combined directive that consists of two implicit
3056 // directives: 'parallel' with 'for' directive.
3057 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3058 Action.Enter(CGF);
3059 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
3060 emitDispatchForLoopBounds);
3061 };
3062 emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
3063 emitEmptyBoundParameters);
3064 }
3065
3066 void CodeGenFunction::EmitOMPParallelMasterDirective(
3067 const OMPParallelMasterDirective &S) {
3068 // Emit directive as a combined directive that consists of two implicit
3069 // directives: 'parallel' with 'master' directive.
3070 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3071 Action.Enter(CGF);
3072 OMPPrivateScope PrivateScope(CGF);
3073 bool Copyins = CGF.EmitOMPCopyinClause(S);
3074 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3075 if (Copyins) {
3076 // Emit implicit barrier to synchronize threads and avoid data races on
3077 // propagation master's thread values of threadprivate variables to local
3078 // instances of that variables of all other implicit threads.
3079 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
3080 CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3081 /*ForceSimpleCall=*/true);
3082 }
3083 CGF.EmitOMPPrivateClause(S, PrivateScope);
3084 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
3085 (void)PrivateScope.Privatize();
3086 emitMaster(CGF, S);
3087 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
3088 };
3089 emitCommonOMPParallelDirective(*this, S, OMPD_master, CodeGen,
3090 emitEmptyBoundParameters);
3091 emitPostUpdateForReductionClause(*this, S,
3092 [](CodeGenFunction &) { return nullptr; });
3093 }
3094
3095 void CodeGenFunction::EmitOMPParallelSectionsDirective(
3096 const OMPParallelSectionsDirective &S) {
3097 // Emit directive as a combined directive that consists of two implicit
3098 // directives: 'parallel' with 'sections' directive.
3099 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3100 Action.Enter(CGF);
3101 CGF.EmitSections(S);
3102 };
3103 emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
3104 emitEmptyBoundParameters);
3105 }
3106
3107 void CodeGenFunction::EmitOMPTaskBasedDirective(
3108 const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
3109 const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
3110 OMPTaskDataTy &Data) {
3111 // Emit outlined function for task construct.
3112 const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
3113 auto I = CS->getCapturedDecl()->param_begin();
3114 auto PartId = std::next(I);
3115 auto TaskT = std::next(I, 4);
3116 // Check if the task is final
3117 if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
3118 // If the condition constant folds and can be elided, try to avoid emitting
3119 // the condition and the dead arm of the if/else.
3120 const Expr *Cond = Clause->getCondition();
3121 bool CondConstant;
3122 if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
3123 Data.Final.setInt(CondConstant);
3124 else
3125 Data.Final.setPointer(EvaluateExprAsBool(Cond));
3126 } else {
3127 // By default the task is not final.
3128 Data.Final.setInt(/*IntVal=*/false);
3129 }
3130 // Check if the task has 'priority' clause.
3131 if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
3132 const Expr *Prio = Clause->getPriority();
3133 Data.Priority.setInt(/*IntVal=*/true);
3134 Data.Priority.setPointer(EmitScalarConversion(
3135 EmitScalarExpr(Prio), Prio->getType(),
3136 getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
3137 Prio->getExprLoc()));
3138 }
3139 // The first function argument for tasks is a thread id, the second one is a
3140 // part id (0 for tied tasks, >=0 for untied task).
3141 llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
3142 // Get list of private variables.
3143 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
3144 auto IRef = C->varlist_begin();
3145 for (const Expr *IInit : C->private_copies()) {
3146 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
3147 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
3148 Data.PrivateVars.push_back(*IRef);
3149 Data.PrivateCopies.push_back(IInit);
3150 }
3151 ++IRef;
3152 }
3153 }
3154 EmittedAsPrivate.clear();
3155 // Get list of firstprivate variables.
3156 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3157 auto IRef = C->varlist_begin();
3158 auto IElemInitRef = C->inits().begin();
3159 for (const Expr *IInit : C->private_copies()) {
3160 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
3161 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
3162 Data.FirstprivateVars.push_back(*IRef);
3163 Data.FirstprivateCopies.push_back(IInit);
3164 Data.FirstprivateInits.push_back(*IElemInitRef);
3165 }
3166 ++IRef;
3167 ++IElemInitRef;
3168 }
3169 }
3170 // Get list of lastprivate variables (for taskloops).
3171 llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
3172 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
3173 auto IRef = C->varlist_begin();
3174 auto ID = C->destination_exprs().begin();
3175 for (const Expr *IInit : C->private_copies()) {
3176 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
3177 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
3178 Data.LastprivateVars.push_back(*IRef);
3179 Data.LastprivateCopies.push_back(IInit);
3180 }
3181 LastprivateDstsOrigs.insert(
3182 {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
3183 cast<DeclRefExpr>(*IRef)});
3184 ++IRef;
3185 ++ID;
3186 }
3187 }
3188 SmallVector<const Expr *, 4> LHSs;
3189 SmallVector<const Expr *, 4> RHSs;
3190 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
3191 auto IPriv = C->privates().begin();
3192 auto IRed = C->reduction_ops().begin();
3193 auto ILHS = C->lhs_exprs().begin();
3194 auto IRHS = C->rhs_exprs().begin();
3195 for (const Expr *Ref : C->varlists()) {
3196 Data.ReductionVars.emplace_back(Ref);
3197 Data.ReductionCopies.emplace_back(*IPriv);
3198 Data.ReductionOps.emplace_back(*IRed);
3199 LHSs.emplace_back(*ILHS);
3200 RHSs.emplace_back(*IRHS);
3201 std::advance(IPriv, 1);
3202 std::advance(IRed, 1);
3203 std::advance(ILHS, 1);
3204 std::advance(IRHS, 1);
3205 }
3206 }
3207 Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
3208 *this, S.getBeginLoc(), LHSs, RHSs, Data);
3209 // Build list of dependences.
3210 for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3211 for (const Expr *IRef : C->varlists())
3212 Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3213 auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
3214 CapturedRegion](CodeGenFunction &CGF,
3215 PrePostActionTy &Action) {
3216 // Set proper addresses for generated private copies.
3217 OMPPrivateScope Scope(CGF);
3218 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
3219 !Data.LastprivateVars.empty()) {
3220 llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3221 CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
3222 enum { PrivatesParam = 2, CopyFnParam = 3 };
3223 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3224 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3225 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3226 CS->getCapturedDecl()->getParam(PrivatesParam)));
3227 // Map privates.
3228 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
3229 llvm::SmallVector<llvm::Value *, 16> CallArgs;
3230 CallArgs.push_back(PrivatesPtr);
3231 for (const Expr *E : Data.PrivateVars) {
3232 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3233 Address PrivatePtr = CGF.CreateMemTemp(
3234 CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
3235 PrivatePtrs.emplace_back(VD, PrivatePtr);
3236 CallArgs.push_back(PrivatePtr.getPointer());
3237 }
3238 for (const Expr *E : Data.FirstprivateVars) {
3239 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3240 Address PrivatePtr =
3241 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3242 ".firstpriv.ptr.addr");
3243 PrivatePtrs.emplace_back(VD, PrivatePtr);
3244 CallArgs.push_back(PrivatePtr.getPointer());
3245 }
3246 for (const Expr *E : Data.LastprivateVars) {
3247 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3248 Address PrivatePtr =
3249 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3250 ".lastpriv.ptr.addr");
3251 PrivatePtrs.emplace_back(VD, PrivatePtr);
3252 CallArgs.push_back(PrivatePtr.getPointer());
3253 }
3254 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3255 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
3256 for (const auto &Pair : LastprivateDstsOrigs) {
3257 const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
3258 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
3259 /*RefersToEnclosingVariableOrCapture=*/
3260 CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
3261 Pair.second->getType(), VK_LValue,
3262 Pair.second->getExprLoc());
3263 Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
3264 return CGF.EmitLValue(&DRE).getAddress(CGF);
3265 });
3266 }
3267 for (const auto &Pair : PrivatePtrs) {
3268 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3269 CGF.getContext().getDeclAlign(Pair.first));
3270 Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3271 }
3272 }
3273 if (Data.Reductions) {
3274 OMPLexicalScope LexScope(CGF, S, CapturedRegion);
3275 ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionCopies,
3276 Data.ReductionOps);
3277 llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
3278 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
3279 for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
3280 RedCG.emitSharedLValue(CGF, Cnt);
3281 RedCG.emitAggregateType(CGF, Cnt);
3282 // FIXME: This must removed once the runtime library is fixed.
3283 // Emit required threadprivate variables for
3284 // initializer/combiner/finalizer.
3285 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3286 RedCG, Cnt);
3287 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3288 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3289 Replacement =
3290 Address(CGF.EmitScalarConversion(
3291 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3292 CGF.getContext().getPointerType(
3293 Data.ReductionCopies[Cnt]->getType()),
3294 Data.ReductionCopies[Cnt]->getExprLoc()),
3295 Replacement.getAlignment());
3296 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3297 Scope.addPrivate(RedCG.getBaseDecl(Cnt),
3298 [Replacement]() { return Replacement; });
3299 }
3300 }
3301 // Privatize all private variables except for in_reduction items.
3302 (void)Scope.Privatize();
3303 SmallVector<const Expr *, 4> InRedVars;
3304 SmallVector<const Expr *, 4> InRedPrivs;
3305 SmallVector<const Expr *, 4> InRedOps;
3306 SmallVector<const Expr *, 4> TaskgroupDescriptors;
3307 for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
3308 auto IPriv = C->privates().begin();
3309 auto IRed = C->reduction_ops().begin();
3310 auto ITD = C->taskgroup_descriptors().begin();
3311 for (const Expr *Ref : C->varlists()) {
3312 InRedVars.emplace_back(Ref);
3313 InRedPrivs.emplace_back(*IPriv);
3314 InRedOps.emplace_back(*IRed);
3315 TaskgroupDescriptors.emplace_back(*ITD);
3316 std::advance(IPriv, 1);
3317 std::advance(IRed, 1);
3318 std::advance(ITD, 1);
3319 }
3320 }
3321 // Privatize in_reduction items here, because taskgroup descriptors must be
3322 // privatized earlier.
3323 OMPPrivateScope InRedScope(CGF);
3324 if (!InRedVars.empty()) {
3325 ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
3326 for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
3327 RedCG.emitSharedLValue(CGF, Cnt);
3328 RedCG.emitAggregateType(CGF, Cnt);
3329 // The taskgroup descriptor variable is always implicit firstprivate and
3330 // privatized already during processing of the firstprivates.
3331 // FIXME: This must removed once the runtime library is fixed.
3332 // Emit required threadprivate variables for
3333 // initializer/combiner/finalizer.
3334 CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3335 RedCG, Cnt);
3336 llvm::Value *ReductionsPtr =
3337 CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
3338 TaskgroupDescriptors[Cnt]->getExprLoc());
3339 Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3340 CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3341 Replacement = Address(
3342 CGF.EmitScalarConversion(
3343 Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3344 CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
3345 InRedPrivs[Cnt]->getExprLoc()),
3346 Replacement.getAlignment());
3347 Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3348 InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
3349 [Replacement]() { return Replacement; });
3350 }
3351 }
3352 (void)InRedScope.Privatize();
3353
3354 Action.Enter(CGF);
3355 BodyGen(CGF);
3356 };
3357 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3358 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
3359 Data.NumberOfParts);
3360 OMPLexicalScope Scope(*this, S, llvm::None,
3361 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
3362 !isOpenMPSimdDirective(S.getDirectiveKind()));
3363 TaskGen(*this, OutlinedFn, Data);
3364 }
3365
3366 static ImplicitParamDecl *
3367 createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
3368 QualType Ty, CapturedDecl *CD,
3369 SourceLocation Loc) {
3370 auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3371 ImplicitParamDecl::Other);
3372 auto *OrigRef = DeclRefExpr::Create(
3373 C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3374 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3375 auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3376 ImplicitParamDecl::Other);
3377 auto *PrivateRef = DeclRefExpr::Create(
3378 C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
3379 /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3380 QualType ElemType = C.getBaseElementType(Ty);
3381 auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3382 ImplicitParamDecl::Other);
3383 auto *InitRef = DeclRefExpr::Create(
3384 C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3385 /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3386 PrivateVD->setInitStyle(VarDecl::CInit);
3387 PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3388 InitRef, /*BasePath=*/nullptr,
3389 VK_RValue));
3390 Data.FirstprivateVars.emplace_back(OrigRef);
3391 Data.FirstprivateCopies.emplace_back(PrivateRef);
3392 Data.FirstprivateInits.emplace_back(InitRef);
3393 return OrigVD;
3394 }
3395
3396 void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
3397 const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3398 OMPTargetDataInfo &InputInfo) {
3399 // Emit outlined function for task construct.
3400 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3401 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3402 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3403 auto I = CS->getCapturedDecl()->param_begin();
3404 auto PartId = std::next(I);
3405 auto TaskT = std::next(I, 4);
3406 OMPTaskDataTy Data;
3407 // The task is not final.
3408 Data.Final.setInt(/*IntVal=*/false);
3409 // Get list of firstprivate variables.
3410 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3411 auto IRef = C->varlist_begin();
3412 auto IElemInitRef = C->inits().begin();
3413 for (auto *IInit : C->private_copies()) {
3414 Data.FirstprivateVars.push_back(*IRef);
3415 Data.FirstprivateCopies.push_back(IInit);
3416 Data.FirstprivateInits.push_back(*IElemInitRef);
3417 ++IRef;
3418 ++IElemInitRef;
3419 }
3420 }
3421 OMPPrivateScope TargetScope(*this);
3422 VarDecl *BPVD = nullptr;
3423 VarDecl *PVD = nullptr;
3424 VarDecl *SVD = nullptr;
3425 if (InputInfo.NumberOfTargetItems > 0) {
3426 auto *CD = CapturedDecl::Create(
3427 getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3428 llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3429 QualType BaseAndPointersType = getContext().getConstantArrayType(
3430 getContext().VoidPtrTy, ArrSize, nullptr, ArrayType::Normal,
3431 /*IndexTypeQuals=*/0);
3432 BPVD = createImplicitFirstprivateForType(
3433 getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3434 PVD = createImplicitFirstprivateForType(
3435 getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3436 QualType SizesType = getContext().getConstantArrayType(
3437 getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),
3438 ArrSize, nullptr, ArrayType::Normal,
3439 /*IndexTypeQuals=*/0);
3440 SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3441 S.getBeginLoc());
3442 TargetScope.addPrivate(
3443 BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3444 TargetScope.addPrivate(PVD,
3445 [&InputInfo]() { return InputInfo.PointersArray; });
3446 TargetScope.addPrivate(SVD,
3447 [&InputInfo]() { return InputInfo.SizesArray; });
3448 }
3449 (void)TargetScope.Privatize();
3450 // Build list of dependences.
3451 for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3452 for (const Expr *IRef : C->varlists())
3453 Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3454 auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3455 &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3456 // Set proper addresses for generated private copies.
3457 OMPPrivateScope Scope(CGF);
3458 if (!Data.FirstprivateVars.empty()) {
3459 llvm::FunctionType *CopyFnTy = llvm::FunctionType::get(
3460 CGF.Builder.getVoidTy(), {CGF.Builder.getInt8PtrTy()}, true);
3461 enum { PrivatesParam = 2, CopyFnParam = 3 };
3462 llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3463 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3464 llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3465 CS->getCapturedDecl()->getParam(PrivatesParam)));
3466 // Map privates.
3467 llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
3468 llvm::SmallVector<llvm::Value *, 16> CallArgs;
3469 CallArgs.push_back(PrivatesPtr);
3470 for (const Expr *E : Data.FirstprivateVars) {
3471 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3472 Address PrivatePtr =
3473 CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3474 ".firstpriv.ptr.addr");
3475 PrivatePtrs.emplace_back(VD, PrivatePtr);
3476 CallArgs.push_back(PrivatePtr.getPointer());
3477 }
3478 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
3479 CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
3480 for (const auto &Pair : PrivatePtrs) {
3481 Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3482 CGF.getContext().getDeclAlign(Pair.first));
3483 Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3484 }
3485 }
3486 // Privatize all private variables except for in_reduction items.
3487 (void)Scope.Privatize();
3488 if (InputInfo.NumberOfTargetItems > 0) {
3489 InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3490 CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
3491 InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3492 CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
3493 InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3494 CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
3495 }
3496
3497 Action.Enter(CGF);
3498 OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3499 BodyGen(CGF);
3500 };
3501 llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3502 S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3503 Data.NumberOfParts);
3504 llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3505 IntegerLiteral IfCond(getContext(), TrueOrFalse,
3506 getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3507 SourceLocation());
3508
3509 CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
3510 SharedsTy, CapturedStruct, &IfCond, Data);
3511 }
3512
3513 void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
3514 // Emit outlined function for task construct.
3515 const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3516 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3517 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3518 const Expr *IfCond = nullptr;
3519 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3520 if (C->getNameModifier() == OMPD_unknown ||
3521 C->getNameModifier() == OMPD_task) {
3522 IfCond = C->getCondition();
3523 break;
3524 }
3525 }
3526
3527 OMPTaskDataTy Data;
3528 // Check if we should emit tied or untied task.
3529 Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3530 auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3531 CGF.EmitStmt(CS->getCapturedStmt());
3532 };
3533 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3534 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
3535 const OMPTaskDataTy &Data) {
3536 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
3537 SharedsTy, CapturedStruct, IfCond,
3538 Data);
3539 };
3540 EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3541 }
3542
3543 void CodeGenFunction::EmitOMPTaskyieldDirective(
3544 const OMPTaskyieldDirective &S) {
3545 CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
3546 }
3547
3548 void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
3549 CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
3550 }
3551
3552 void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
3553 CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
3554 }
3555
3556 void CodeGenFunction::EmitOMPTaskgroupDirective(
3557 const OMPTaskgroupDirective &S) {
3558 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3559 Action.Enter(CGF);
3560 if (const Expr *E = S.getReductionRef()) {
3561 SmallVector<const Expr *, 4> LHSs;
3562 SmallVector<const Expr *, 4> RHSs;
3563 OMPTaskDataTy Data;
3564 for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3565 auto IPriv = C->privates().begin();
3566 auto IRed = C->reduction_ops().begin();
3567 auto ILHS = C->lhs_exprs().begin();
3568 auto IRHS = C->rhs_exprs().begin();
3569 for (const Expr *Ref : C->varlists()) {
3570 Data.ReductionVars.emplace_back(Ref);
3571 Data.ReductionCopies.emplace_back(*IPriv);
3572 Data.ReductionOps.emplace_back(*IRed);
3573 LHSs.emplace_back(*ILHS);
3574 RHSs.emplace_back(*IRHS);
3575 std::advance(IPriv, 1);
3576 std::advance(IRed, 1);
3577 std::advance(ILHS, 1);
3578 std::advance(IRHS, 1);
3579 }
3580 }
3581 llvm::Value *ReductionDesc =
3582 CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
3583 LHSs, RHSs, Data);
3584 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3585 CGF.EmitVarDecl(*VD);
3586 CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3587 /*Volatile=*/false, E->getType());
3588 }
3589 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
3590 };
3591 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3592 CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
3593 }
3594
3595 void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
3596 CGM.getOpenMPRuntime().emitFlush(
3597 *this,
3598 [&S]() -> ArrayRef<const Expr *> {
3599 if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3600 return llvm::makeArrayRef(FlushClause->varlist_begin(),
3601 FlushClause->varlist_end());
3602 return llvm::None;
3603 }(),
3604 S.getBeginLoc());
3605 }
3606
3607 void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
3608 const CodeGenLoopTy &CodeGenLoop,
3609 Expr *IncExpr) {
3610 // Emit the loop iteration variable.
3611 const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3612 const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3613 EmitVarDecl(*IVDecl);
3614
3615 // Emit the iterations count variable.
3616 // If it is not a variable, Sema decided to calculate iterations count on each
3617 // iteration (e.g., it is foldable into a constant).
3618 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3619 EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3620 // Emit calculation of the iterations count.
3621 EmitIgnoredExpr(S.getCalcLastIteration());
3622 }
3623
3624 CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3625
3626 bool HasLastprivateClause = false;
3627 // Check pre-condition.
3628 {
3629 OMPLoopScope PreInitScope(*this, S);
3630 // Skip the entire loop if we don't meet the precondition.
3631 // If the condition constant folds and can be elided, avoid emitting the
3632 // whole loop.
3633 bool CondConstant;
3634 llvm::BasicBlock *ContBlock = nullptr;
3635 if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3636 if (!CondConstant)
3637 return;
3638 } else {
3639 llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3640 ContBlock = createBasicBlock("omp.precond.end");
3641 emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3642 getProfileCount(&S));
3643 EmitBlock(ThenBlock);
3644 incrementProfileCounter(&S);
3645 }
3646
3647 emitAlignedClause(*this, S);
3648 // Emit 'then' code.
3649 {
3650 // Emit helper vars inits.
3651
3652 LValue LB = EmitOMPHelperVar(
3653 *this, cast<DeclRefExpr>(
3654 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3655 ? S.getCombinedLowerBoundVariable()
3656 : S.getLowerBoundVariable())));
3657 LValue UB = EmitOMPHelperVar(
3658 *this, cast<DeclRefExpr>(
3659 (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3660 ? S.getCombinedUpperBoundVariable()
3661 : S.getUpperBoundVariable())));
3662 LValue ST =
3663 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3664 LValue IL =
3665 EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3666
3667 OMPPrivateScope LoopScope(*this);
3668 if (EmitOMPFirstprivateClause(S, LoopScope)) {
3669 // Emit implicit barrier to synchronize threads and avoid data races
3670 // on initialization of firstprivate variables and post-update of
3671 // lastprivate variables.
3672 CGM.getOpenMPRuntime().emitBarrierCall(
3673 *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3674 /*ForceSimpleCall=*/true);
3675 }
3676 EmitOMPPrivateClause(S, LoopScope);
3677 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
3678 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
3679 !isOpenMPTeamsDirective(S.getDirectiveKind()))
3680 EmitOMPReductionClauseInit(S, LoopScope);
3681 HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3682 EmitOMPPrivateLoopCounters(S, LoopScope);
3683 (void)LoopScope.Privatize();
3684 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
3685 CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
3686
3687 // Detect the distribute schedule kind and chunk.
3688 llvm::Value *Chunk = nullptr;
3689 OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
3690 if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3691 ScheduleKind = C->getDistScheduleKind();
3692 if (const Expr *Ch = C->getChunkSize()) {
3693 Chunk = EmitScalarExpr(Ch);
3694 Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3695 S.getIterationVariable()->getType(),
3696 S.getBeginLoc());
3697 }
3698 } else {
3699 // Default behaviour for dist_schedule clause.
3700 CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
3701 *this, S, ScheduleKind, Chunk);
3702 }
3703 const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3704 const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3705
3706 // OpenMP [2.10.8, distribute Construct, Description]
3707 // If dist_schedule is specified, kind must be static. If specified,
3708 // iterations are divided into chunks of size chunk_size, chunks are
3709 // assigned to the teams of the league in a round-robin fashion in the
3710 // order of the team number. When no chunk_size is specified, the
3711 // iteration space is divided into chunks that are approximately equal
3712 // in size, and at most one chunk is distributed to each team of the
3713 // league. The size of the chunks is unspecified in this case.
3714 bool StaticChunked = RT.isStaticChunked(
3715 ScheduleKind, /* Chunked */ Chunk != nullptr) &&
3716 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
3717 if (RT.isStaticNonchunked(ScheduleKind,
3718 /* Chunked */ Chunk != nullptr) ||
3719 StaticChunked) {
3720 CGOpenMPRuntime::StaticRTInput StaticInit(
3721 IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(*this),
3722 LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
3723 StaticChunked ? Chunk : nullptr);
3724 RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
3725 StaticInit);
3726 JumpDest LoopExit =
3727 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3728 // UB = min(UB, GlobalUB);
3729 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3730 ? S.getCombinedEnsureUpperBound()
3731 : S.getEnsureUpperBound());
3732 // IV = LB;
3733 EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3734 ? S.getCombinedInit()
3735 : S.getInit());
3736
3737 const Expr *Cond =
3738 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
3739 ? S.getCombinedCond()
3740 : S.getCond();
3741
3742 if (StaticChunked)
3743 Cond = S.getCombinedDistCond();
3744
3745 // For static unchunked schedules generate:
3746 //
3747 // 1. For distribute alone, codegen
3748 // while (idx <= UB) {
3749 // BODY;
3750 // ++idx;
3751 // }
3752 //
3753 // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
3754 // while (idx <= UB) {
3755 // <CodeGen rest of pragma>(LB, UB);
3756 // idx += ST;
3757 // }
3758 //
3759 // For static chunk one schedule generate:
3760 //
3761 // while (IV <= GlobalUB) {
3762 // <CodeGen rest of pragma>(LB, UB);
3763 // LB += ST;
3764 // UB += ST;
3765 // UB = min(UB, GlobalUB);
3766 // IV = LB;
3767 // }
3768 //
3769 emitCommonSimdLoop(
3770 *this, S,
3771 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3772 if (isOpenMPSimdDirective(S.getDirectiveKind()))
3773 CGF.EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3774 },
3775 [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,
3776 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
3777 CGF.EmitOMPInnerLoop(
3778 S, LoopScope.requiresCleanups(), Cond, IncExpr,
3779 [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3780 CodeGenLoop(CGF, S, LoopExit);
3781 },
3782 [&S, StaticChunked](CodeGenFunction &CGF) {
3783 if (StaticChunked) {
3784 CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
3785 CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
3786 CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
3787 CGF.EmitIgnoredExpr(S.getCombinedInit());
3788 }
3789 });
3790 });
3791 EmitBlock(LoopExit.getBlock());
3792 // Tell the runtime we are done.
3793 RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
3794 } else {
3795 // Emit the outer loop, which requests its work chunk [LB..UB] from
3796 // runtime and runs the inner loop to process it.
3797 const OMPLoopArguments LoopArguments = {
3798 LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
3799 IL.getAddress(*this), Chunk};
3800 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3801 CodeGenLoop);
3802 }
3803 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3804 EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3805 return CGF.Builder.CreateIsNotNull(
3806 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3807 });
3808 }
3809 if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
3810 !isOpenMPParallelDirective(S.getDirectiveKind()) &&
3811 !isOpenMPTeamsDirective(S.getDirectiveKind())) {
3812 EmitOMPReductionClauseFinal(S, OMPD_simd);
3813 // Emit post-update of the reduction variables if IsLastIter != 0.
3814 emitPostUpdateForReductionClause(
3815 *this, S, [IL, &S](CodeGenFunction &CGF) {
3816 return CGF.Builder.CreateIsNotNull(
3817 CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3818 });
3819 }
3820 // Emit final copy of the lastprivate variables if IsLastIter != 0.
3821 if (HasLastprivateClause) {
3822 EmitOMPLastprivateClauseFinal(
3823 S, /*NoFinals=*/false,
3824 Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3825 }
3826 }
3827
3828 // We're now done with the loop, so jump to the continuation block.
3829 if (ContBlock) {
3830 EmitBranch(ContBlock);
3831 EmitBlock(ContBlock, true);
3832 }
3833 }
3834 }
3835
3836 void CodeGenFunction::EmitOMPDistributeDirective(
3837 const OMPDistributeDirective &S) {
3838 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3839 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
3840 };
3841 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3842 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3843 }
3844
3845 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3846 const CapturedStmt *S) {
3847 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3848 CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
3849 CGF.CapturedStmtInfo = &CapStmtInfo;
3850 llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3851 Fn->setDoesNotRecurse();
3852 return Fn;
3853 }
3854
3855 void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
3856 if (S.hasClausesOfKind<OMPDependClause>()) {
3857 assert(!S.getAssociatedStmt() &&
3858 "No associated statement must be in ordered depend construct.");
3859 for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3860 CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3861 return;
3862 }
3863 const auto *C = S.getSingleClause<OMPSIMDClause>();
3864 auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3865 PrePostActionTy &Action) {
3866 const CapturedStmt *CS = S.getInnermostCapturedStmt();
3867 if (C) {
3868 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
3869 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3870 llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3871 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3872 OutlinedFn, CapturedVars);
3873 } else {
3874 Action.Enter(CGF);
3875 CGF.EmitStmt(CS->getCapturedStmt());
3876 }
3877 };
3878 OMPLexicalScope Scope(*this, S, OMPD_unknown);
3879 CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
3880 }
3881
3882 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
3883 QualType SrcType, QualType DestType,
3884 SourceLocation Loc) {
3885 assert(CGF.hasScalarEvaluationKind(DestType) &&
3886 "DestType must have scalar evaluation kind.");
3887 assert(!Val.isAggregate() && "Must be a scalar or complex.");
3888 return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3889 DestType, Loc)
3890 : CGF.EmitComplexToScalarConversion(
3891 Val.getComplexVal(), SrcType, DestType, Loc);
3892 }
3893
3894 static CodeGenFunction::ComplexPairTy
3895 convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
3896 QualType DestType, SourceLocation Loc) {
3897 assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3898 "DestType must have complex evaluation kind.");
3899 CodeGenFunction::ComplexPairTy ComplexVal;
3900 if (Val.isScalar()) {
3901 // Convert the input element to the element type of the complex.
3902 QualType DestElementType =
3903 DestType->castAs<ComplexType>()->getElementType();
3904 llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3905 Val.getScalarVal(), SrcType, DestElementType, Loc);
3906 ComplexVal = CodeGenFunction::ComplexPairTy(
3907 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3908 } else {
3909 assert(Val.isComplex() && "Must be a scalar or complex.");
3910 QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3911 QualType DestElementType =
3912 DestType->castAs<ComplexType>()->getElementType();
3913 ComplexVal.first = CGF.EmitScalarConversion(
3914 Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3915 ComplexVal.second = CGF.EmitScalarConversion(
3916 Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3917 }
3918 return ComplexVal;
3919 }
3920
3921 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3922 LValue LVal, RValue RVal) {
3923 if (LVal.isGlobalReg()) {
3924 CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3925 } else {
3926 CGF.EmitAtomicStore(RVal, LVal,
3927 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3928 : llvm::AtomicOrdering::Monotonic,
3929 LVal.isVolatile(), /*isInit=*/false);
3930 }
3931 }
3932
3933 void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
3934 QualType RValTy, SourceLocation Loc) {
3935 switch (getEvaluationKind(LVal.getType())) {
3936 case TEK_Scalar:
3937 EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3938 *this, RVal, RValTy, LVal.getType(), Loc)),
3939 LVal);
3940 break;
3941 case TEK_Complex:
3942 EmitStoreOfComplex(
3943 convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3944 /*isInit=*/false);
3945 break;
3946 case TEK_Aggregate:
3947 llvm_unreachable("Must be a scalar or complex.");
3948 }
3949 }
3950
3951 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3952 const Expr *X, const Expr *V,
3953 SourceLocation Loc) {
3954 // v = x;
3955 assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3956 assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3957 LValue XLValue = CGF.EmitLValue(X);
3958 LValue VLValue = CGF.EmitLValue(V);
3959 RValue Res = XLValue.isGlobalReg()
3960 ? CGF.EmitLoadOfLValue(XLValue, Loc)
3961 : CGF.EmitAtomicLoad(
3962 XLValue, Loc,
3963 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3964 : llvm::AtomicOrdering::Monotonic,
3965 XLValue.isVolatile());
3966 // OpenMP, 2.12.6, atomic Construct
3967 // Any atomic construct with a seq_cst clause forces the atomically
3968 // performed operation to include an implicit flush operation without a
3969 // list.
3970 if (IsSeqCst)
3971 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3972 CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3973 }
3974
3975 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3976 const Expr *X, const Expr *E,
3977 SourceLocation Loc) {
3978 // x = expr;
3979 assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3980 emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3981 // OpenMP, 2.12.6, atomic Construct
3982 // Any atomic construct with a seq_cst clause forces the atomically
3983 // performed operation to include an implicit flush operation without a
3984 // list.
3985 if (IsSeqCst)
3986 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3987 }
3988
3989 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3990 RValue Update,
3991 BinaryOperatorKind BO,
3992 llvm::AtomicOrdering AO,
3993 bool IsXLHSInRHSPart) {
3994 ASTContext &Context = CGF.getContext();
3995 // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3996 // expression is simple and atomic is allowed for the given type for the
3997 // target platform.
3998 if (BO == BO_Comma || !Update.isScalar() ||
3999 !Update.getScalarVal()->getType()->isIntegerTy() || !X.isSimple() ||
4000 (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
4001 (Update.getScalarVal()->getType() !=
4002 X.getAddress(CGF).getElementType())) ||
4003 !X.getAddress(CGF).getElementType()->isIntegerTy() ||
4004 !Context.getTargetInfo().hasBuiltinAtomic(
4005 Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
4006 return std::make_pair(false, RValue::get(nullptr));
4007
4008 llvm::AtomicRMWInst::BinOp RMWOp;
4009 switch (BO) {
4010 case BO_Add:
4011 RMWOp = llvm::AtomicRMWInst::Add;
4012 break;
4013 case BO_Sub:
4014 if (!IsXLHSInRHSPart)
4015 return std::make_pair(false, RValue::get(nullptr));
4016 RMWOp = llvm::AtomicRMWInst::Sub;
4017 break;
4018 case BO_And:
4019 RMWOp = llvm::AtomicRMWInst::And;
4020 break;
4021 case BO_Or:
4022 RMWOp = llvm::AtomicRMWInst::Or;
4023 break;
4024 case BO_Xor:
4025 RMWOp = llvm::AtomicRMWInst::Xor;
4026 break;
4027 case BO_LT:
4028 RMWOp = X.getType()->hasSignedIntegerRepresentation()
4029 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
4030 : llvm::AtomicRMWInst::Max)
4031 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
4032 : llvm::AtomicRMWInst::UMax);
4033 break;
4034 case BO_GT:
4035 RMWOp = X.getType()->hasSignedIntegerRepresentation()
4036 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
4037 : llvm::AtomicRMWInst::Min)
4038 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
4039 : llvm::AtomicRMWInst::UMin);
4040 break;
4041 case BO_Assign:
4042 RMWOp = llvm::AtomicRMWInst::Xchg;
4043 break;
4044 case BO_Mul:
4045 case BO_Div:
4046 case BO_Rem:
4047 case BO_Shl:
4048 case BO_Shr:
4049 case BO_LAnd:
4050 case BO_LOr:
4051 return std::make_pair(false, RValue::get(nullptr));
4052 case BO_PtrMemD:
4053 case BO_PtrMemI:
4054 case BO_LE:
4055 case BO_GE:
4056 case BO_EQ:
4057 case BO_NE:
4058 case BO_Cmp:
4059 case BO_AddAssign:
4060 case BO_SubAssign:
4061 case BO_AndAssign:
4062 case BO_OrAssign:
4063 case BO_XorAssign:
4064 case BO_MulAssign:
4065 case BO_DivAssign:
4066 case BO_RemAssign:
4067 case BO_ShlAssign:
4068 case BO_ShrAssign:
4069 case BO_Comma:
4070 llvm_unreachable("Unsupported atomic update operation");
4071 }
4072 llvm::Value *UpdateVal = Update.getScalarVal();
4073 if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
4074 UpdateVal = CGF.Builder.CreateIntCast(
4075 IC, X.getAddress(CGF).getElementType(),
4076 X.getType()->hasSignedIntegerRepresentation());
4077 }
4078 llvm::Value *Res =
4079 CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(CGF), UpdateVal, AO);
4080 return std::make_pair(true, RValue::get(Res));
4081 }
4082
4083 std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
4084 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
4085 llvm::AtomicOrdering AO, SourceLocation Loc,
4086 const llvm::function_ref<RValue(RValue)> CommonGen) {
4087 // Update expressions are allowed to have the following forms:
4088 // x binop= expr; -> xrval + expr;
4089 // x++, ++x -> xrval + 1;
4090 // x--, --x -> xrval - 1;
4091 // x = x binop expr; -> xrval binop expr
4092 // x = expr Op x; - > expr binop xrval;
4093 auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
4094 if (!Res.first) {
4095 if (X.isGlobalReg()) {
4096 // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
4097 // 'xrval'.
4098 EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
4099 } else {
4100 // Perform compare-and-swap procedure.
4101 EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
4102 }
4103 }
4104 return Res;
4105 }
4106
4107 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
4108 const Expr *X, const Expr *E,
4109 const Expr *UE, bool IsXLHSInRHSPart,
4110 SourceLocation Loc) {
4111 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
4112 "Update expr in 'atomic update' must be a binary operator.");
4113 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
4114 // Update expressions are allowed to have the following forms:
4115 // x binop= expr; -> xrval + expr;
4116 // x++, ++x -> xrval + 1;
4117 // x--, --x -> xrval - 1;
4118 // x = x binop expr; -> xrval binop expr
4119 // x = expr Op x; - > expr binop xrval;
4120 assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
4121 LValue XLValue = CGF.EmitLValue(X);
4122 RValue ExprRValue = CGF.EmitAnyExpr(E);
4123 llvm::AtomicOrdering AO = IsSeqCst
4124 ? llvm::AtomicOrdering::SequentiallyConsistent
4125 : llvm::AtomicOrdering::Monotonic;
4126 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
4127 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
4128 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
4129 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
4130 auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
4131 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
4132 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
4133 return CGF.EmitAnyExpr(UE);
4134 };
4135 (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
4136 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
4137 // OpenMP, 2.12.6, atomic Construct
4138 // Any atomic construct with a seq_cst clause forces the atomically
4139 // performed operation to include an implicit flush operation without a
4140 // list.
4141 if (IsSeqCst)
4142 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
4143 }
4144
4145 static RValue convertToType(CodeGenFunction &CGF, RValue Value,
4146 QualType SourceType, QualType ResType,
4147 SourceLocation Loc) {
4148 switch (CGF.getEvaluationKind(ResType)) {
4149 case TEK_Scalar:
4150 return RValue::get(
4151 convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
4152 case TEK_Complex: {
4153 auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
4154 return RValue::getComplex(Res.first, Res.second);
4155 }
4156 case TEK_Aggregate:
4157 break;
4158 }
4159 llvm_unreachable("Must be a scalar or complex.");
4160 }
4161
4162 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
4163 bool IsPostfixUpdate, const Expr *V,
4164 const Expr *X, const Expr *E,
4165 const Expr *UE, bool IsXLHSInRHSPart,
4166 SourceLocation Loc) {
4167 assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
4168 assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
4169 RValue NewVVal;
4170 LValue VLValue = CGF.EmitLValue(V);
4171 LValue XLValue = CGF.EmitLValue(X);
4172 RValue ExprRValue = CGF.EmitAnyExpr(E);
4173 llvm::AtomicOrdering AO = IsSeqCst
4174 ? llvm::AtomicOrdering::SequentiallyConsistent
4175 : llvm::AtomicOrdering::Monotonic;
4176 QualType NewVValType;
4177 if (UE) {
4178 // 'x' is updated with some additional value.
4179 assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
4180 "Update expr in 'atomic capture' must be a binary operator.");
4181 const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
4182 // Update expressions are allowed to have the following forms:
4183 // x binop= expr; -> xrval + expr;
4184 // x++, ++x -> xrval + 1;
4185 // x--, --x -> xrval - 1;
4186 // x = x binop expr; -> xrval binop expr
4187 // x = expr Op x; - > expr binop xrval;
4188 const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
4189 const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
4190 const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
4191 NewVValType = XRValExpr->getType();
4192 const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
4193 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
4194 IsPostfixUpdate](RValue XRValue) {
4195 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
4196 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
4197 RValue Res = CGF.EmitAnyExpr(UE);
4198 NewVVal = IsPostfixUpdate ? XRValue : Res;
4199 return Res;
4200 };
4201 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
4202 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
4203 if (Res.first) {
4204 // 'atomicrmw' instruction was generated.
4205 if (IsPostfixUpdate) {
4206 // Use old value from 'atomicrmw'.
4207 NewVVal = Res.second;
4208 } else {
4209 // 'atomicrmw' does not provide new value, so evaluate it using old
4210 // value of 'x'.
4211 CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
4212 CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
4213 NewVVal = CGF.EmitAnyExpr(UE);
4214 }
4215 }
4216 } else {
4217 // 'x' is simply rewritten with some 'expr'.
4218 NewVValType = X->getType().getNonReferenceType();
4219 ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
4220 X->getType().getNonReferenceType(), Loc);
4221 auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
4222 NewVVal = XRValue;
4223 return ExprRValue;
4224 };
4225 // Try to perform atomicrmw xchg, otherwise simple exchange.
4226 auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
4227 XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
4228 Loc, Gen);
4229 if (Res.first) {
4230 // 'atomicrmw' instruction was generated.
4231 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
4232 }
4233 }
4234 // Emit post-update store to 'v' of old/new 'x' value.
4235 CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
4236 // OpenMP, 2.12.6, atomic Construct
4237 // Any atomic construct with a seq_cst clause forces the atomically
4238 // performed operation to include an implicit flush operation without a
4239 // list.
4240 if (IsSeqCst)
4241 CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
4242 }
4243
4244 static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
4245 bool IsSeqCst, bool IsPostfixUpdate,
4246 const Expr *X, const Expr *V, const Expr *E,
4247 const Expr *UE, bool IsXLHSInRHSPart,
4248 SourceLocation Loc) {
4249 switch (Kind) {
4250 case OMPC_read:
4251 emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
4252 break;
4253 case OMPC_write:
4254 emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
4255 break;
4256 case OMPC_unknown:
4257 case OMPC_update:
4258 emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
4259 break;
4260 case OMPC_capture:
4261 emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
4262 IsXLHSInRHSPart, Loc);
4263 break;
4264 case OMPC_if:
4265 case OMPC_final:
4266 case OMPC_num_threads:
4267 case OMPC_private:
4268 case OMPC_firstprivate:
4269 case OMPC_lastprivate:
4270 case OMPC_reduction:
4271 case OMPC_task_reduction:
4272 case OMPC_in_reduction:
4273 case OMPC_safelen:
4274 case OMPC_simdlen:
4275 case OMPC_allocator:
4276 case OMPC_allocate:
4277 case OMPC_collapse:
4278 case OMPC_default:
4279 case OMPC_seq_cst:
4280 case OMPC_shared:
4281 case OMPC_linear:
4282 case OMPC_aligned:
4283 case OMPC_copyin:
4284 case OMPC_copyprivate:
4285 case OMPC_flush:
4286 case OMPC_proc_bind:
4287 case OMPC_schedule:
4288 case OMPC_ordered:
4289 case OMPC_nowait:
4290 case OMPC_untied:
4291 case OMPC_threadprivate:
4292 case OMPC_depend:
4293 case OMPC_mergeable:
4294 case OMPC_device:
4295 case OMPC_threads:
4296 case OMPC_simd:
4297 case OMPC_map:
4298 case OMPC_num_teams:
4299 case OMPC_thread_limit:
4300 case OMPC_priority:
4301 case OMPC_grainsize:
4302 case OMPC_nogroup:
4303 case OMPC_num_tasks:
4304 case OMPC_hint:
4305 case OMPC_dist_schedule:
4306 case OMPC_defaultmap:
4307 case OMPC_uniform:
4308 case OMPC_to:
4309 case OMPC_from:
4310 case OMPC_use_device_ptr:
4311 case OMPC_is_device_ptr:
4312 case OMPC_unified_address:
4313 case OMPC_unified_shared_memory:
4314 case OMPC_reverse_offload:
4315 case OMPC_dynamic_allocators:
4316 case OMPC_atomic_default_mem_order:
4317 case OMPC_device_type:
4318 case OMPC_match:
4319 case OMPC_nontemporal:
4320 llvm_unreachable("Clause is not allowed in 'omp atomic'.");
4321 }
4322 }
4323
4324 void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
4325 bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
4326 OpenMPClauseKind Kind = OMPC_unknown;
4327 for (const OMPClause *C : S.clauses()) {
4328 // Find first clause (skip seq_cst clause, if it is first).
4329 if (C->getClauseKind() != OMPC_seq_cst) {
4330 Kind = C->getClauseKind();
4331 break;
4332 }
4333 }
4334
4335 const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
4336 if (const auto *FE = dyn_cast<FullExpr>(CS))
4337 enterFullExpression(FE);
4338 // Processing for statements under 'atomic capture'.
4339 if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
4340 for (const Stmt *C : Compound->body()) {
4341 if (const auto *FE = dyn_cast<FullExpr>(C))
4342 enterFullExpression(FE);
4343 }
4344 }
4345
4346 auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
4347 PrePostActionTy &) {
4348 CGF.EmitStopPoint(CS);
4349 emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
4350 S.getV(), S.getExpr(), S.getUpdateExpr(),
4351 S.isXLHSInRHSPart(), S.getBeginLoc());
4352 };
4353 OMPLexicalScope Scope(*this, S, OMPD_unknown);
4354 CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
4355 }
4356
4357 static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
4358 const OMPExecutableDirective &S,
4359 const RegionCodeGenTy &CodeGen) {
4360 assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
4361 CodeGenModule &CGM = CGF.CGM;
4362
4363 // On device emit this construct as inlined code.
4364 if (CGM.getLangOpts().OpenMPIsDevice) {
4365 OMPLexicalScope Scope(CGF, S, OMPD_target);
4366 CGM.getOpenMPRuntime().emitInlinedDirective(
4367 CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4368 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4369 });
4370 return;
4371 }
4372
4373 llvm::Function *Fn = nullptr;
4374 llvm::Constant *FnID = nullptr;
4375
4376 const Expr *IfCond = nullptr;
4377 // Check for the at most one if clause associated with the target region.
4378 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4379 if (C->getNameModifier() == OMPD_unknown ||
4380 C->getNameModifier() == OMPD_target) {
4381 IfCond = C->getCondition();
4382 break;
4383 }
4384 }
4385
4386 // Check if we have any device clause associated with the directive.
4387 const Expr *Device = nullptr;
4388 if (auto *C = S.getSingleClause<OMPDeviceClause>())
4389 Device = C->getDevice();
4390
4391 // Check if we have an if clause whose conditional always evaluates to false
4392 // or if we do not have any targets specified. If so the target region is not
4393 // an offload entry point.
4394 bool IsOffloadEntry = true;
4395 if (IfCond) {
4396 bool Val;
4397 if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
4398 IsOffloadEntry = false;
4399 }
4400 if (CGM.getLangOpts().OMPTargetTriples.empty())
4401 IsOffloadEntry = false;
4402
4403 assert(CGF.CurFuncDecl && "No parent declaration for target region!");
4404 StringRef ParentName;
4405 // In case we have Ctors/Dtors we use the complete type variant to produce
4406 // the mangling of the device outlined kernel.
4407 if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
4408 ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
4409 else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
4410 ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
4411 else
4412 ParentName =
4413 CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
4414
4415 // Emit target region as a standalone region.
4416 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4417 IsOffloadEntry, CodeGen);
4418 OMPLexicalScope Scope(CGF, S, OMPD_task);
4419 auto &&SizeEmitter =
4420 [IsOffloadEntry](CodeGenFunction &CGF,
4421 const OMPLoopDirective &D) -> llvm::Value * {
4422 if (IsOffloadEntry) {
4423 OMPLoopScope(CGF, D);
4424 // Emit calculation of the iterations count.
4425 llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
4426 NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
4427 /*isSigned=*/false);
4428 return NumIterations;
4429 }
4430 return nullptr;
4431 };
4432 CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
4433 SizeEmitter);
4434 }
4435
4436 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
4437 PrePostActionTy &Action) {
4438 Action.Enter(CGF);
4439 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4440 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4441 CGF.EmitOMPPrivateClause(S, PrivateScope);
4442 (void)PrivateScope.Privatize();
4443 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4444 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4445
4446 CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4447 }
4448
4449 void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
4450 StringRef ParentName,
4451 const OMPTargetDirective &S) {
4452 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4453 emitTargetRegion(CGF, S, Action);
4454 };
4455 llvm::Function *Fn;
4456 llvm::Constant *Addr;
4457 // Emit target region as a standalone region.
4458 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4459 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4460 assert(Fn && Addr && "Target device function emission failed.");
4461 }
4462
4463 void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
4464 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4465 emitTargetRegion(CGF, S, Action);
4466 };
4467 emitCommonOMPTargetDirective(*this, S, CodeGen);
4468 }
4469
4470 static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
4471 const OMPExecutableDirective &S,
4472 OpenMPDirectiveKind InnermostKind,
4473 const RegionCodeGenTy &CodeGen) {
4474 const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4475 llvm::Function *OutlinedFn =
4476 CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4477 S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4478
4479 const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4480 const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4481 if (NT || TL) {
4482 const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4483 const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
4484
4485 CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4486 S.getBeginLoc());
4487 }
4488
4489 OMPTeamsScope Scope(CGF, S);
4490 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
4491 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4492 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
4493 CapturedVars);
4494 }
4495
4496 void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
4497 // Emit teams region as a standalone region.
4498 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4499 Action.Enter(CGF);
4500 OMPPrivateScope PrivateScope(CGF);
4501 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4502 CGF.EmitOMPPrivateClause(S, PrivateScope);
4503 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4504 (void)PrivateScope.Privatize();
4505 CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4506 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4507 };
4508 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4509 emitPostUpdateForReductionClause(*this, S,
4510 [](CodeGenFunction &) { return nullptr; });
4511 }
4512
4513 static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
4514 const OMPTargetTeamsDirective &S) {
4515 auto *CS = S.getCapturedStmt(OMPD_teams);
4516 Action.Enter(CGF);
4517 // Emit teams region as a standalone region.
4518 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4519 Action.Enter(CGF);
4520 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4521 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4522 CGF.EmitOMPPrivateClause(S, PrivateScope);
4523 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4524 (void)PrivateScope.Privatize();
4525 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
4526 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4527 CGF.EmitStmt(CS->getCapturedStmt());
4528 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4529 };
4530 emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4531 emitPostUpdateForReductionClause(CGF, S,
4532 [](CodeGenFunction &) { return nullptr; });
4533 }
4534
4535 void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
4536 CodeGenModule &CGM, StringRef ParentName,
4537 const OMPTargetTeamsDirective &S) {
4538 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4539 emitTargetTeamsRegion(CGF, Action, S);
4540 };
4541 llvm::Function *Fn;
4542 llvm::Constant *Addr;
4543 // Emit target region as a standalone region.
4544 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4545 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4546 assert(Fn && Addr && "Target device function emission failed.");
4547 }
4548
4549 void CodeGenFunction::EmitOMPTargetTeamsDirective(
4550 const OMPTargetTeamsDirective &S) {
4551 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4552 emitTargetTeamsRegion(CGF, Action, S);
4553 };
4554 emitCommonOMPTargetDirective(*this, S, CodeGen);
4555 }
4556
4557 static void
4558 emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
4559 const OMPTargetTeamsDistributeDirective &S) {
4560 Action.Enter(CGF);
4561 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4562 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4563 };
4564
4565 // Emit teams region as a standalone region.
4566 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4567 PrePostActionTy &Action) {
4568 Action.Enter(CGF);
4569 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4570 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4571 (void)PrivateScope.Privatize();
4572 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4573 CodeGenDistribute);
4574 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4575 };
4576 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4577 emitPostUpdateForReductionClause(CGF, S,
4578 [](CodeGenFunction &) { return nullptr; });
4579 }
4580
4581 void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
4582 CodeGenModule &CGM, StringRef ParentName,
4583 const OMPTargetTeamsDistributeDirective &S) {
4584 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4585 emitTargetTeamsDistributeRegion(CGF, Action, S);
4586 };
4587 llvm::Function *Fn;
4588 llvm::Constant *Addr;
4589 // Emit target region as a standalone region.
4590 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4591 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4592 assert(Fn && Addr && "Target device function emission failed.");
4593 }
4594
4595 void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
4596 const OMPTargetTeamsDistributeDirective &S) {
4597 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4598 emitTargetTeamsDistributeRegion(CGF, Action, S);
4599 };
4600 emitCommonOMPTargetDirective(*this, S, CodeGen);
4601 }
4602
4603 static void emitTargetTeamsDistributeSimdRegion(
4604 CodeGenFunction &CGF, PrePostActionTy &Action,
4605 const OMPTargetTeamsDistributeSimdDirective &S) {
4606 Action.Enter(CGF);
4607 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4608 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4609 };
4610
4611 // Emit teams region as a standalone region.
4612 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4613 PrePostActionTy &Action) {
4614 Action.Enter(CGF);
4615 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4616 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4617 (void)PrivateScope.Privatize();
4618 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4619 CodeGenDistribute);
4620 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4621 };
4622 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4623 emitPostUpdateForReductionClause(CGF, S,
4624 [](CodeGenFunction &) { return nullptr; });
4625 }
4626
4627 void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
4628 CodeGenModule &CGM, StringRef ParentName,
4629 const OMPTargetTeamsDistributeSimdDirective &S) {
4630 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4631 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4632 };
4633 llvm::Function *Fn;
4634 llvm::Constant *Addr;
4635 // Emit target region as a standalone region.
4636 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4637 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4638 assert(Fn && Addr && "Target device function emission failed.");
4639 }
4640
4641 void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
4642 const OMPTargetTeamsDistributeSimdDirective &S) {
4643 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4644 emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4645 };
4646 emitCommonOMPTargetDirective(*this, S, CodeGen);
4647 }
4648
4649 void CodeGenFunction::EmitOMPTeamsDistributeDirective(
4650 const OMPTeamsDistributeDirective &S) {
4651
4652 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4653 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4654 };
4655
4656 // Emit teams region as a standalone region.
4657 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4658 PrePostActionTy &Action) {
4659 Action.Enter(CGF);
4660 OMPPrivateScope PrivateScope(CGF);
4661 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4662 (void)PrivateScope.Privatize();
4663 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4664 CodeGenDistribute);
4665 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4666 };
4667 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4668 emitPostUpdateForReductionClause(*this, S,
4669 [](CodeGenFunction &) { return nullptr; });
4670 }
4671
4672 void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
4673 const OMPTeamsDistributeSimdDirective &S) {
4674 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4675 CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
4676 };
4677
4678 // Emit teams region as a standalone region.
4679 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4680 PrePostActionTy &Action) {
4681 Action.Enter(CGF);
4682 OMPPrivateScope PrivateScope(CGF);
4683 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4684 (void)PrivateScope.Privatize();
4685 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4686 CodeGenDistribute);
4687 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4688 };
4689 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4690 emitPostUpdateForReductionClause(*this, S,
4691 [](CodeGenFunction &) { return nullptr; });
4692 }
4693
4694 void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
4695 const OMPTeamsDistributeParallelForDirective &S) {
4696 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4697 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4698 S.getDistInc());
4699 };
4700
4701 // Emit teams region as a standalone region.
4702 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4703 PrePostActionTy &Action) {
4704 Action.Enter(CGF);
4705 OMPPrivateScope PrivateScope(CGF);
4706 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4707 (void)PrivateScope.Privatize();
4708 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4709 CodeGenDistribute);
4710 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4711 };
4712 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4713 emitPostUpdateForReductionClause(*this, S,
4714 [](CodeGenFunction &) { return nullptr; });
4715 }
4716
4717 void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
4718 const OMPTeamsDistributeParallelForSimdDirective &S) {
4719 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4720 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4721 S.getDistInc());
4722 };
4723
4724 // Emit teams region as a standalone region.
4725 auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4726 PrePostActionTy &Action) {
4727 Action.Enter(CGF);
4728 OMPPrivateScope PrivateScope(CGF);
4729 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4730 (void)PrivateScope.Privatize();
4731 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4732 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4733 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4734 };
4735 emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for_simd,
4736 CodeGen);
4737 emitPostUpdateForReductionClause(*this, S,
4738 [](CodeGenFunction &) { return nullptr; });
4739 }
4740
4741 static void emitTargetTeamsDistributeParallelForRegion(
4742 CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,
4743 PrePostActionTy &Action) {
4744 Action.Enter(CGF);
4745 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4746 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4747 S.getDistInc());
4748 };
4749
4750 // Emit teams region as a standalone region.
4751 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4752 PrePostActionTy &Action) {
4753 Action.Enter(CGF);
4754 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4755 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4756 (void)PrivateScope.Privatize();
4757 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4758 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4759 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4760 };
4761
4762 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4763 CodeGenTeams);
4764 emitPostUpdateForReductionClause(CGF, S,
4765 [](CodeGenFunction &) { return nullptr; });
4766 }
4767
4768 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
4769 CodeGenModule &CGM, StringRef ParentName,
4770 const OMPTargetTeamsDistributeParallelForDirective &S) {
4771 // Emit SPMD target teams distribute parallel for region as a standalone
4772 // region.
4773 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4774 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
4775 };
4776 llvm::Function *Fn;
4777 llvm::Constant *Addr;
4778 // Emit target region as a standalone region.
4779 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4780 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4781 assert(Fn && Addr && "Target device function emission failed.");
4782 }
4783
4784 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
4785 const OMPTargetTeamsDistributeParallelForDirective &S) {
4786 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4787 emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
4788 };
4789 emitCommonOMPTargetDirective(*this, S, CodeGen);
4790 }
4791
4792 static void emitTargetTeamsDistributeParallelForSimdRegion(
4793 CodeGenFunction &CGF,
4794 const OMPTargetTeamsDistributeParallelForSimdDirective &S,
4795 PrePostActionTy &Action) {
4796 Action.Enter(CGF);
4797 auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4798 CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
4799 S.getDistInc());
4800 };
4801
4802 // Emit teams region as a standalone region.
4803 auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4804 PrePostActionTy &Action) {
4805 Action.Enter(CGF);
4806 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4807 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4808 (void)PrivateScope.Privatize();
4809 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4810 CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4811 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4812 };
4813
4814 emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4815 CodeGenTeams);
4816 emitPostUpdateForReductionClause(CGF, S,
4817 [](CodeGenFunction &) { return nullptr; });
4818 }
4819
4820 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
4821 CodeGenModule &CGM, StringRef ParentName,
4822 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
4823 // Emit SPMD target teams distribute parallel for simd region as a standalone
4824 // region.
4825 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4826 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
4827 };
4828 llvm::Function *Fn;
4829 llvm::Constant *Addr;
4830 // Emit target region as a standalone region.
4831 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4832 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4833 assert(Fn && Addr && "Target device function emission failed.");
4834 }
4835
4836 void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
4837 const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
4838 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4839 emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
4840 };
4841 emitCommonOMPTargetDirective(*this, S, CodeGen);
4842 }
4843
4844 void CodeGenFunction::EmitOMPCancellationPointDirective(
4845 const OMPCancellationPointDirective &S) {
4846 CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
4847 S.getCancelRegion());
4848 }
4849
4850 void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
4851 const Expr *IfCond = nullptr;
4852 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4853 if (C->getNameModifier() == OMPD_unknown ||
4854 C->getNameModifier() == OMPD_cancel) {
4855 IfCond = C->getCondition();
4856 break;
4857 }
4858 }
4859 if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder()) {
4860 // TODO: This check is necessary as we only generate `omp parallel` through
4861 // the OpenMPIRBuilder for now.
4862 if (S.getCancelRegion() == OMPD_parallel) {
4863 llvm::Value *IfCondition = nullptr;
4864 if (IfCond)
4865 IfCondition = EmitScalarExpr(IfCond,
4866 /*IgnoreResultAssign=*/true);
4867 return Builder.restoreIP(
4868 OMPBuilder->CreateCancel(Builder, IfCondition, S.getCancelRegion()));
4869 }
4870 }
4871
4872 CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
4873 S.getCancelRegion());
4874 }
4875
4876 CodeGenFunction::JumpDest
4877 CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
4878 if (Kind == OMPD_parallel || Kind == OMPD_task ||
4879 Kind == OMPD_target_parallel)
4880 return ReturnBlock;
4881 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4882 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4883 Kind == OMPD_distribute_parallel_for ||
4884 Kind == OMPD_target_parallel_for ||
4885 Kind == OMPD_teams_distribute_parallel_for ||
4886 Kind == OMPD_target_teams_distribute_parallel_for);
4887 return OMPCancelStack.getExitBlock();
4888 }
4889
4890 void CodeGenFunction::EmitOMPUseDevicePtrClause(
4891 const OMPClause &NC, OMPPrivateScope &PrivateScope,
4892 const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4893 const auto &C = cast<OMPUseDevicePtrClause>(NC);
4894 auto OrigVarIt = C.varlist_begin();
4895 auto InitIt = C.inits().begin();
4896 for (const Expr *PvtVarIt : C.private_copies()) {
4897 const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4898 const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4899 const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4900
4901 // In order to identify the right initializer we need to match the
4902 // declaration used by the mapping logic. In some cases we may get
4903 // OMPCapturedExprDecl that refers to the original declaration.
4904 const ValueDecl *MatchingVD = OrigVD;
4905 if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4906 // OMPCapturedExprDecl are used to privative fields of the current
4907 // structure.
4908 const auto *ME = cast<MemberExpr>(OED->getInit());
4909 assert(isa<CXXThisExpr>(ME->getBase()) &&
4910 "Base should be the current struct!");
4911 MatchingVD = ME->getMemberDecl();
4912 }
4913
4914 // If we don't have information about the current list item, move on to
4915 // the next one.
4916 auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4917 if (InitAddrIt == CaptureDeviceAddrMap.end())
4918 continue;
4919
4920 bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4921 InitAddrIt, InitVD,
4922 PvtVD]() {
4923 // Initialize the temporary initialization variable with the address we
4924 // get from the runtime library. We have to cast the source address
4925 // because it is always a void *. References are materialized in the
4926 // privatization scope, so the initialization here disregards the fact
4927 // the original variable is a reference.
4928 QualType AddrQTy =
4929 getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4930 llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4931 Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4932 setAddrOfLocalVar(InitVD, InitAddr);
4933
4934 // Emit private declaration, it will be initialized by the value we
4935 // declaration we just added to the local declarations map.
4936 EmitDecl(*PvtVD);
4937
4938 // The initialization variables reached its purpose in the emission
4939 // of the previous declaration, so we don't need it anymore.
4940 LocalDeclMap.erase(InitVD);
4941
4942 // Return the address of the private variable.
4943 return GetAddrOfLocalVar(PvtVD);
4944 });
4945 assert(IsRegistered && "firstprivate var already registered as private");
4946 // Silence the warning about unused variable.
4947 (void)IsRegistered;
4948
4949 ++OrigVarIt;
4950 ++InitIt;
4951 }
4952 }
4953
4954 // Generate the instructions for '#pragma omp target data' directive.
4955 void CodeGenFunction::EmitOMPTargetDataDirective(
4956 const OMPTargetDataDirective &S) {
4957 CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4958
4959 // Create a pre/post action to signal the privatization of the device pointer.
4960 // This action can be replaced by the OpenMP runtime code generation to
4961 // deactivate privatization.
4962 bool PrivatizeDevicePointers = false;
4963 class DevicePointerPrivActionTy : public PrePostActionTy {
4964 bool &PrivatizeDevicePointers;
4965
4966 public:
4967 explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4968 : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4969 void Enter(CodeGenFunction &CGF) override {
4970 PrivatizeDevicePointers = true;
4971 }
4972 };
4973 DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4974
4975 auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4976 CodeGenFunction &CGF, PrePostActionTy &Action) {
4977 auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4978 CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
4979 };
4980
4981 // Codegen that selects whether to generate the privatization code or not.
4982 auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4983 &InnermostCodeGen](CodeGenFunction &CGF,
4984 PrePostActionTy &Action) {
4985 RegionCodeGenTy RCG(InnermostCodeGen);
4986 PrivatizeDevicePointers = false;
4987
4988 // Call the pre-action to change the status of PrivatizeDevicePointers if
4989 // needed.
4990 Action.Enter(CGF);
4991
4992 if (PrivatizeDevicePointers) {
4993 OMPPrivateScope PrivateScope(CGF);
4994 // Emit all instances of the use_device_ptr clause.
4995 for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4996 CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4997 Info.CaptureDeviceAddrMap);
4998 (void)PrivateScope.Privatize();
4999 RCG(CGF);
5000 } else {
5001 RCG(CGF);
5002 }
5003 };
5004
5005 // Forward the provided action to the privatization codegen.
5006 RegionCodeGenTy PrivRCG(PrivCodeGen);
5007 PrivRCG.setAction(Action);
5008
5009 // Notwithstanding the body of the region is emitted as inlined directive,
5010 // we don't use an inline scope as changes in the references inside the
5011 // region are expected to be visible outside, so we do not privative them.
5012 OMPLexicalScope Scope(CGF, S);
5013 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
5014 PrivRCG);
5015 };
5016
5017 RegionCodeGenTy RCG(CodeGen);
5018
5019 // If we don't have target devices, don't bother emitting the data mapping
5020 // code.
5021 if (CGM.getLangOpts().OMPTargetTriples.empty()) {
5022 RCG(*this);
5023 return;
5024 }
5025
5026 // Check if we have any if clause associated with the directive.
5027 const Expr *IfCond = nullptr;
5028 if (const auto *C = S.getSingleClause<OMPIfClause>())
5029 IfCond = C->getCondition();
5030
5031 // Check if we have any device clause associated with the directive.
5032 const Expr *Device = nullptr;
5033 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
5034 Device = C->getDevice();
5035
5036 // Set the action to signal privatization of device pointers.
5037 RCG.setAction(PrivAction);
5038
5039 // Emit region code.
5040 CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
5041 Info);
5042 }
5043
5044 void CodeGenFunction::EmitOMPTargetEnterDataDirective(
5045 const OMPTargetEnterDataDirective &S) {
5046 // If we don't have target devices, don't bother emitting the data mapping
5047 // code.
5048 if (CGM.getLangOpts().OMPTargetTriples.empty())
5049 return;
5050
5051 // Check if we have any if clause associated with the directive.
5052 const Expr *IfCond = nullptr;
5053 if (const auto *C = S.getSingleClause<OMPIfClause>())
5054 IfCond = C->getCondition();
5055
5056 // Check if we have any device clause associated with the directive.
5057 const Expr *Device = nullptr;
5058 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
5059 Device = C->getDevice();
5060
5061 OMPLexicalScope Scope(*this, S, OMPD_task);
5062 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
5063 }
5064
5065 void CodeGenFunction::EmitOMPTargetExitDataDirective(
5066 const OMPTargetExitDataDirective &S) {
5067 // If we don't have target devices, don't bother emitting the data mapping
5068 // code.
5069 if (CGM.getLangOpts().OMPTargetTriples.empty())
5070 return;
5071
5072 // Check if we have any if clause associated with the directive.
5073 const Expr *IfCond = nullptr;
5074 if (const auto *C = S.getSingleClause<OMPIfClause>())
5075 IfCond = C->getCondition();
5076
5077 // Check if we have any device clause associated with the directive.
5078 const Expr *Device = nullptr;
5079 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
5080 Device = C->getDevice();
5081
5082 OMPLexicalScope Scope(*this, S, OMPD_task);
5083 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
5084 }
5085
5086 static void emitTargetParallelRegion(CodeGenFunction &CGF,
5087 const OMPTargetParallelDirective &S,
5088 PrePostActionTy &Action) {
5089 // Get the captured statement associated with the 'parallel' region.
5090 const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
5091 Action.Enter(CGF);
5092 auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
5093 Action.Enter(CGF);
5094 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5095 (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
5096 CGF.EmitOMPPrivateClause(S, PrivateScope);
5097 CGF.EmitOMPReductionClauseInit(S, PrivateScope);
5098 (void)PrivateScope.Privatize();
5099 if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
5100 CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
5101 // TODO: Add support for clauses.
5102 CGF.EmitStmt(CS->getCapturedStmt());
5103 CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
5104 };
5105 emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
5106 emitEmptyBoundParameters);
5107 emitPostUpdateForReductionClause(CGF, S,
5108 [](CodeGenFunction &) { return nullptr; });
5109 }
5110
5111 void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
5112 CodeGenModule &CGM, StringRef ParentName,
5113 const OMPTargetParallelDirective &S) {
5114 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5115 emitTargetParallelRegion(CGF, S, Action);
5116 };
5117 llvm::Function *Fn;
5118 llvm::Constant *Addr;
5119 // Emit target region as a standalone region.
5120 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
5121 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
5122 assert(Fn && Addr && "Target device function emission failed.");
5123 }
5124
5125 void CodeGenFunction::EmitOMPTargetParallelDirective(
5126 const OMPTargetParallelDirective &S) {
5127 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5128 emitTargetParallelRegion(CGF, S, Action);
5129 };
5130 emitCommonOMPTargetDirective(*this, S, CodeGen);
5131 }
5132
5133 static void emitTargetParallelForRegion(CodeGenFunction &CGF,
5134 const OMPTargetParallelForDirective &S,
5135 PrePostActionTy &Action) {
5136 Action.Enter(CGF);
5137 // Emit directive as a combined directive that consists of two implicit
5138 // directives: 'parallel' with 'for' directive.
5139 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5140 Action.Enter(CGF);
5141 CodeGenFunction::OMPCancelStackRAII CancelRegion(
5142 CGF, OMPD_target_parallel_for, S.hasCancel());
5143 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
5144 emitDispatchForLoopBounds);
5145 };
5146 emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
5147 emitEmptyBoundParameters);
5148 }
5149
5150 void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
5151 CodeGenModule &CGM, StringRef ParentName,
5152 const OMPTargetParallelForDirective &S) {
5153 // Emit SPMD target parallel for region as a standalone region.
5154 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5155 emitTargetParallelForRegion(CGF, S, Action);
5156 };
5157 llvm::Function *Fn;
5158 llvm::Constant *Addr;
5159 // Emit target region as a standalone region.
5160 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
5161 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
5162 assert(Fn && Addr && "Target device function emission failed.");
5163 }
5164
5165 void CodeGenFunction::EmitOMPTargetParallelForDirective(
5166 const OMPTargetParallelForDirective &S) {
5167 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5168 emitTargetParallelForRegion(CGF, S, Action);
5169 };
5170 emitCommonOMPTargetDirective(*this, S, CodeGen);
5171 }
5172
5173 static void
5174 emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
5175 const OMPTargetParallelForSimdDirective &S,
5176 PrePostActionTy &Action) {
5177 Action.Enter(CGF);
5178 // Emit directive as a combined directive that consists of two implicit
5179 // directives: 'parallel' with 'for' directive.
5180 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5181 Action.Enter(CGF);
5182 CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
5183 emitDispatchForLoopBounds);
5184 };
5185 emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
5186 emitEmptyBoundParameters);
5187 }
5188
5189 void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
5190 CodeGenModule &CGM, StringRef ParentName,
5191 const OMPTargetParallelForSimdDirective &S) {
5192 // Emit SPMD target parallel for region as a standalone region.
5193 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5194 emitTargetParallelForSimdRegion(CGF, S, Action);
5195 };
5196 llvm::Function *Fn;
5197 llvm::Constant *Addr;
5198 // Emit target region as a standalone region.
5199 CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
5200 S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
5201 assert(Fn && Addr && "Target device function emission failed.");
5202 }
5203
5204 void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
5205 const OMPTargetParallelForSimdDirective &S) {
5206 auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5207 emitTargetParallelForSimdRegion(CGF, S, Action);
5208 };
5209 emitCommonOMPTargetDirective(*this, S, CodeGen);
5210 }
5211
5212 /// Emit a helper variable and return corresponding lvalue.
5213 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
5214 const ImplicitParamDecl *PVD,
5215 CodeGenFunction::OMPPrivateScope &Privates) {
5216 const auto *VDecl = cast<VarDecl>(Helper->getDecl());
5217 Privates.addPrivate(VDecl,
5218 [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
5219 }
5220
5221 void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
5222 assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
5223 // Emit outlined function for task construct.
5224 const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
5225 Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
5226 QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
5227 const Expr *IfCond = nullptr;
5228 for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
5229 if (C->getNameModifier() == OMPD_unknown ||
5230 C->getNameModifier() == OMPD_taskloop) {
5231 IfCond = C->getCondition();
5232 break;
5233 }
5234 }
5235
5236 OMPTaskDataTy Data;
5237 // Check if taskloop must be emitted without taskgroup.
5238 Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
5239 // TODO: Check if we should emit tied or untied task.
5240 Data.Tied = true;
5241 // Set scheduling for taskloop
5242 if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
5243 // grainsize clause
5244 Data.Schedule.setInt(/*IntVal=*/false);
5245 Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
5246 } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
5247 // num_tasks clause
5248 Data.Schedule.setInt(/*IntVal=*/true);
5249 Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
5250 }
5251
5252 auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
5253 // if (PreCond) {
5254 // for (IV in 0..LastIteration) BODY;
5255 // <Final counter/linear vars updates>;
5256 // }
5257 //
5258
5259 // Emit: if (PreCond) - begin.
5260 // If the condition constant folds and can be elided, avoid emitting the
5261 // whole loop.
5262 bool CondConstant;
5263 llvm::BasicBlock *ContBlock = nullptr;
5264 OMPLoopScope PreInitScope(CGF, S);
5265 if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
5266 if (!CondConstant)
5267 return;
5268 } else {
5269 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
5270 ContBlock = CGF.createBasicBlock("taskloop.if.end");
5271 emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
5272 CGF.getProfileCount(&S));
5273 CGF.EmitBlock(ThenBlock);
5274 CGF.incrementProfileCounter(&S);
5275 }
5276
5277 (void)CGF.EmitOMPLinearClauseInit(S);
5278
5279 OMPPrivateScope LoopScope(CGF);
5280 // Emit helper vars inits.
5281 enum { LowerBound = 5, UpperBound, Stride, LastIter };
5282 auto *I = CS->getCapturedDecl()->param_begin();
5283 auto *LBP = std::next(I, LowerBound);
5284 auto *UBP = std::next(I, UpperBound);
5285 auto *STP = std::next(I, Stride);
5286 auto *LIP = std::next(I, LastIter);
5287 mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
5288 LoopScope);
5289 mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
5290 LoopScope);
5291 mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
5292 mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
5293 LoopScope);
5294 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
5295 CGF.EmitOMPLinearClause(S, LoopScope);
5296 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
5297 (void)LoopScope.Privatize();
5298 // Emit the loop iteration variable.
5299 const Expr *IVExpr = S.getIterationVariable();
5300 const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
5301 CGF.EmitVarDecl(*IVDecl);
5302 CGF.EmitIgnoredExpr(S.getInit());
5303
5304 // Emit the iterations count variable.
5305 // If it is not a variable, Sema decided to calculate iterations count on
5306 // each iteration (e.g., it is foldable into a constant).
5307 if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
5308 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
5309 // Emit calculation of the iterations count.
5310 CGF.EmitIgnoredExpr(S.getCalcLastIteration());
5311 }
5312
5313 {
5314 OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
5315 emitCommonSimdLoop(
5316 CGF, S,
5317 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
5318 if (isOpenMPSimdDirective(S.getDirectiveKind()))
5319 CGF.EmitOMPSimdInit(S);
5320 },
5321 [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
5322 CGF.EmitOMPInnerLoop(
5323 S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
5324 [&S](CodeGenFunction &CGF) {
5325 CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest());
5326 CGF.EmitStopPoint(&S);
5327 },
5328 [](CodeGenFunction &) {});
5329 });
5330 }
5331 // Emit: if (PreCond) - end.
5332 if (ContBlock) {
5333 CGF.EmitBranch(ContBlock);
5334 CGF.EmitBlock(ContBlock, true);
5335 }
5336 // Emit final copy of the lastprivate variables if IsLastIter != 0.
5337 if (HasLastprivateClause) {
5338 CGF.EmitOMPLastprivateClauseFinal(
5339 S, isOpenMPSimdDirective(S.getDirectiveKind()),
5340 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
5341 CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
5342 (*LIP)->getType(), S.getBeginLoc())));
5343 }
5344 CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {
5345 return CGF.Builder.CreateIsNotNull(
5346 CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
5347 (*LIP)->getType(), S.getBeginLoc()));
5348 });
5349 };
5350 auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
5351 IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
5352 const OMPTaskDataTy &Data) {
5353 auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
5354 &Data](CodeGenFunction &CGF, PrePostActionTy &) {
5355 OMPLoopScope PreInitScope(CGF, S);
5356 CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
5357 OutlinedFn, SharedsTy,
5358 CapturedStruct, IfCond, Data);
5359 };
5360 CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
5361 CodeGen);
5362 };
5363 if (Data.Nogroup) {
5364 EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
5365 } else {
5366 CGM.getOpenMPRuntime().emitTaskgroupRegion(
5367 *this,
5368 [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
5369 PrePostActionTy &Action) {
5370 Action.Enter(CGF);
5371 CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
5372 Data);
5373 },
5374 S.getBeginLoc());
5375 }
5376 }
5377
5378 void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
5379 EmitOMPTaskLoopBasedDirective(S);
5380 }
5381
5382 void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
5383 const OMPTaskLoopSimdDirective &S) {
5384 OMPLexicalScope Scope(*this, S);
5385 EmitOMPTaskLoopBasedDirective(S);
5386 }
5387
5388 void CodeGenFunction::EmitOMPMasterTaskLoopDirective(
5389 const OMPMasterTaskLoopDirective &S) {
5390 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5391 Action.Enter(CGF);
5392 EmitOMPTaskLoopBasedDirective(S);
5393 };
5394 OMPLexicalScope Scope(*this, S, llvm::None, /*EmitPreInitStmt=*/false);
5395 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
5396 }
5397
5398 void CodeGenFunction::EmitOMPMasterTaskLoopSimdDirective(
5399 const OMPMasterTaskLoopSimdDirective &S) {
5400 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5401 Action.Enter(CGF);
5402 EmitOMPTaskLoopBasedDirective(S);
5403 };
5404 OMPLexicalScope Scope(*this, S);
5405 CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
5406 }
5407
5408 void CodeGenFunction::EmitOMPParallelMasterTaskLoopDirective(
5409 const OMPParallelMasterTaskLoopDirective &S) {
5410 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5411 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
5412 PrePostActionTy &Action) {
5413 Action.Enter(CGF);
5414 CGF.EmitOMPTaskLoopBasedDirective(S);
5415 };
5416 OMPLexicalScope Scope(CGF, S, llvm::None, /*EmitPreInitStmt=*/false);
5417 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
5418 S.getBeginLoc());
5419 };
5420 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop, CodeGen,
5421 emitEmptyBoundParameters);
5422 }
5423
5424 void CodeGenFunction::EmitOMPParallelMasterTaskLoopSimdDirective(
5425 const OMPParallelMasterTaskLoopSimdDirective &S) {
5426 auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
5427 auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
5428 PrePostActionTy &Action) {
5429 Action.Enter(CGF);
5430 CGF.EmitOMPTaskLoopBasedDirective(S);
5431 };
5432 OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
5433 CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
5434 S.getBeginLoc());
5435 };
5436 emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop_simd, CodeGen,
5437 emitEmptyBoundParameters);
5438 }
5439
5440 // Generate the instructions for '#pragma omp target update' directive.
5441 void CodeGenFunction::EmitOMPTargetUpdateDirective(
5442 const OMPTargetUpdateDirective &S) {
5443 // If we don't have target devices, don't bother emitting the data mapping
5444 // code.
5445 if (CGM.getLangOpts().OMPTargetTriples.empty())
5446 return;
5447
5448 // Check if we have any if clause associated with the directive.
5449 const Expr *IfCond = nullptr;
5450 if (const auto *C = S.getSingleClause<OMPIfClause>())
5451 IfCond = C->getCondition();
5452
5453 // Check if we have any device clause associated with the directive.
5454 const Expr *Device = nullptr;
5455 if (const auto *C = S.getSingleClause<OMPDeviceClause>())
5456 Device = C->getDevice();
5457
5458 OMPLexicalScope Scope(*this, S, OMPD_task);
5459 CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
5460 }
5461
5462 void CodeGenFunction::EmitSimpleOMPExecutableDirective(
5463 const OMPExecutableDirective &D) {
5464 if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
5465 return;
5466 auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
5467 if (isOpenMPSimdDirective(D.getDirectiveKind())) {
5468 emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
5469 } else {
5470 OMPPrivateScope LoopGlobals(CGF);
5471 if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
5472 for (const Expr *E : LD->counters()) {
5473 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5474 if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
5475 LValue GlobLVal = CGF.EmitLValue(E);
5476 LoopGlobals.addPrivate(
5477 VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); });
5478 }
5479 if (isa<OMPCapturedExprDecl>(VD)) {
5480 // Emit only those that were not explicitly referenced in clauses.
5481 if (!CGF.LocalDeclMap.count(VD))
5482 CGF.EmitVarDecl(*VD);
5483 }
5484 }
5485 for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
5486 if (!C->getNumForLoops())
5487 continue;
5488 for (unsigned I = LD->getCollapsedNumber(),
5489 E = C->getLoopNumIterations().size();
5490 I < E; ++I) {
5491 if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
5492 cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
5493 // Emit only those that were not explicitly referenced in clauses.
5494 if (!CGF.LocalDeclMap.count(VD))
5495 CGF.EmitVarDecl(*VD);
5496 }
5497 }
5498 }
5499 }
5500 LoopGlobals.Privatize();
5501 CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());
5502 }
5503 };
5504 OMPSimdLexicalScope Scope(*this, D);
5505 CGM.getOpenMPRuntime().emitInlinedDirective(
5506 *this,
5507 isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
5508 : D.getDirectiveKind(),
5509 CodeGen);
5510 }
clang::CodeGen::CodeGenTypes::arrangeBuiltinFunctionDeclaration
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:653
clang::CodeGen::RValue::isAggregate
bool isAggregate() const
Definition: CGValue.h:54
clang::OMPParallelDirective::hasCancel
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:410
clang::CodeGen::CodeGenFunction::EmitOMPMasterTaskLoopDirective
void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S)
Definition: CGStmtOpenMP.cpp:5388
clang::Builtin::ID
ID
Definition: Builtins.h:46
ASTContext.h
Defines the clang::ASTContext interface.
clang::OMPDistributeSimdDirective
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3757
clang::OMPLoopDirective::getNextUpperBound
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:994
clang::OMPMasterDirective
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1591
clang::OMPTaskDirective
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1986
getCanonicalDecl
static const Decl * getCanonicalDecl(const Decl *D)
Definition: IndexingContext.cpp:292
clang::FunctionDecl
Represents a function declaration or definition.
Definition: Decl.h:1783
OMPThreadLimitClause
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:5164
clang::CodeGen::CodeGenFunction::VlaSizePair
Definition: CodeGenFunction.h:2434
clang::OMPLoopDirective::getUpperBoundVariable
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:962
clang::ImplicitParamDecl::Other
Other implicit parameter.
Definition: Decl.h:1555
clang::CodeGen::CodeGenFunction::EmitOMPTaskwaitDirective
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
Definition: CGStmtOpenMP.cpp:3552
clang::CodeGen::CodeGenFunction::ConstantEmission::getReferenceLValue
LValue getReferenceLValue(CodeGenFunction &CGF, Expr *refExpr) const
Definition: CodeGenFunction.h:3558
clang::CodeGen::CodeGenFunction::EmitOMPTargetTaskBasedDirective
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
Definition: CGStmtOpenMP.cpp:3396
emitOMPAtomicWriteExpr
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3975
clang::CodeGen::OMPTaskDataTy
Definition: CGOpenMPRuntime.h:93
OMPCopyinClause
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:3541
clang::Ctor_Complete
Complete object ctor.
Definition: ABI.h:25
clang::PointerType
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2614
clang::OpenMPScheduleTy
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:198
clang::QualType
A (possibly-)qualified type.
Definition: Type.h:654
emitMaster
static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Definition: CGStmtOpenMP.cpp:3012
clang::CapturedStmt::capture_init_begin
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:3550
clang::CodeGen::CodeGenModule::getTypes
CodeGenTypes & getTypes()
Definition: CodeGenModule.h:698
clang::Type::isArrayType
bool isArrayType() const
Definition: Type.h:6570
clang::OMPExecutableDirective::clauses
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:325
clang::CodeGen::CodeGenFunction::ConvertTypeForMem
llvm::Type * ConvertTypeForMem(QualType T)
Definition: CodeGenFunction.cpp:229
clang::CodeGen::CodeGenFunction::EmitBranchOnBoolExpr
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
Definition: CodeGenFunction.cpp:1470
emitOMPAtomicReadExpr
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3951
clang::CodeGen::CodeGenFunction::EmitVarDecl
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:168
clang::CapturedDecl::param_begin
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:4306
clang::CodeGen::CodeGenFunction::CreateMemTemp
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:139
emitBody
static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop, int MaxLevel, int Level=0)
Definition: CGStmtOpenMP.cpp:1445
clang::OMPLoopDirective::dependent_counters
ArrayRef< Expr * > dependent_counters()
Definition: StmtOpenMP.h:1134
clang::CodeGen::CodeGenFunction::HaveInsertPoint
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
Definition: CodeGenFunction.h:2112
clang::CodeGen::CodeGenFunction::EmitOMPDistributeDirective
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
Definition: CGStmtOpenMP.cpp:3836
clang::OMPC_match
Definition: OpenMPKinds.h:72
clang::CodeGen::CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
Definition: CGStmtOpenMP.cpp:5189
clang::diff::Update
Definition: ASTDiff.h:30
llvm
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
Definition: Dominators.h:30
emitDistributeParallelForInnerBounds
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
Definition: CGStmtOpenMP.cpp:2322
clang::CodeGen::CGOpenMPRuntime::LastprivateConditionalRAII
Manages list of lastprivate conditional decls for the specified directive.
Definition: CGOpenMPRuntime.h:243
clang::OMPClauseWithPreInit::get
static OMPClauseWithPreInit * get(OMPClause *C)
Definition: OpenMPClause.cpp:55
clang::Stmt
Stmt - This represents one statement.
Definition: Stmt.h:66
clang::OMPInReductionClause
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Definition: OpenMPClause.h:2910
clang::OMPLoopDirective::getLowerBoundVariable
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:954
clang::CodeGen::CodeGenFunction::EmitStoreThroughGlobalRegLValue
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2165
clang::Type::getPointeeType
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:557
clang::CodeGen::CodeGenFunction::OMPTargetDataInfo
Definition: CodeGenFunction.h:3118
clang::CodeGen::CodeGenFunction::RunCleanupsScope::requiresCleanups
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
Definition: CodeGenFunction.h:744
clang::CodeGen::CodeGenFunction::EmitOMPTargetSimdDirective
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
Definition: CGStmtOpenMP.cpp:2475
clang::CodeGen::CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
Definition: CGStmtOpenMP.cpp:4083
clang::CodeGen::OMPTaskDataTy::Dependences
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
Definition: CGOpenMPRuntime.h:104
clang::OMPForSimdDirective
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1337
clang::CodeGen::OMPTaskDataTy::LastprivateCopies
SmallVector< const Expr *, 4 > LastprivateCopies
Definition: CGOpenMPRuntime.h:100
clang::ASTContext::getLValueReferenceType
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Definition: ASTContext.cpp:3222
clang::Decl
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:88
OMPGrainsizeClause
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:5314
clang::OMPTeamsDistributeParallelForDirective
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:4171
clang::OMPExecutableDirective::getBeginLoc
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:225
PrettyStackTrace.h
Defines the PrettyStackTraceEntry class, which is used to make crashes give more contextual informati...
clang::CapturedDecl::isNothrow
bool isNothrow() const
Definition: Decl.cpp:4751
clang::CodeGen::CodeGenFunction::CurCodeDecl
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
Definition: CodeGenFunction.h:298
emitOMPAtomicUpdateExpr
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:4107
clang::CodeGen::CodeGenFunction::getTypeSize
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
Definition: CGStmtOpenMP.cpp:271
clang::OMPIfClause
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:425
llvm::DenseSet
Definition: Sema.h:71
clang::OMPParallelMasterDirective
This represents &#39;#pragma omp parallel master&#39; directive.
Definition: StmtOpenMP.h:1863
clang::OMPExecutableDirective::getInnermostCapturedStmt
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
Definition: StmtOpenMP.h:283
clang::QualType::getNonReferenceType
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:6452
clang::CodeGen::CodeGenFunction::EmitOMPOrderedDirective
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
Definition: CGStmtOpenMP.cpp:3855
clang::CodeGen::LValue::isVolatile
bool isVolatile() const
Definition: CGValue.h:301
clang::IntegerLiteral
Definition: Expr.h:1446
OMPPriorityClause
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:5240
mapParam
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
Definition: CGStmtOpenMP.cpp:5213
clang::Type
The base class of the type hierarchy.
Definition: Type.h:1450
clang::OMPTargetTeamsDistributeDirective
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:4310
clang::OMPC_DIST_SCHEDULE_unknown
Definition: OpenMPKinds.h:154
clang::CodeGen::CodeGenFunction::EmitStoreThroughLValue
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1929
emitOMPAtomicExpr
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:4244
clang::CapturedStmt::getCapturedRecordDecl
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:3494
clang::ArrayType
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition: Type.h:2889
clang::CodeGen::CodeGenFunction::CGCapturedStmtInfo::lookup
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
Definition: CodeGenFunction.h:378
clang::ASTContext::getTargetInfo
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:707
clang::CodeGen::AlignmentSource::Decl
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
clang::CodeGen::CodeGenFunction::TaskGenTy
const llvm::function_ref< void(CodeGenFunction &, llvm::Function *, const OMPTaskDataTy &)> TaskGenTy
Definition: CodeGenFunction.h:3113
emitCommonOMPParallelDirective
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Definition: CGStmtOpenMP.cpp:1291
clang::CodeGen::CodeGenFunction::EmitLoadOfPointer
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition: CGExpr.cpp:2345
clang::FPOptions
Floating point control options.
Definition: LangOptions.h:357
clang::FunctionDecl::Create
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, ConstexprSpecKind ConstexprKind=CSK_unspecified, Expr *TrailingRequiresClause=nullptr)
Definition: Decl.h:1955
clang::OMPParallelForDirective
This represents &#39;#pragma omp parallel for&#39; directive.
Definition: StmtOpenMP.h:1715
clang::CodeGen::CodeGenFunction::GenerateOpenMPCapturedVars
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
Definition: CGStmtOpenMP.cpp:291
clang::OMPTargetTeamsDistributeParallelForDirective
This represents &#39;#pragma omp target teams distribute parallel for&#39; combined directive.
Definition: StmtOpenMP.h:4379
clang::OMPLoopDirective::getCombinedEnsureUpperBound
Expr * getCombinedEnsureUpperBound() const
Definition: StmtOpenMP.h:1046
clang::CodeGen::CodeGenFunction::EmitAtomicStore
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1954
clang::CodeGen::CodeGenFunction::ConstantFoldsToSimpleInteger
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Definition: CodeGenFunction.cpp:1433
clang::LoopExit
Represents a point when we exit a loop.
Definition: ProgramPoint.h:713
clang::CodeGen::CodeGenFunction::EmitOMPCopy
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
Definition: CGStmtOpenMP.cpp:705
Value
Value
Definition: UninitializedValues.cpp:101
clang::CodeGen::OMPTaskDataTy::ReductionCopies
SmallVector< const Expr *, 4 > ReductionCopies
Definition: CGOpenMPRuntime.h:102
clang::OMPExecutableDirective::getEndLoc
SourceLocation getEndLoc() const
Returns ending location of directive.
Definition: StmtOpenMP.h:227
clang::CodeGen::CodeGenFunction::EmitOMPParallelForSimdDirective
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
Definition: CGStmtOpenMP.cpp:3053
createSectionLVal
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
Definition: CGStmtOpenMP.cpp:2796
clang::OMPTargetExitDataDirective
This represents &#39;#pragma omp target exit data&#39; directive.
Definition: StmtOpenMP.h:2690
clang::CodeGen::CodeGenFunction::GetAddrOfLocalVar
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
Definition: CodeGenFunction.h:2371
clang::ASTContext::getCanonicalParamType
CanQualType getCanonicalParamType(QualType T) const
Return the canonical parameter type corresponding to the specific potentially non-canonical one...
Definition: ASTContext.cpp:5319
clang::CodeGen::CodeGenFunction::GenerateOpenMPCapturedStmtFunction
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
Definition: CGStmtOpenMP.cpp:563
clang::VarDecl
Represents a variable declaration or definition.
Definition: Decl.h:820
clang::OMPPrivateClause
This represents clause &#39;private&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:1885
clang::CodeGen::CodeGenFunction::EmitOMPCriticalDirective
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
Definition: CGStmtOpenMP.cpp:3025
clang::OMPTargetParallelForDirective::hasCancel
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:2873
clang::OMPNumThreadsClause
This represents &#39;num_threads&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:594
clang::SC_Static
Definition: Specifiers.h:239
clang::OMPLoopDirective::dependent_inits
ArrayRef< Expr * > dependent_inits()
Definition: StmtOpenMP.h:1140
clang::SC_None
Definition: Specifiers.h:237
clang::FunctionProtoType::ExtProtoInfo
Extra information about a function prototype.
Definition: Type.h:3837
clang::CodeGen::CodeGenFunction::getProfileCount
uint64_t getProfileCount(const Stmt *S)
Get the profiler&#39;s count for the given statement.
Definition: CodeGenFunction.h:1363
clang::OMPLoopDirective::getCombinedUpperBoundVariable
Expr * getCombinedUpperBoundVariable() const
Definition: StmtOpenMP.h:1040
clang::OMPLoopDirective::getCalcLastIteration
Expr * getCalcLastIteration() const
Definition: StmtOpenMP.h:922
clang::CodeGen::Address::getPointer
llvm::Value * getPointer() const
Definition: Address.h:37
clang::CapturedDecl::getContextParamPosition
unsigned getContextParamPosition() const
Definition: Decl.h:4300
clang::CodeGen::CodeGenFunction::EmitOMPInnerLoop
void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
Definition: CGStmtOpenMP.cpp:1523
OMPFlushClause
This represents implicit clause &#39;flush&#39; for the &#39;#pragma omp flush&#39; directive.
Definition: OpenMPClause.h:3885
clang::tooling::fixit::internal::getSourceRange
CharSourceRange getSourceRange(const SourceRange &Range)
Returns the token CharSourceRange corresponding to Range.
Definition: FixIt.h:32
clang::CodeGen::CodeGenFunction::EmitOMPLastprivateClauseFinal
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
Definition: CGStmtOpenMP.cpp:1033
clang::CodeGen::CodeGenFunction::EmitOMPParallelMasterTaskLoopSimdDirective
void EmitOMPParallelMasterTaskLoopSimdDirective(const OMPParallelMasterTaskLoopSimdDirective &S)
Definition: CGStmtOpenMP.cpp:5424
clang::CodeGen::CodeGenFunction::EmitOMPTeamsDistributeDirective
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
Definition: CGStmtOpenMP.cpp:4649
emitTargetTeamsDistributeRegion
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
Definition: CGStmtOpenMP.cpp:4558
clang::OMPAtomicDirective::isXLHSInRHSPart
bool isXLHSInRHSPart() const
Return true if helper update expression has form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; and...
Definition: StmtOpenMP.h:2486
clang::CodeGen::CodeGenFunction::EmitOMPTargetParallelDeviceFunction
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
Definition: CGStmtOpenMP.cpp:5111
OMPNogroupClause
This represents &#39;nogroup&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:5380
clang::CodeGen::CodeGenFunction::JumpDest
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
Definition: CodeGenFunction.h:235
clang::CodeGen::CodeGenFunction::EmitOMPWorksharingLoop
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
Definition: CGStmtOpenMP.cpp:2495
clang::OMPSafelenClause
This represents &#39;safelen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:670
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Definition: CGStmtOpenMP.cpp:4836
clang::CodeGen::TEK_Aggregate
Definition: CodeGenFunction.h:108
clang::NamedDecl::getIdentifier
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
Definition: Decl.h:244
clang::CodeGen::CodeGenFunction::EmitOMPTaskLoopSimdDirective
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
Definition: CGStmtOpenMP.cpp:5382
clang::CodeGen::OMPTaskDataTy::PrivateVars
SmallVector< const Expr *, 4 > PrivateVars
Definition: CGOpenMPRuntime.h:94
clang::RecordDecl
Represents a struct/union/class.
Definition: Decl.h:3748
clang::CodeGen::CodeGenFunction::LambdaCaptureFields
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
Definition: CodeGenFunction.h:491
clang::CodeGen::CodeGenFunction::OMPLocalDeclMapRAII
Save/restore original map of previously emitted local vars in case when we need to duplicate emission...
Definition: CodeGenFunction.h:954
clang::IdentifierInfo
One of these records is kept for each identifier that is lexed.
Definition: IdentifierTable.h:57
clang::CodeGen::CodeGenFunction::EmitOMPSimdDirective
void EmitOMPSimdDirective(const OMPSimdDirective &S)
Definition: CGStmtOpenMP.cpp:2033
clang::OMPParallelDirective
This represents &#39;#pragma omp parallel&#39; directive.
Definition: StmtOpenMP.h:357
clang::CodeGen::CodeGenFunction
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Definition: CodeGenFunction.h:227
clang::CodeGen::CodeGenFunction::EmitOMPMasterTaskLoopSimdDirective
void EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S)
Definition: CGStmtOpenMP.cpp:5398
clang::ASTContext
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:168
clang::NestedNameSpecifierLoc
A C++ nested-name-specifier augmented with source location information.
Definition: NestedNameSpecifier.h:236
llvm::Optional
Definition: LLVM.h:37
OMPSIMDClause
This represents &#39;simd&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:4200
clang::CodeGen::CodeGenTypeCache::Int64Ty
llvm::IntegerType * Int64Ty
Definition: CodeGenTypeCache.h:37
clang::CodeGen::CodeGenFunction::EmitOMPTaskLoopBasedDirective
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
Definition: CGStmtOpenMP.cpp:5221
clang::CodeGen::TEK_Complex
Definition: CodeGenFunction.h:107
clang::CodeGen::CodeGenFunction::OMPPrivateScope
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
Definition: CodeGenFunction.h:903
clang::RecordDecl::fields
field_range fields() const
Definition: Decl.h:3963
emitTargetRegion
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:4436
clang::OMPLoopDirective::getEnsureUpperBound
Expr * getEnsureUpperBound() const
Definition: StmtOpenMP.h:978
clang::CodeGen::OMPTaskDataTy::LastprivateVars
SmallVector< const Expr *, 4 > LastprivateVars
Definition: CGOpenMPRuntime.h:99
clang::Stmt::getBeginLoc
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Stmt.cpp:275
clang::CodeGen::CodeGenFunction::EmitLoadOfReference
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
Definition: CGExpr.cpp:2323
clang::FieldDecl
Represents a member of a struct/union/class.
Definition: Decl.h:2729
clang::OMPLastprivateClause
This represents clause &#39;lastprivate&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:2132
clang::CodeGen::CodeGenFunction::EmitOMPTaskBasedDirective
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
Definition: CGStmtOpenMP.cpp:3107
clang::CodeGen::CodeGenFunction::EmitOMPDistributeLoop
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
Definition: CGStmtOpenMP.cpp:3607
clang::CodeGen::LValue::getAlignment
CharUnits getAlignment() const
Definition: CGValue.h:316
clang::CodeGen::CodeGenTypeCache::SizeTy
llvm::IntegerType * SizeTy
Definition: CodeGenTypeCache.h:47
clang::OMPExecutableDirective::getCapturedStmt
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive...
Definition: StmtOpenMP.h:266
clang::CodeGen::CodeGenFunction::OpaqueValueMapping
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Definition: CodeGenFunction.h:1163
clang::OMPLoopDirective::getInc
Expr * getInc() const
Definition: StmtOpenMP.h:938
emitSimdlenSafelenClause
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, bool IsMonotonic)
Definition: CGStmtOpenMP.cpp:1805
clang::OMPLoopDirective::updates
ArrayRef< Expr * > updates()
Definition: StmtOpenMP.h:1122
clang::Type::isReferenceType
bool isReferenceType() const
Definition: Type.h:6516
clang::CodeGen::CodeGenFunction::EmitOMPAggregateAssign
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
Definition: CGStmtOpenMP.cpp:644
clang::CodeGen::CodeGenFunction::EmitOMPPrivateLoopCounters
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
Definition: CGStmtOpenMP.cpp:1682
clang::OMPTargetSimdDirective
This represents &#39;#pragma omp target simd&#39; directive.
Definition: StmtOpenMP.h:3895
clang::CodeGen::CodeGenFunction::Builder
CGBuilderTy Builder
Definition: CodeGenFunction.h:263
clang::CodeGen::CodeGenFunction::AutoVarEmission::getAllocatedAddress
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself...
Definition: CodeGenFunction.h:2753
clang::OMPExecutableDirective::getDirectiveKind
OpenMPDirectiveKind getDirectiveKind() const
Definition: StmtOpenMP.h:301
clang::DeclaratorDecl::getBeginLoc
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Decl.h:723
clang::OMPBarrierDirective
This represents &#39;#pragma omp barrier&#39; directive.
Definition: StmtOpenMP.h:2101
clang::OMPLoopDirective
This is a common base class for loop directives (&#39;omp simd&#39;, &#39;omp for&#39;, &#39;omp for simd&#39; etc...
Definition: StmtOpenMP.h:420
clang::OMPCriticalDirective
This represents &#39;#pragma omp critical&#39; directive.
Definition: StmtOpenMP.h:1640
emitOMPAtomicRMW
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
Definition: CGStmtOpenMP.cpp:3989
clang::OMPCancellationPointDirective::getCancelRegion
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:2991
llvm::SmallVectorImpl
Definition: LLVM.h:36
clang::ASTContext::Idents
IdentifierTable & Idents
Definition: ASTContext.h:580
emitOMPAtomicCaptureExpr
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:4162
clang::VK_RValue
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
Definition: Specifiers.h:125
clang::CodeGen::CodeGenFunction::EmitStoreOfScalar
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
Definition: CodeGenFunction.h:3456
clang::CodeGen::CodeGenFunction::EmitOMPDistributeParallelForSimdDirective
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
Definition: CGStmtOpenMP.cpp:2442
clang::CodeGen::CodeGenFunction::EmitOMPLastprivateClauseInit
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
Definition: CGStmtOpenMP.cpp:974
clang::CodeGen::CodeGenFunction::EmitAnyExpr
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
Definition: CGExpr.cpp:194
OMPCopyprivateClause
This represents clause &#39;copyprivate&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:3718
clang::OpenMPDistScheduleClauseKind
OpenMPDistScheduleClauseKind
OpenMP attributes for &#39;dist_schedule&#39; clause.
Definition: OpenMPKinds.h:151
clang::OMPDistributeParallelForDirective
This represents &#39;#pragma omp distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3606
clang::CapturedDecl::getBody
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
Definition: Decl.cpp:4748
clang::OMPTeamsDistributeParallelForSimdDirective
This represents &#39;#pragma omp teams distribute parallel for simd&#39; composite directive.
Definition: StmtOpenMP.h:4100
clang::BinaryOperatorKind
BinaryOperatorKind
Definition: OperationKinds.h:25
clang::OMPC_uniform
Definition: OpenMPKinds.h:70
clang::CodeGen::CodeGenFunction::hasScalarEvaluationKind
static bool hasScalarEvaluationKind(QualType T)
Definition: CodeGenFunction.h:2062
clang::CapturedStmt::const_capture_init_iterator
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
Definition: Stmt.h:3537
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
Definition: CGStmtOpenMP.cpp:4641
clang::OMPLoopDirective::finals
ArrayRef< Expr * > finals()
Definition: StmtOpenMP.h:1128
clang::OMPLoopDirective::getIsLastIterVariable
Expr * getIsLastIterVariable() const
Definition: StmtOpenMP.h:946
clang::CodeGen::CodeGenFunction::emitOMPSimpleStore
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3933
clang::CodeGen::CGOpenMPRuntime::NontemporalDeclsRAII
Manages list of nontemporal decls for the specified directive.
Definition: CGOpenMPRuntime.h:217
clang::OpenMPScheduleTy::Schedule
OpenMPScheduleClauseKind Schedule
Definition: OpenMPKinds.h:199
clang::CharUnits
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
emitEmptyOrdered
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
Definition: CGStmtOpenMP.cpp:2258
clang::isOpenMPTeamsDirective
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
Definition: OpenMPKinds.cpp:1047
emitDistributeParallelForDistributeInnerBoundParams
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value *> &CapturedVars)
Definition: CGStmtOpenMP.cpp:2382
clang::TargetInfo::hasBuiltinAtomic
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
Definition: TargetInfo.h:630
clang::CodeGen::CodeGenFunction::CodeGenLoopBoundsTy
llvm::function_ref< std::pair< LValue, LValue > CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
Definition: CodeGenFunction.h:280
clang::OMPAtomicDirective::getX
Expr * getX()
Get &#39;x&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2470
clang::Type::getAsArrayTypeUnsafe
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
Definition: Type.h:7053
clang::CodeGen::CodeGenFunction::EmitOMPCancellationPointDirective
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
Definition: CGStmtOpenMP.cpp:4844
clang::CodeGen::CodeGenFunction::CGCapturedStmtRAII
RAII for correct setting/restoring of CapturedStmtInfo.
Definition: CodeGenFunction.h:415
clang::CapturedDecl::param_end
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
Definition: Decl.h:4308
clang::CodeGen::Address::getAlignment
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:66
clang::CodeGen::CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
Definition: CGStmtOpenMP.cpp:4717
clang::CodeGen::CodeGenFunction::EmitOMPTargetDirective
void EmitOMPTargetDirective(const OMPTargetDirective &S)
Definition: CGStmtOpenMP.cpp:4463
clang::OMPLoopDirective::getIterationVariable
Expr * getIterationVariable() const
Definition: StmtOpenMP.h:914
clang::CodeGen::RValue::isComplex
bool isComplex() const
Definition: CGValue.h:53
clang::BinaryOperator
A builtin binary operation expression such as "x + y" or "x <= y".
Definition: Expr.h:3434
clang::QualType::isVolatileQualified
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
Definition: Type.h:6326
clang::OpenMPScheduleTy::M2
OpenMPScheduleClauseModifier M2
Definition: OpenMPKinds.h:201
clang::CodeGen::LoopInfoStack::setVectorizeWidth
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
Definition: CGLoopInfo.h:254
clang::CodeGen::OMPTaskDataTy::PrivateCopies
SmallVector< const Expr *, 4 > PrivateCopies
Definition: CGOpenMPRuntime.h:95
clang::CodeGen::CodeGenFunction::OMPMapVars::restore
void restore(CodeGenFunction &CGF)
Restores original addresses of the variables.
Definition: CodeGenFunction.h:874
emitOutlinedOrderedFunction
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S)
Definition: CGStmtOpenMP.cpp:3845
clang::CodeGen::CodeGenFunction::EmitOMPTargetUpdateDirective
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
Definition: CGStmtOpenMP.cpp:5441
clang::OMPCancellationPointDirective
This represents &#39;#pragma omp cancellation point&#39; directive.
Definition: StmtOpenMP.h:2946
clang::PrettyStackTraceLoc
If a crash happens while one of these objects are live, the message is printed out along with the spe...
Definition: PrettyStackTrace.h:26
clang::ASTContext::getDeclAlign
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
Definition: ASTContext.cpp:1724
EmitOMPHelperVar
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
Definition: CGStmtOpenMP.cpp:1895
clang::CodeGen::ReductionCodeGen::adjustPrivateAddress
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations...
Definition: CGOpenMPRuntime.cpp:1174
clang::VariableArrayType::getSizeExpr
Expr * getSizeExpr() const
Definition: Type.h:3058
clang::Scope
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:40
clang::RecordDecl::field_begin
field_iterator field_begin() const
Definition: Decl.cpp:4425
clang::OMPFinalClause
This represents &#39;final&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:525
clang::isOpenMPTargetExecutionDirective
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
Definition: OpenMPKinds.cpp:1025
clang::CodeGen::CodeGenFunction::EmitOMPParallelDirective
void EmitOMPParallelDirective(const OMPParallelDirective &S)
Definition: CGStmtOpenMP.cpp:1337
clang::CodeGen::CodeGenFunction::createBasicBlock
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
Definition: CodeGenFunction.h:2071
clang::OMPTeamsDirective
This represents &#39;#pragma omp teams&#39; directive.
Definition: StmtOpenMP.h:2888
clang::CodeGen::CodeGenFunction::EmitIgnoredExpr
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition: CGExpr.cpp:182
clang::CodeGen::OMPTaskDataTy::FirstprivateCopies
SmallVector< const Expr *, 4 > FirstprivateCopies
Definition: CGOpenMPRuntime.h:97
clang::OMPLoopDirective::getInit
Expr * getInit() const
Definition: StmtOpenMP.h:934
emitPostUpdateForReductionClause
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Definition: CGStmtOpenMP.cpp:1256
clang::OMPReductionClause
This represents clause &#39;reduction&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:2443
clang::CodeGen::ReductionCodeGen::getBaseDecl
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
Definition: CGOpenMPRuntime.h:195
clang::OMPTeamsDistributeSimdDirective
This represents &#39;#pragma omp teams distribute simd&#39; combined directive.
Definition: StmtOpenMP.h:4029
clang::CodeGen::CodeGenFunction::RunCleanupsScope::ForceCleanup
void ForceCleanup(std::initializer_list< llvm::Value **> ValuesToReload={})
Force the emission of cleanups now, instead of waiting until this object is destroyed.
Definition: CodeGenFunction.h:754
clang::CodeGen::LValue::isSimple
bool isSimple() const
Definition: CGValue.h:252
clang::isOpenMPWorksharingDirective
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
Definition: OpenMPKinds.cpp:987
clang::CodeGen::CodeGenFunction::EmitOMPSimdFinal
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Definition: CGStmtOpenMP.cpp:1840
clang::OK_Ordinary
An ordinary object is located at an address in memory.
Definition: Specifiers.h:141
clang::CodeGen::OMPTaskDataTy::ReductionOps
SmallVector< const Expr *, 4 > ReductionOps
Definition: CGOpenMPRuntime.h:103
clang::CodeGen::CodeGenFunction::OMPCancelStackRAII
Controls insertion of cancellation exit blocks in worksharing constructs.
Definition: CodeGenFunction.h:2968
clang::CodeGen::CodeGenFunction::incrementProfileCounter
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler&#39;s counter for the given statement by StepV.
Definition: CodeGenFunction.h:1356
clang::CodeGen::CodeGenFunction::CodeGenDispatchBoundsTy
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * > CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
Definition: CodeGenFunction.h:286
clang::CapturedDecl
Represents the body of a CapturedStmt, and serves as its DeclContext.
Definition: Decl.h:4226
llvm::ArrayRef
Definition: LLVM.h:31
llvm::SmallString
Definition: LLVM.h:34
clang::OpenMPDirectiveKind
llvm::omp::Directive OpenMPDirectiveKind
OpenMP directives.
Definition: OpenMPKinds.h:62
clang::CodeGen::OMPTaskDataTy::ReductionVars
SmallVector< const Expr *, 4 > ReductionVars
Definition: CGOpenMPRuntime.h:101
convertToScalarValue
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3882
clang::CodeGen::OMPTaskDataTy::Reductions
llvm::Value * Reductions
Definition: CGOpenMPRuntime.h:108
emitOutlinedFunctionPrologue
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address >> &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value *>> &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
Definition: CGStmtOpenMP.cpp:389
clang::CodeGen::CodeGenFunction::ConstantEmission::isReference
bool isReference() const
Definition: CodeGenFunction.h:3557
clang::Expr::IgnoreImpCasts
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point...
Definition: Expr.cpp:2970
clang::CodeGen::CodeGenFunction::EmitLoadOfScalar
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
Definition: CodeGenFunction.h:3434
convertToComplexValue
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3895
clang::ImplicitParamDecl::Create
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
Definition: Decl.cpp:4685
clang::OMPTargetParallelForSimdDirective
This represents &#39;#pragma omp target parallel for simd&#39; directive.
Definition: StmtOpenMP.h:3825
clang::OMPLoopDirective::private_counters
ArrayRef< Expr * > private_counters()
Definition: StmtOpenMP.h:1110
clang::Decl::hasAttr
bool hasAttr() const
Definition: DeclBase.h:542
clang::CompoundStmt
CompoundStmt - This represents a group of statements like { stmt stmt }.
Definition: Stmt.h:1332
clang::CodeGen::CodeGenFunction::ComplexPairTy
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
Definition: CodeGenFunction.h:261
clang::CodeGen::CodeGenFunction::EmitAutoVarAlloca
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
Definition: CGDecl.cpp:1379
clang::CodeGen::CodeGenFunction::getContext
ASTContext & getContext() const
Definition: CodeGenFunction.h:1627
clang::OMPSectionDirective::hasCancel
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1520
clang::CodeGen::CodeGenFunction::EmitOMPParallelSectionsDirective
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
Definition: CGStmtOpenMP.cpp:3095
clang::OMPTaskgroupDirective
This represents &#39;#pragma omp taskgroup&#39; directive.
Definition: StmtOpenMP.h:2193
clang::CodeGen::RValue
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
Definition: CGValue.h:39
OMPAlignedClause
This represents clause &#39;aligned&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:3443
clang::ImplicitCastExpr::Create
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat)
Definition: Expr.cpp:1993
clang::CodeGen::ReductionCodeGen
Class intended to support codegen of all kind of the reduction clauses.
Definition: CGOpenMPRuntime.h:115
clang::CodeGen::CodeGenFunction::OMPPrivateScope::isGlobalVarCaptured
bool isGlobalVarCaptured(const VarDecl *VD) const
Checks if the global variable is captured in current function.
Definition: CodeGenFunction.h:945
clang::OMPLoopDirective::getCombinedLowerBoundVariable
Expr * getCombinedLowerBoundVariable() const
Definition: StmtOpenMP.h:1034
clang::CodeGen::CodeGenFunction::OMPMapVars
The class used to assign some variables some temporarily addresses.
Definition: CodeGenFunction.h:824
clang::OMPTaskReductionClause
This represents clause &#39;task_reduction&#39; in the &#39;#pragma omp taskgroup&#39; directives.
Definition: OpenMPClause.h:2679
clang::CodeGen::CodeGenFunction::EmitOMPLoopBody
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
Definition: CGStmtOpenMP.cpp:1480
clang::CharUnits::getQuantity
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:179
clang::CodeGen::CodeGenFunction::OMPPrivateScope::ForceCleanup
void ForceCleanup()
Definition: CodeGenFunction.h:933
clang::ASTContext::getTrivialTypeSourceInfo
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
Definition: ASTContext.cpp:2816
createImplicitFirstprivateForType
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:3367
clang::ArrayType::Normal
Definition: Type.h:2896
clang::OMPDistributeDirective
This represents &#39;#pragma omp distribute&#39; directive.
Definition: StmtOpenMP.h:3480
clang::CodeGen::CodeGenFunction::EmitOMPParallelMasterDirective
void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S)
Definition: CGStmtOpenMP.cpp:3066
OMPDependClause
This represents implicit clause &#39;depend&#39; for the &#39;#pragma omp task&#39; directive.
Definition: OpenMPClause.h:3958
clang::CodeGen::CodeGenFunction::getEvaluationKind
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
Definition: CodeGenFunction.cpp:237
clang::CodeGen::CodeGenFunction::OMPTargetDataInfo::NumberOfTargetItems
unsigned NumberOfTargetItems
Definition: CodeGenFunction.h:3122
clang::OMPExecutableDirective::getAssociatedStmt
const Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
Definition: StmtOpenMP.h:253
clang::CodeGen::CodeGenFunction::OMPPrivateScope::addPrivate
bool addPrivate(const VarDecl *LocalVD, const llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
Definition: CodeGenFunction.h:917
clang::OMPParallelForDirective::hasCancel
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1779
clang::OMPProcBindClause
This represents &#39;proc_bind&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:942
clang::ValueDecl
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Definition: Decl.h:619
clang::Expr
This represents one expression.
Definition: Expr.h:108
emitTargetParallelForRegion
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:5133
clang::CodeGen::CodeGenFunction::EmitAutoVarInit
void EmitAutoVarInit(const AutoVarEmission &emission)
Definition: CGDecl.cpp:1745
clang::CodeGen::Address::invalid
static Address invalid()
Definition: Address.h:34
CodeGenModule.h
clang::CodeGen::LValue::getAddress
Address getAddress(CodeGenFunction &CGF) const
Definition: CGValue.h:327
clang::CodeGen::CodeGenFunction::LoopStack
LoopInfoStack LoopStack
Definition: CodeGenFunction.h:262
clang::VarDecl::hasLocalStorage
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
Definition: Decl.h:1045
clang::CodeGen::CodeGenFunction::RunCleanupsScope
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
Definition: CodeGenFunction.h:709
clang::OMPSimdlenClause
This represents &#39;simdlen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:735
clang::Stmt::IgnoreContainers
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top, if IgnoreCaptured is true.
Definition: Stmt.cpp:134
clang::CodeGen::CodeGenFunction::EmitComplexToScalarConversion
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type, where the destination type is an LLVM scalar type.
Definition: CGExprScalar.cpp:4563
clang::CodeGen::CodeGenFunction::CodeGenOrderedTy
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
Definition: CodeGenFunction.h:275
clang::CodeGen::RValue::getComplexVal
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:66
clang::isOpenMPParallelDirective
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
Definition: OpenMPKinds.cpp:1009
clang::OMPMasterTaskLoopDirective
This represents &#39;#pragma omp master taskloop&#39; directive.
Definition: StmtOpenMP.h:3204
clang::OMPC_threadprivate
Definition: OpenMPKinds.h:69
clang::CodeGen::CodeGenFunction::EmitOMPTeamsDirective
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
Definition: CGStmtOpenMP.cpp:4496
clang::CodeGen::CodeGenFunction::VlaSizePair::NumElts
llvm::Value * NumElts
Definition: CodeGenFunction.h:2435
clang::Type::castAs
const T * castAs() const
Member-template castAs<specific type>.
Definition: Type.h:7067
clang::CodeGen::CodeGenFunction::JumpDest::getBlock
llvm::BasicBlock * getBlock() const
Definition: CodeGenFunction.h:243
V
#define V(N, I)
Definition: ASTContext.h:2941
emitTargetTeamsDistributeParallelForRegion
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:4741
clang::OMPForDirective::hasCancel
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1321
emitDispatchForLoopBounds
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
Definition: CGStmtOpenMP.cpp:2748
clang::OMPTargetTeamsDistributeParallelForSimdDirective
This represents &#39;#pragma omp target teams distribute parallel for simd&#39; combined directive.
Definition: StmtOpenMP.h:4463
emitPreCond
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
Definition: CGStmtOpenMP.cpp:1732
clang::OpenMPClauseKind
OpenMPClauseKind
OpenMP clauses.
Definition: OpenMPKinds.h:65
clang::CodeGen::Address::getType
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:43
clang::OMPExecutableDirective::getClausesOfKind
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause *> Clauses)
Definition: StmtOpenMP.h:187
clang::OMPTargetTeamsDistributeSimdDirective
This represents &#39;#pragma omp target teams distribute simd&#39; combined directive.
Definition: StmtOpenMP.h:4536
clang::CodeGen::CodeGenFunction::EmitOMPSimdInit
void EmitOMPSimdInit(const OMPLoopDirective &D, bool IsMonotonic=false)
Helpers for the OpenMP loop directives.
Definition: CGStmtOpenMP.cpp:1832
clang::OMPOrderedClause
This represents &#39;ordered&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:1474
clang::CapturedStmt::children
child_range children()
Definition: Stmt.cpp:1289
clang::OMPLoopDirective::getDistInc
Expr * getDistInc() const
Definition: StmtOpenMP.h:1022
clang::CodeGen::CodeGenFunction::getLLVMContext
llvm::LLVMContext & getLLVMContext()
Definition: CodeGenFunction.h:1670
clang::OMPLoopDirective::getNextLowerBound
Expr * getNextLowerBound() const
Definition: StmtOpenMP.h:986
clang::Expr::getType
QualType getType() const
Definition: Expr.h:137
clang::OMPLoopDirective::getPrevEnsureUpperBound
Expr * getPrevEnsureUpperBound() const
Definition: StmtOpenMP.h:1028
clang::OMPForDirective
This represents &#39;#pragma omp for&#39; directive.
Definition: StmtOpenMP.h:1259
emitTargetTeamsRegion
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
Definition: CGStmtOpenMP.cpp:4513
Stmt.h
clang::CodeGen::OMPTaskDataTy::FirstprivateVars
SmallVector< const Expr *, 4 > FirstprivateVars
Definition: CGOpenMPRuntime.h:96
clang::CapturedDecl::Create
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
Definition: Decl.cpp:4736
clang::CodeGen::CodeGenFunction::EmitOMPMasterDirective
void EmitOMPMasterDirective(const OMPMasterDirective &S)
Definition: CGStmtOpenMP.cpp:3020
emitDistributeParallelForDispatchBounds
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the &#39;for&#39; loop has a dispatch schedule (e.g.
Definition: CGStmtOpenMP.cpp:2365
clang::CharUnits::alignmentOfArrayElement
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
Definition: CharUnits.h:201
clang::CodeGen::LoopInfoStack::setParallel
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
Definition: CGLoopInfo.h:224
clang::OMPTargetTeamsDirective
This represents &#39;#pragma omp target teams&#39; directive.
Definition: StmtOpenMP.h:4251
clang::CodeGen::CodeGenFunction::MakeNaturalAlignAddrLValue
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
Definition: CodeGenFunction.cpp:209
clang::SourceRange::getEnd
SourceLocation getEnd() const
Definition: SourceLocation.h:217
clang::UnaryOperator
UnaryOperator - This represents the unary-expression&#39;s (except sizeof and alignof), the postinc/postdec operators from postfix-expression, and various extensions.
Definition: Expr.h:2046
clang::isOpenMPTaskLoopDirective
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
Definition: OpenMPKinds.cpp:1002
clang::ASTContext::getFunctionType
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
Definition: ASTContext.h:1401
clang::CodeGen::CodeGenFunction::EmitOMPBarrierDirective
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
Definition: CGStmtOpenMP.cpp:3548
clang::CodeGen::CodeGenFunction::EmitOMPLinearClauseInit
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
Definition: CGStmtOpenMP.cpp:1572
clang::OMPCancelDirective
This represents &#39;#pragma omp cancel&#39; directive.
Definition: StmtOpenMP.h:3005
clang::OMPFirstprivateClause
This represents clause &#39;firstprivate&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:1991
clang::DeclRefExpr::getDecl
ValueDecl * getDecl()
Definition: Expr.h:1247
clang::CodeGen::CodeGenModule::getLangOpts
const LangOptions & getLangOpts() const
Definition: CodeGenModule.h:675
clang::CodeGen::CodeGenModule::getContext
ASTContext & getContext() const
Definition: CodeGenModule.h:674
emitCommonOMPTargetDirective
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
Definition: CGStmtOpenMP.cpp:4357
clang::CodeGen::CodeGenFunction::EmitAtomicLoad
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
Definition: CGAtomic.cpp:1530
CodeGenFunction.h
emitForLoopBounds
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
Definition: CGStmtOpenMP.cpp:2734
clang::OMPLoopDirective::tryToFindNextInnerLoop
static Stmt * tryToFindNextInnerLoop(Stmt *CurStmt, bool TryImperfectlyNestedLoops)
Try to find the next loop sub-statement in the specified statement CurStmt.
Definition: StmtOpenMP.cpp:45
clang::VarDecl::getCanonicalDecl
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
Definition: Decl.cpp:2088
clang::GlobalDecl
GlobalDecl - represents a global declaration.
Definition: GlobalDecl.h:40
clang::CodeGen::CodeGenFunction::EmitSimpleOMPExecutableDirective
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
Definition: CGStmtOpenMP.cpp:5462
clang::OMPFlushDirective
This represents &#39;#pragma omp flush&#39; directive.
Definition: StmtOpenMP.h:2267
clang::OMPExecutableDirective::hasClausesOfKind
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind. ...
Definition: StmtOpenMP.h:219
clang::OMPParallelForSimdDirective
This represents &#39;#pragma omp parallel for simd&#39; directive.
Definition: StmtOpenMP.h:1796
clang::CodeGen::CGOpenMPRuntime
Definition: CGOpenMPRuntime.h:203
clang::OMPC_device_type
Definition: OpenMPKinds.h:71
clang::OMPSeqCstClause
This represents &#39;seq_cst&#39; clause in the &#39;#pragma omp atomic&#39; directive.
Definition: OpenMPClause.h:1845
clang::OMPUntiedClause
This represents &#39;untied&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:1608
clang::Decl::hasBody
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
Definition: DeclBase.h:992
emitAlignedClause
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
Definition: CGStmtOpenMP.cpp:1647
clang::CodeGen::AlignmentSource::Type
The l-value was considered opaque, so the alignment was determined from a type.
clang::OMPParallelMasterTaskLoopDirective
This represents &#39;#pragma omp parallel master taskloop&#39; directive.
Definition: StmtOpenMP.h:3340
clang::CodeGen::CodeGenFunction::CodeGenLoopTy
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
Definition: CodeGenFunction.h:272
clang::OMPLoopDirective::getLastIteration
Expr * getLastIteration() const
Definition: StmtOpenMP.h:918
clang::OMPExecutableDirective::getSingleClause
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
Definition: StmtOpenMP.h:205
clang::OMPAtomicDirective::isPostfixUpdate
bool isPostfixUpdate() const
Return true if &#39;v&#39; expression must be updated to original value of &#39;x&#39;, false if &#39;v&#39; must be updated ...
Definition: StmtOpenMP.h:2489
clang::OMPTargetEnterDataDirective
This represents &#39;#pragma omp target enter data&#39; directive.
Definition: StmtOpenMP.h:2631
clang::CodeGen::CodeGenFunction::EmitOMPFlushDirective
void EmitOMPFlushDirective(const OMPFlushDirective &S)
Definition: CGStmtOpenMP.cpp:3595
CGCleanup.h
clang::OMPLoopDirective::getStrideVariable
Expr * getStrideVariable() const
Definition: StmtOpenMP.h:970
clang::OMPMasterTaskLoopSimdDirective
This represents &#39;#pragma omp master taskloop simd&#39; directive.
Definition: StmtOpenMP.h:3272
OMPNumTeamsClause
This represents &#39;num_teams&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:5089
clang::OpaqueValueExpr
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
Definition: Expr.h:1075
clang::OMPLoopDirective::getCombinedDistCond
Expr * getCombinedDistCond() const
Definition: StmtOpenMP.h:1076
clang::OMPLoopDirective::getPreInits
const Stmt * getPreInits() const
Definition: StmtOpenMP.h:942
Kind
Kind
Definition: ChrootChecker.cpp:29
clang::CapturedStmt
This captures a statement into a function.
Definition: Stmt.h:3376
Attr.h
clang::CodeGen::OMPTaskDataTy::Tied
bool Tied
Definition: CGOpenMPRuntime.h:110
emitTargetParallelRegion
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:5086
emitOMPSimdRegion
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:1936
omp
clang::OMPSingleDirective
This represents &#39;#pragma omp single&#39; directive.
Definition: StmtOpenMP.h:1535
clang::SourceLocation
Encodes a location in the source.
Definition: SourceLocation.h:86
clang::IdentifierTable::get
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
Definition: IdentifierTable.h:529
OMPHintClause
This represents &#39;hint&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:5488
clang::CodeGen::CodeGenFunction::EmitOMPForDirective
void EmitOMPForDirective(const OMPForDirective &S)
Definition: CGStmtOpenMP.cpp:2758
APSInt
llvm::APSInt APSInt
Definition: ByteCodeEmitter.cpp:18
TargetInfo.h
clang::ASTContext::getUIntPtrType
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
Definition: ASTContext.cpp:5292
clang::ASTContext::getOpenMPDefaultSimdAlign
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
Definition: ASTContext.cpp:2360
clang::OMPCapturedExprDecl
Pseudo declaration for capturing expressions.
Definition: DeclOpenMP.h:312
clang::CodeGen::OMPTaskDataTy::Final
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
Definition: CGOpenMPRuntime.h:105
clang::Type::isVariablyModifiedType
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Definition: Type.h:2166
emitInnerParallelForWhenCombined
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
Definition: CGStmtOpenMP.cpp:2402
emitTargetParallelForSimdRegion
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:5174
clang::OMPExecutableDirective
This is a basic class for representing single OpenMP executable directive.
Definition: StmtOpenMP.h:33
clang::OMPScheduleClause
This represents &#39;schedule&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:1281
clang::OMPAtomicDirective::getExpr
Expr * getExpr()
Get &#39;expr&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2496
clang::CodeGen::LValue::getPointer
llvm::Value * getPointer(CodeGenFunction &CGF) const
Definition: CGValue.h:323
clang::CodeGen::CodeGenFunction::OMPTargetDataInfo::PointersArray
Address PointersArray
Definition: CodeGenFunction.h:3120
clang::ASTContext::getBaseElementType
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
Definition: ASTContext.cpp:5822
clang::OMPCriticalDirective::getDirectiveName
DeclarationNameInfo getDirectiveName() const
Return name of the directive.
Definition: StmtOpenMP.h:1699
clang::CodeGen::CodeGenFunction::EmitStmt
void EmitStmt(const Stmt *S, ArrayRef< const Attr *> Attrs=None)
EmitStmt - Emit the code for the statement.
Definition: CGStmt.cpp:45
clang::VarDecl::hasGlobalStorage
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
Definition: Decl.h:1087
clang::VarDecl::CInit
C-style initialization with assignment.
Definition: Decl.h:825
clang::OMPTaskwaitDirective
This represents &#39;#pragma omp taskwait&#39; directive.
Definition: StmtOpenMP.h:2147
clang::CodeGen::CodeGenFunction::EmitOMPTaskgroupDirective
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
Definition: CGStmtOpenMP.cpp:3556
DeclOpenMP.h
This file defines OpenMP nodes for declarative directives.
clang::CodeGen::CodeGenFunction::EmitOMPTargetParallelForSimdDirective
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
Definition: CGStmtOpenMP.cpp:5204
clang::CodeGen::CodeGenFunction::EmitOMPTargetSimdDeviceFunction
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
Definition: CGStmtOpenMP.cpp:2461
clang::CodeGen::CodeGenFunction::ConstantEmission
Definition: CodeGenFunction.h:3540
clang::OMPClause
This is a basic class for representing single OpenMP clause.
Definition: OpenMPClause.h:51
clang::CodeGen::CodeGenFunction::EmitOMPTeamsDistributeSimdDirective
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
Definition: CGStmtOpenMP.cpp:4672
clang::ASTContext::VoidTy
CanQualType VoidTy
Definition: ASTContext.h:1016
clang::isOpenMPLoopBoundSharingDirective
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
Definition: OpenMPKinds.cpp:1102
clang::CodeGen::ReductionCodeGen::emitAggregateType
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
Definition: CGOpenMPRuntime.cpp:997
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
Definition: CGStmtOpenMP.cpp:4768
llvm::SmallVector< llvm::Value *, 4 >
clang::Type::isAnyPointerType
bool isAnyPointerType() const
Definition: Type.h:6508
clang::CodeGen::CodeGenFunction::VlaSizePair::Type
QualType Type
Definition: CodeGenFunction.h:2436
clang::CodeGen::PrePostActionTy::Enter
virtual void Enter(CodeGenFunction &CGF)
Definition: CGOpenMPRuntime.h:59
clang::CodeGen::CodeGenFunction::EmitOMPSingleDirective
void EmitOMPSingleDirective(const OMPSingleDirective &S)
Definition: CGStmtOpenMP.cpp:2971
emitCommonSimdLoop
static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S, const RegionCodeGenTy &SimdInitGen, const RegionCodeGenTy &BodyCodeGen)
Definition: CGStmtOpenMP.cpp:1902
clang::CodeGen::Address
An aligned address.
Definition: Address.h:24
clang::interp::APInt
llvm::APInt APInt
Definition: Integral.h:27
clang::CodeGen::CodeGenFunction::StartFunction
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
Definition: CodeGenFunction.cpp:700
clang::CapturedStmt::getCapturedStmt
Stmt * getCapturedStmt()
Retrieve the statement being captured.
Definition: Stmt.h:3477
clang::Expr::isLValue
bool isLValue() const
isLValue - True if this expression is an "l-value" according to the rules of the current language...
Definition: Expr.h:258
clang::OMPTargetDirective
This represents &#39;#pragma omp target&#39; directive.
Definition: StmtOpenMP.h:2514
clang::CodeGen::CodeGenFunction::EmitOMPForSimdDirective
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
Definition: CGStmtOpenMP.cpp:2778
clang::OMPAtomicDirective::getV
Expr * getV()
Get &#39;v&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2491
clang::Dtor_Complete
Complete object dtor.
Definition: ABI.h:35
clang::XRayInstrKind::None
constexpr XRayInstrMask None
Definition: XRayInstr.h:37
clang::CodeGen::CodeGenFunction::EmitOMPAtomicDirective
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
Definition: CGStmtOpenMP.cpp:4324
clang::CodeGen::CodeGenFunction::EmitOMPSectionDirective
void EmitOMPSectionDirective(const OMPSectionDirective &S)
Definition: CGStmtOpenMP.cpp:2962
clang::CodeGen::CodeGenFunction::EmitOMPSectionsDirective
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
Definition: CGStmtOpenMP.cpp:2950
clang::OMPOrderedDirective
This represents &#39;#pragma omp ordered&#39; directive.
Definition: StmtOpenMP.h:2323
clang::CodeGen::LValue::getType
QualType getType() const
Definition: CGValue.h:264
clang::OMPTargetUpdateDirective
This represents &#39;#pragma omp target update&#39; directive.
Definition: StmtOpenMP.h:3547
clang::CodeGen::CodeGenFunction::EmitOMPTaskLoopDirective
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
Definition: CGStmtOpenMP.cpp:5378
clang::OMPParallelMasterTaskLoopSimdDirective
This represents &#39;#pragma omp parallel master taskloop simd&#39; directive.
Definition: StmtOpenMP.h:3411
clang::MultiVersionKind::None
clang::Expr::getExprLoc
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:224
clang::OMPLoopDirective::finals_conditions
ArrayRef< Expr * > finals_conditions()
Definition: StmtOpenMP.h:1146
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
Definition: CGStmtOpenMP.cpp:4581
clang::ImplicitParamDecl
Definition: Decl.h:1531
clang::CodeGen::CodeGenFunction::getOMPCancelDestination
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
Definition: CGStmtOpenMP.cpp:4877
clang::CodeGen::CodeGenFunction::EmitScalarExpr
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
Definition: CGExprScalar.cpp:4543
clang::CodeGen::FunctionArgList
FunctionArgList - Type for representing both the decl and type of parameters to a function...
Definition: CGCall.h:355
clang::CodeGen::RValue::getScalarVal
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:59
CGOpenMPRuntime.h
clang::CodeGen::RegionCodeGenTy::setAction
void setAction(PrePostActionTy &Action) const
Definition: CGOpenMPRuntime.h:89
clang::CodeGen::CGFunctionInfo
CGFunctionInfo - Class to encapsulate the information about a function definition.
Definition: CGFunctionInfo.h:476
clang::CodeGen::CodeGenModule
This class organizes the cross-function state that is used while generating LLVM code.
Definition: CodeGenModule.h:280
clang::CodeGen::CodeGenModule::getOpenMPRuntime
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
Definition: CodeGenModule.h:587
clang::OMPC_SCHEDULE_unknown
Definition: OpenMPKinds.h:89
clang::OpenMPScheduleClauseModifier
OpenMPScheduleClauseModifier
OpenMP modifiers for &#39;schedule&#39; clause.
Definition: OpenMPKinds.h:93
clang::CodeGen::CodeGenFunction::EmitOMPSharedLValue
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
Definition: CGStmtOpenMP.cpp:255
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDirective
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
Definition: CGStmtOpenMP.cpp:4549
clang::ParmVarDecl::Create
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
Definition: Decl.cpp:2672
clang
Dataflow Directional Tag Classes.
Definition: CFGReachabilityAnalysis.h:21
clang::DeclarationNameInfo::getAsString
std::string getAsString() const
getAsString - Retrieve the human-readable string for this name.
Definition: DeclarationName.cpp:448
clang::OMPLoopDirective::getPrevUpperBoundVariable
Expr * getPrevUpperBoundVariable() const
Definition: StmtOpenMP.h:1016
clang::CodeGen::RegionCodeGenTy
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
Definition: CGOpenMPRuntime.h:66
OMPDeviceClause
This represents &#39;device&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:4089
emitEmptyBoundParameters
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value *> &)
Definition: CGStmtOpenMP.cpp:1333
clang::CodeGen::CodeGenFunction::EmitOMPTaskyieldDirective
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
Definition: CGStmtOpenMP.cpp:3543
clang::CodeGen::RValue::getComplex
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:93
clang::Expr::EvalResult
EvalResult is a struct with detailed info about an evaluated expression.
Definition: Expr.h:586
clang::CodeGen::CodeGenFunction::LexicalScope
Definition: CodeGenFunction.h:768
clang::prec::And
Definition: OperatorPrecedence.h:35
clang::CodeGen::CodeGenFunction::EmitOMPTargetParallelDirective
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
Definition: CGStmtOpenMP.cpp:5125
castValueFromUintptr
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV)
Definition: CGStmtOpenMP.cpp:337
clang::CodeGen::AggValueSlot::ignored
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
Definition: CGValue.h:501
clang::CodeGen::CodeGenFunction::EmitOMPParallelForDirective
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
Definition: CGStmtOpenMP.cpp:3039
clang::OMPLoopDirective::getCond
Expr * getCond() const
Definition: StmtOpenMP.h:930
clang::CodeGen::OMPTaskDataTy::Nogroup
bool Nogroup
Definition: CGOpenMPRuntime.h:111
clang::OMPSectionDirective
This represents &#39;#pragma omp section&#39; directive.
Definition: StmtOpenMP.h:1472
clang::LangOptions::OMPTargetTriples
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Definition: LangOptions.h:293
clang::OpenMPScheduleTy::M1
OpenMPScheduleClauseModifier M1
Definition: OpenMPKinds.h:200
clang::OMPTeamsDistributeDirective
This represents &#39;#pragma omp teams distribute&#39; directive.
Definition: StmtOpenMP.h:3961
clang::CodeGen::PrePostActionTy
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
Definition: CGOpenMPRuntime.h:56
emitOMPLoopBodyWithStopPoint
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
Definition: CGStmtOpenMP.cpp:1886
clang::CodeGen::CodeGenFunction::EmitOMPPrivateClause
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Definition: CGStmtOpenMP.cpp:879
clang::CodeGen::ReductionCodeGen::getSharedLValue
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
Definition: CGOpenMPRuntime.h:188
clang::CodeGen::CodeGenFunction::EmitOMPTargetParallelForDirective
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
Definition: CGStmtOpenMP.cpp:5165
clang::VarDecl::getInit
const Expr * getInit() const
Definition: Decl.h:1229
clang::OMPSimdDirective
This represents &#39;#pragma omp simd&#39; directive.
Definition: StmtOpenMP.h:1194
clang::CodeGen::CGBuilderTy::CreateLoad
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:69
emitCommonOMPTeamsDirective
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Definition: CGStmtOpenMP.cpp:4470
clang::DeclarationName
The name of a declaration.
Definition: DeclarationName.h:145
clang::isOpenMPSimdDirective
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
Definition: OpenMPKinds.cpp:1055
clang::OMPTaskgroupDirective::getReductionRef
const Expr * getReductionRef() const
Returns reference to the task_reduction return variable.
Definition: StmtOpenMP.h:2245
clang::CodeGen::OMPTaskDataTy::Priority
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
Definition: CGOpenMPRuntime.h:107
convertToType
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
Definition: CGStmtOpenMP.cpp:4145
clang::CodeGen::CodeGenFunction::EmitOMPLinearClause
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
Definition: CGStmtOpenMP.cpp:1770
clang::CodeGen::CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
Definition: CGStmtOpenMP.cpp:4694
clang::CapturedStmt::getEndLoc
SourceLocation getEndLoc() const LLVM_READONLY
Definition: Stmt.h:3572
clang::OMPLinearClause
This represents clause &#39;linear&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:3166
clang::CodeGen::CodeGenFunction::EmitOMPFirstprivateClause
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Definition: CGStmtOpenMP.cpp:742
clang::CodeGen::CodeGenFunction::EmitOMPCancelDirective
void EmitOMPCancelDirective(const OMPCancelDirective &S)
Definition: CGStmtOpenMP.cpp:4850
clang::OMPAtomicDirective::getUpdateExpr
Expr * getUpdateExpr()
Get helper expression of the form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; or &#39;OpaqueValueExp...
Definition: StmtOpenMP.h:2477
clang::CapturedStmt::getBeginLoc
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Stmt.h:3568
clang::CodeGen::CodeGenModule::getModule
llvm::Module & getModule() const
Definition: CodeGenModule.h:681
clang::OMPAtomicDirective
This represents &#39;#pragma omp atomic&#39; directive.
Definition: StmtOpenMP.h:2379
clang::CodeGen::CodeGenFunction::MakeAddrLValue
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
Definition: CodeGenFunction.h:2133
clang::OMPLoopDirective::getCombinedInit
Expr * getCombinedInit() const
Definition: StmtOpenMP.h:1052
clang::OMPLoopDirective::getCollapsedNumber
unsigned getCollapsedNumber() const
Get number of collapsed loops.
Definition: StmtOpenMP.h:912
clang::OMPLoopDirective::getCombinedNextLowerBound
Expr * getCombinedNextLowerBound() const
Definition: StmtOpenMP.h:1064
clang::OMPLoopDirective::counters
ArrayRef< Expr * > counters()
Definition: StmtOpenMP.h:1104
emitTargetTeamsDistributeSimdRegion
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
Definition: CGStmtOpenMP.cpp:4603
clang::CodeGen::ReductionCodeGen::emitInitialization
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
Definition: CGOpenMPRuntime.cpp:1054
clang::CodeGen::CodeGenFunction::EmitOMPReductionClauseFinal
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
Definition: CGStmtOpenMP.cpp:1227
clang::SwitchStmt
SwitchStmt - This represents a &#39;switch&#39; stmt.
Definition: Stmt.h:2043
clang::OMPLoopDirective::getCombinedNextUpperBound
Expr * getCombinedNextUpperBound() const
Definition: StmtOpenMP.h:1070
clang::CodeGen::CodeGenFunction::EmitAutoVarCleanups
void EmitAutoVarCleanups(const AutoVarEmission &emission)
Definition: CGDecl.cpp:1980
clang::CodeGen::CodeGenFunction::CGCapturedStmtInfo
API for captured statement code generation.
Definition: CodeGenFunction.h:347
clang::getOpenMPCaptureRegions
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
Definition: OpenMPKinds.cpp:1111
clang::ComplexType
Complex values, per C99 6.2.5p11.
Definition: Type.h:2554
clang::isOpenMPDistributeDirective
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
Definition: OpenMPKinds.cpp:1076
emitSimpleAtomicStore
static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, LValue LVal, RValue RVal)
Definition: CGStmtOpenMP.cpp:3921
StmtOpenMP.h
This file defines OpenMP AST classes for executable directives and clauses.
clang::OMPLoopDirective::inits
ArrayRef< Expr * > inits()
Definition: StmtOpenMP.h:1116
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDeviceFunction
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
Definition: CGStmtOpenMP.cpp:4535
clang::CodeGen::Address::getElementType
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:51
clang::DeclRefExpr::Create
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr, NonOdrUseReason NOUR=NOUR_None)
Definition: Expr.cpp:545
clang::VarDecl::isStaticLocal
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
Definition: Decl.h:1069
clang::CodeGen::CodeGenFunction::EmitOMPParallelMasterTaskLoopDirective
void EmitOMPParallelMasterTaskLoopDirective(const OMPParallelMasterTaskLoopDirective &S)
Definition: CGStmtOpenMP.cpp:5408
clang::OpenMPScheduleClauseKind
OpenMPScheduleClauseKind
OpenMP attributes for &#39;schedule&#39; clause.
Definition: OpenMPKinds.h:85
clang::OMPLoopDirective::getNumIterations
Expr * getNumIterations() const
Definition: StmtOpenMP.h:1002
clang::ReferenceType
Base for LValueReferenceType and RValueReferenceType.
Definition: Type.h:2750
clang::CodeGen::CodeGenModule::SetInternalFunctionAttributes
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
Definition: CodeGenModule.cpp:1763
clang::CodeGen::CodeGenModule::getMangledName
StringRef getMangledName(GlobalDecl GD)
Definition: CodeGenModule.cpp:1089
clang::CodeGen::CodeGenFunction::OMPTargetDataInfo::BasePointersArray
Address BasePointersArray
Definition: CodeGenFunction.h:3119
clang::InternalLinkage
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
Definition: Linkage.h:31
clang::CodeGen::CodeGenFunction::OMPMapVars::setVarAddr
bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD, Address TempAddr)
Sets the address of the variable LocalVD to be TempAddr in function CGF.
Definition: CodeGenFunction.h:839
clang::CodeGen::CodeGenFunction::EmitBlock
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition: CGStmt.cpp:473
clang::CodeGen::CodeGenFunction::EmitOMPTargetEnterDataDirective
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
Definition: CGStmtOpenMP.cpp:5044
clang::OMPExecutableDirective::hasAssociatedStmt
bool hasAssociatedStmt() const
Returns true if directive has associated statement.
Definition: StmtOpenMP.h:250
clang::OMPLoopDirective::getPrevLowerBoundVariable
Expr * getPrevLowerBoundVariable() const
Definition: StmtOpenMP.h:1010
clang::CodeGen::CodeGenFunction::AutoVarEmission
Definition: CodeGenFunction.h:2699
clang::ASTContext::getTypeSize
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
Definition: ASTContext.h:2104
clang::ASTContext::getSourceManager
SourceManager & getSourceManager()
Definition: ASTContext.h:679
clang::CodeGen::CodeGenFunction::EmitOMPDistributeSimdDirective
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
Definition: CGStmtOpenMP.cpp:2452
clang::CodeGen::CodeGenFunction::CurFuncDecl
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
Definition: CodeGenFunction.h:296
clang::CodeGen::CodeGenFunction::EmitStopPoint
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
Definition: CGStmt.cpp:35
clang::ASTContext::getCanonicalType
CanQualType getCanonicalType(QualType T) const
Return the canonical (structural) type corresponding to the specified potentially non-canonical type ...
Definition: ASTContext.h:2305
clang::Type::isLValueReferenceType
bool isLValueReferenceType() const
Definition: Type.h:6520
X
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
Definition: SemaDecl.cpp:14780
clang::ASTContext::toCharUnitsFromBits
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
Definition: ASTContext.cpp:2372
clang::CapturedStmt::getCapturedDecl
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
Definition: Stmt.cpp:1298
clang::Expr::EvaluateAsInt
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
Definition: ExprConstant.cpp:13770
clang::ASTContext::toBits
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
Definition: ASTContext.cpp:2377
clang::ASTContext::getTranslationUnitDecl
TranslationUnitDecl * getTranslationUnitDecl() const
Definition: ASTContext.h:1009
clang::CodeGen::CodeGenFunction::EmitOMPReductionClauseInit
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for reduction variables.
Definition: CGStmtOpenMP.cpp:1123
clang::OMPTargetParallelDirective
This represents &#39;#pragma omp target parallel&#39; directive.
Definition: StmtOpenMP.h:2748
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeDirective
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
Definition: CGStmtOpenMP.cpp:4595
getCanonicalParamType
static QualType getCanonicalParamType(ASTContext &C, QualType T)
Definition: CGStmtOpenMP.cpp:351
clang::OMPNowaitClause
This represents &#39;nowait&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:1569
clang::CodeGen::OMPTaskDataTy::Schedule
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
Definition: CGOpenMPRuntime.h:106
clang::Type::hasSignedIntegerRepresentation
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g., it is an signed integer type or a vector.
Definition: Type.cpp:1958
clang::CodeGen::CodeGenFunction::EmitBranch
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
Definition: CGStmt.cpp:493
clang::OMPLoopDirective::getBody
Stmt * getBody()
Definition: StmtOpenMP.cpp:89
OMPNumTasksClause
This represents &#39;num_tasks&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:5421
clang::CodeGen::OMPTaskDataTy::NumberOfParts
unsigned NumberOfParts
Definition: CGOpenMPRuntime.h:109
Privates
Privates[]
Gets the list of initial values for linear variables.
Definition: OpenMPClause.h:150
clang::QualType::getQualifiers
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition: Type.h:6283
clang::OMPLoopDirective::getCombinedCond
Expr * getCombinedCond() const
Definition: StmtOpenMP.h:1058
clang::CodeGen::CodeGenFunction::EmitOMPLinearClauseFinal
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
Definition: CGStmtOpenMP.cpp:1609
clang::interp::Add
bool Add(InterpState &S, CodePtr OpPC)
Definition: Interp.h:132
clang::CodeGen::CodeGenFunction::EmitOMPUseDevicePtrClause
void EmitOMPUseDevicePtrClause(const OMPClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, Address > &CaptureDeviceAddrMap)
Definition: CGStmtOpenMP.cpp:4890
clang::CodeGen::CodeGenFunction::EmitLValue
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
Definition: CGExpr.cpp:1246
emitTargetTeamsDistributeParallelForSimdRegion
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
Definition: CGStmtOpenMP.cpp:4792
clang::ASTContext::getPointerType
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
Definition: ASTContext.cpp:3102
clang::OMPC_unknown
Definition: OpenMPKinds.h:73
clang::CodeGen::CallArg
Definition: CGCall.h:216
clang::Stmt::getSourceRange
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Definition: Stmt.cpp:263
clang::CodeGen::CodeGenFunction::EmitLoadOfLValue
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition: CGExpr.cpp:1777
clang::OMPTaskLoopSimdDirective
This represents &#39;#pragma omp taskloop simd&#39; directive.
Definition: StmtOpenMP.h:3137
clang::CodeGen::CodeGenFunction::EmitOMPCopyinClause
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
Definition: CGStmtOpenMP.cpp:905
clang::OMPLoopDirective::getPreCond
Expr * getPreCond() const
Definition: StmtOpenMP.h:926
OMPDistScheduleClause
This represents &#39;dist_schedule&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:5550
clang::CodeGen::CodeGenFunction::CapturedStmtInfo
CGCapturedStmtInfo * CapturedStmtInfo
Definition: CodeGenFunction.h:412
clang::CodeGen::LValue::isGlobalReg
bool isGlobalReg() const
Definition: CGValue.h:256
clang::CodeGen::CodeGenFunction::EmitScalarConversion
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
Definition: CGExprScalar.cpp:4553
clang::ASTContext::getAsVariableArrayType
const VariableArrayType * getAsVariableArrayType(QualType T) const
Definition: ASTContext.h:2469
clang::CR_OpenMP
Definition: CapturedStmt.h:19
clang::OMPSectionsDirective
This represents &#39;#pragma omp sections&#39; directive.
Definition: StmtOpenMP.h:1403
clang::OMPTargetDataDirective
This represents &#39;#pragma omp target data&#39; directive.
Definition: StmtOpenMP.h:2573
clang::CodeGen::CodeGenFunction::EmitOMPTargetExitDataDirective
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
Definition: CGStmtOpenMP.cpp:5065
clang::OMPCancelDirective::getCancelRegion
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:3055
clang::CapturedStmt::captures
capture_range captures()
Definition: Stmt.h:3511
clang::DeclRefExpr
A reference to a declared variable, function, enum, etc.
Definition: Expr.h:1171
clang::CodeGen::RValue::get
static RValue get(llvm::Value *V)
Definition: CGValue.h:86
clang::Type::isPointerType
bool isPointerType() const
Definition: Type.h:6504
AttributeLangSupport::C
Definition: SemaDeclAttr.cpp:45
clang::CodeGen::LoopInfoStack::setVectorizeEnable
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop &#39;vectorize.enable&#39;.
Definition: CGLoopInfo.h:227
clang::CodeGen::CodeGenFunction::EmitOMPTargetDeviceFunction
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
Definition: CGStmtOpenMP.cpp:4449
clang::CodeGen::CodeGenFunction::EmitOMPTargetDataDirective
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
Definition: CGStmtOpenMP.cpp:4955
clang::CodeGen::TEK_Scalar
Definition: CodeGenFunction.h:106
clang::ASTContext::getTypeSizeInChars
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
Definition: ASTContext.cpp:2383
clang::CodeGen::CodeGenFunction::EmitOMPTargetParallelForDeviceFunction
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
Definition: CGStmtOpenMP.cpp:5150
clang::CodeGen::CodeGenFunction::EmitOMPDistributeParallelForDirective
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
Definition: CGStmtOpenMP.cpp:2432
clang::CodeGen::CodeGenFunction::EmitAlignmentAssumption
void EmitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
Definition: CodeGenFunction.cpp:2149
clang::ValueDecl::getType
QualType getType() const
Definition: Decl.h:630
clang::VK_LValue
An l-value expression is a reference to an object with independent storage.
Definition: Specifiers.h:129
clang::CodeGen::CodeGenFunction::OMPTargetDataInfo::SizesArray
Address SizesArray
Definition: CodeGenFunction.h:3121
clang::SourceRange
A trivial tuple used to represent a source range.
Definition: SourceLocation.h:207
clang::CodeGen::LValue
LValue - This represents an lvalue references.
Definition: CGValue.h:167
clang::OMPTaskyieldDirective
This represents &#39;#pragma omp taskyield&#39; directive.
Definition: StmtOpenMP.h:2055
clang::CodeGen::CodeGenFunction::CGM
CodeGenModule & CGM
Definition: CodeGenFunction.h:258
clang::OMPDistributeParallelForSimdDirective
This represents &#39;#pragma omp distribute parallel for simd&#39; composite directive.
Definition: StmtOpenMP.h:3688
clang::ASTContext::BoolTy
CanQualType BoolTy
Definition: ASTContext.h:1017
clang::VariableArrayType
Represents a C array with a specified size that is not an integer-constant-expression.
Definition: Type.h:3039
clang::OMPParallelSectionsDirective
This represents &#39;#pragma omp parallel sections&#39; directive.
Definition: StmtOpenMP.h:1914
clang::CodeGen::CodeGenFunction::getLangOpts
const LangOptions & getLangOpts() const
Definition: CodeGenFunction.h:1640
clang::CodeGen::CodeGenFunction::OMPMapVars::apply
bool apply(CodeGenFunction &CGF)
Applies new addresses to the list of the variables.
Definition: CodeGenFunction.h:867
clang::prec::Level
Level
Definition: OperatorPrecedence.h:26
clang::SourceRange::getBegin
SourceLocation getBegin() const
Definition: SourceLocation.h:216
clang::CapturedStmt::capture_init_end
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
Definition: Stmt.h:3560
clang::CodeGen::CodeGenFunction::EmitOMPTaskDirective
void EmitOMPTaskDirective(const OMPTaskDirective &S)
Definition: CGStmtOpenMP.cpp:3513
clang::Decl::getBodyRBrace
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
Definition: DeclBase.cpp:898
clang::interp::Sub
bool Sub(InterpState &S, CodePtr OpPC)
Definition: Interp.h:140
clang::CodeGen::RValue::isScalar
bool isScalar() const
Definition: CGValue.h:52
clang::OMPTargetParallelForDirective
This represents &#39;#pragma omp target parallel for&#39; directive.
Definition: StmtOpenMP.h:2808
clang::CodeGen::OMPTaskDataTy::FirstprivateInits
SmallVector< const Expr *, 4 > FirstprivateInits
Definition: CGOpenMPRuntime.h:98
clang::Decl::getLocation
SourceLocation getLocation() const
Definition: DeclBase.h:429
OMPUseDevicePtrClause
This represents clause &#39;use_device_ptr&#39; in the &#39;#pragma omp ...&#39; directives.
Definition: OpenMPClause.h:6016
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
Definition: CGStmtOpenMP.cpp:4627
clang::ASTContext::getIntTypeForBitwidth
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
Definition: ASTContext.cpp:10377
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
Definition: CGStmtOpenMP.cpp:4784
clang::Create
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
clang::CodeGen::CodeGenFunction::OMPPrivateScope::Privatize
bool Privatize()
Privatizes local variables previously registered as private.
Definition: CodeGenFunction.h:931
clang::CodeGen::CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
Definition: CGStmtOpenMP.cpp:4820
clang::CodeGen::CodeGenTypes::GetFunctionType
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1556
clang::OMPTaskLoopDirective
This represents &#39;#pragma omp taskloop&#39; directive.
Definition: StmtOpenMP.h:3071