clang  6.0.0
CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit OpenMP nodes as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCleanup.h"
15 #include "CGOpenMPRuntime.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/Stmt.h"
20 #include "clang/AST/StmtOpenMP.h"
21 #include "clang/AST/DeclOpenMP.h"
22 #include "llvm/IR/CallSite.h"
23 using namespace clang;
24 using namespace CodeGen;
25 
26 namespace {
27 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
28 /// for captured expressions.
29 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
30  void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
31  for (const auto *C : S.clauses()) {
32  if (auto *CPI = OMPClauseWithPreInit::get(C)) {
33  if (auto *PreInit = cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
34  for (const auto *I : PreInit->decls()) {
35  if (!I->hasAttr<OMPCaptureNoInitAttr>())
36  CGF.EmitVarDecl(cast<VarDecl>(*I));
37  else {
39  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
40  CGF.EmitAutoVarCleanups(Emission);
41  }
42  }
43  }
44  }
45  }
46  }
47  CodeGenFunction::OMPPrivateScope InlinedShareds;
48 
49  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
50  return CGF.LambdaCaptureFields.lookup(VD) ||
51  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
52  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
53  }
54 
55 public:
56  OMPLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S,
57  bool AsInlined = false, bool EmitPreInitStmt = true)
58  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
59  InlinedShareds(CGF) {
60  if (EmitPreInitStmt)
61  emitPreInitStmt(CGF, S);
62  if (AsInlined) {
63  if (S.hasAssociatedStmt()) {
64  auto *CS = cast<CapturedStmt>(S.getAssociatedStmt());
65  for (auto &C : CS->captures()) {
66  if (C.capturesVariable() || C.capturesVariableByCopy()) {
67  auto *VD = C.getCapturedVar();
68  assert(VD == VD->getCanonicalDecl() &&
69  "Canonical decl must be captured.");
70  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
71  isCapturedVar(CGF, VD) ||
72  (CGF.CapturedStmtInfo &&
73  InlinedShareds.isGlobalVarCaptured(VD)),
75  SourceLocation());
76  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
77  return CGF.EmitLValue(&DRE).getAddress();
78  });
79  }
80  }
81  (void)InlinedShareds.Privatize();
82  }
83  }
84  }
85 };
86 
87 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
88 /// for captured expressions.
89 class OMPParallelScope final : public OMPLexicalScope {
90  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
92  return !(isOpenMPTargetExecutionDirective(Kind) ||
95  }
96 
97 public:
98  OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
99  : OMPLexicalScope(CGF, S,
100  /*AsInlined=*/false,
101  /*EmitPreInitStmt=*/EmitPreInitStmt(S)) {}
102 };
103 
104 /// Lexical scope for OpenMP teams construct, that handles correct codegen
105 /// for captured expressions.
106 class OMPTeamsScope final : public OMPLexicalScope {
107  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
109  return !isOpenMPTargetExecutionDirective(Kind) &&
111  }
112 
113 public:
114  OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
115  : OMPLexicalScope(CGF, S,
116  /*AsInlined=*/false,
117  /*EmitPreInitStmt=*/EmitPreInitStmt(S)) {}
118 };
119 
120 /// Private scope for OpenMP loop-based directives, that supports capturing
121 /// of used expression from loop statement.
122 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
123  void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
124  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
125  for (auto *E : S.counters()) {
126  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
127  (void)PreCondScope.addPrivate(VD, [&CGF, VD]() {
128  return CGF.CreateMemTemp(VD->getType().getNonReferenceType());
129  });
130  }
131  (void)PreCondScope.Privatize();
132  if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
133  if (auto *PreInits = cast_or_null<DeclStmt>(LD->getPreInits())) {
134  for (const auto *I : PreInits->decls())
135  CGF.EmitVarDecl(cast<VarDecl>(*I));
136  }
137  }
138  }
139 
140 public:
141  OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
142  : CodeGenFunction::RunCleanupsScope(CGF) {
143  emitPreInitStmt(CGF, S);
144  }
145 };
146 
147 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
148  CodeGenFunction::OMPPrivateScope InlinedShareds;
149 
150  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
151  return CGF.LambdaCaptureFields.lookup(VD) ||
152  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
153  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
154  cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
155  }
156 
157 public:
158  OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
159  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
160  InlinedShareds(CGF) {
161  for (const auto *C : S.clauses()) {
162  if (auto *CPI = OMPClauseWithPreInit::get(C)) {
163  if (auto *PreInit = cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
164  for (const auto *I : PreInit->decls()) {
165  if (!I->hasAttr<OMPCaptureNoInitAttr>())
166  CGF.EmitVarDecl(cast<VarDecl>(*I));
167  else {
169  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
170  CGF.EmitAutoVarCleanups(Emission);
171  }
172  }
173  }
174  } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
175  for (const Expr *E : UDP->varlists()) {
176  const Decl *D = cast<DeclRefExpr>(E)->getDecl();
177  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
178  CGF.EmitVarDecl(*OED);
179  }
180  }
181  }
183  CGF.EmitOMPPrivateClause(S, InlinedShareds);
184  if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
185  if (const Expr *E = TG->getReductionRef())
186  CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
187  }
188  const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
189  while (CS) {
190  for (auto &C : CS->captures()) {
191  if (C.capturesVariable() || C.capturesVariableByCopy()) {
192  auto *VD = C.getCapturedVar();
193  assert(VD == VD->getCanonicalDecl() &&
194  "Canonical decl must be captured.");
195  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
196  isCapturedVar(CGF, VD) ||
197  (CGF.CapturedStmtInfo &&
198  InlinedShareds.isGlobalVarCaptured(VD)),
200  SourceLocation());
201  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
202  return CGF.EmitLValue(&DRE).getAddress();
203  });
204  }
205  }
206  CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
207  }
208  (void)InlinedShareds.Privatize();
209  }
210 };
211 
212 } // namespace
213 
215  const OMPExecutableDirective &S,
216  const RegionCodeGenTy &CodeGen);
217 
219  if (auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
220  if (auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
221  OrigVD = OrigVD->getCanonicalDecl();
222  bool IsCaptured =
223  LambdaCaptureFields.lookup(OrigVD) ||
224  (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
225  (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
226  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), IsCaptured,
227  OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
228  return EmitLValue(&DRE);
229  }
230  }
231  return EmitLValue(E);
232 }
233 
235  auto &C = getContext();
236  llvm::Value *Size = nullptr;
237  auto SizeInChars = C.getTypeSizeInChars(Ty);
238  if (SizeInChars.isZero()) {
239  // getTypeSizeInChars() returns 0 for a VLA.
240  while (auto *VAT = C.getAsVariableArrayType(Ty)) {
241  llvm::Value *ArraySize;
242  std::tie(ArraySize, Ty) = getVLASize(VAT);
243  Size = Size ? Builder.CreateNUWMul(Size, ArraySize) : ArraySize;
244  }
245  SizeInChars = C.getTypeSizeInChars(Ty);
246  if (SizeInChars.isZero())
247  return llvm::ConstantInt::get(SizeTy, /*V=*/0);
248  Size = Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
249  } else
250  Size = CGM.getSize(SizeInChars);
251  return Size;
252 }
253 
255  const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
256  const RecordDecl *RD = S.getCapturedRecordDecl();
257  auto CurField = RD->field_begin();
258  auto CurCap = S.captures().begin();
260  E = S.capture_init_end();
261  I != E; ++I, ++CurField, ++CurCap) {
262  if (CurField->hasCapturedVLAType()) {
263  auto VAT = CurField->getCapturedVLAType();
264  auto *Val = VLASizeMap[VAT->getSizeExpr()];
265  CapturedVars.push_back(Val);
266  } else if (CurCap->capturesThis())
267  CapturedVars.push_back(CXXThisValue);
268  else if (CurCap->capturesVariableByCopy()) {
269  llvm::Value *CV =
270  EmitLoadOfLValue(EmitLValue(*I), SourceLocation()).getScalarVal();
271 
272  // If the field is not a pointer, we need to save the actual value
273  // and load it as a void pointer.
274  if (!CurField->getType()->isAnyPointerType()) {
275  auto &Ctx = getContext();
276  auto DstAddr = CreateMemTemp(
277  Ctx.getUIntPtrType(),
278  Twine(CurCap->getCapturedVar()->getName()) + ".casted");
279  LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
280 
281  auto *SrcAddrVal = EmitScalarConversion(
282  DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
283  Ctx.getPointerType(CurField->getType()), SourceLocation());
284  LValue SrcLV =
285  MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
286 
287  // Store the value using the source type pointer.
288  EmitStoreThroughLValue(RValue::get(CV), SrcLV);
289 
290  // Load the value using the destination type pointer.
291  CV = EmitLoadOfLValue(DstLV, SourceLocation()).getScalarVal();
292  }
293  CapturedVars.push_back(CV);
294  } else {
295  assert(CurCap->capturesVariable() && "Expected capture by reference.");
296  CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
297  }
298  }
299 }
300 
302  StringRef Name, LValue AddrLV,
303  bool isReferenceType = false) {
304  ASTContext &Ctx = CGF.getContext();
305 
306  auto *CastedPtr = CGF.EmitScalarConversion(
307  AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
308  Ctx.getPointerType(DstType), SourceLocation());
309  auto TmpAddr =
310  CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
311  .getAddress();
312 
313  // If we are dealing with references we need to return the address of the
314  // reference instead of the reference of the value.
315  if (isReferenceType) {
316  QualType RefType = Ctx.getLValueReferenceType(DstType);
317  auto *RefVal = TmpAddr.getPointer();
318  TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name) + ".ref");
319  auto TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
320  CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit*/ true);
321  }
322 
323  return TmpAddr;
324 }
325 
327  if (T->isLValueReferenceType()) {
328  return C.getLValueReferenceType(
330  /*SpelledAsLValue=*/false);
331  }
332  if (T->isPointerType())
334  if (auto *A = T->getAsArrayTypeUnsafe()) {
335  if (auto *VLA = dyn_cast<VariableArrayType>(A))
336  return getCanonicalParamType(C, VLA->getElementType());
337  else if (!A->isVariablyModifiedType())
338  return C.getCanonicalType(T);
339  }
340  return C.getCanonicalParamType(T);
341 }
342 
343 namespace {
344  /// Contains required data for proper outlined function codegen.
345  struct FunctionOptions {
346  /// Captured statement for which the function is generated.
347  const CapturedStmt *S = nullptr;
348  /// true if cast to/from UIntPtr is required for variables captured by
349  /// value.
350  const bool UIntPtrCastRequired = true;
351  /// true if only casted arguments must be registered as local args or VLA
352  /// sizes.
353  const bool RegisterCastedArgsOnly = false;
354  /// Name of the generated function.
355  const StringRef FunctionName;
356  explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
357  bool RegisterCastedArgsOnly,
358  StringRef FunctionName)
359  : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
360  RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
361  FunctionName(FunctionName) {}
362  };
363 }
364 
365 static llvm::Function *emitOutlinedFunctionPrologue(
366  CodeGenFunction &CGF, FunctionArgList &Args,
367  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
368  &LocalAddrs,
369  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
370  &VLASizes,
371  llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
372  const CapturedDecl *CD = FO.S->getCapturedDecl();
373  const RecordDecl *RD = FO.S->getCapturedRecordDecl();
374  assert(CD->hasBody() && "missing CapturedDecl body");
375 
376  CXXThisValue = nullptr;
377  // Build the argument list.
378  CodeGenModule &CGM = CGF.CGM;
379  ASTContext &Ctx = CGM.getContext();
380  FunctionArgList TargetArgs;
381  Args.append(CD->param_begin(),
382  std::next(CD->param_begin(), CD->getContextParamPosition()));
383  TargetArgs.append(
384  CD->param_begin(),
385  std::next(CD->param_begin(), CD->getContextParamPosition()));
386  auto I = FO.S->captures().begin();
387  FunctionDecl *DebugFunctionDecl = nullptr;
388  if (!FO.UIntPtrCastRequired) {
390  DebugFunctionDecl = FunctionDecl::Create(
391  Ctx, Ctx.getTranslationUnitDecl(), FO.S->getLocStart(),
394  Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI)),
395  SC_Static, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
396  }
397  for (auto *FD : RD->fields()) {
398  QualType ArgType = FD->getType();
399  IdentifierInfo *II = nullptr;
400  VarDecl *CapVar = nullptr;
401 
402  // If this is a capture by copy and the type is not a pointer, the outlined
403  // function argument type should be uintptr and the value properly casted to
404  // uintptr. This is necessary given that the runtime library is only able to
405  // deal with pointers. We can pass in the same way the VLA type sizes to the
406  // outlined function.
407  if ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
408  I->capturesVariableArrayType()) {
409  if (FO.UIntPtrCastRequired)
410  ArgType = Ctx.getUIntPtrType();
411  }
412 
413  if (I->capturesVariable() || I->capturesVariableByCopy()) {
414  CapVar = I->getCapturedVar();
415  II = CapVar->getIdentifier();
416  } else if (I->capturesThis())
417  II = &Ctx.Idents.get("this");
418  else {
419  assert(I->capturesVariableArrayType());
420  II = &Ctx.Idents.get("vla");
421  }
422  if (ArgType->isVariablyModifiedType())
423  ArgType = getCanonicalParamType(Ctx, ArgType);
424  VarDecl *Arg;
425  if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
426  Arg = ParmVarDecl::Create(
427  Ctx, DebugFunctionDecl,
428  CapVar ? CapVar->getLocStart() : FD->getLocStart(),
429  CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
430  /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
431  } else {
432  Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
433  II, ArgType, ImplicitParamDecl::Other);
434  }
435  Args.emplace_back(Arg);
436  // Do not cast arguments if we emit function with non-original types.
437  TargetArgs.emplace_back(
438  FO.UIntPtrCastRequired
439  ? Arg
440  : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
441  ++I;
442  }
443  Args.append(
444  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
445  CD->param_end());
446  TargetArgs.append(
447  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
448  CD->param_end());
449 
450  // Create the function declaration.
451  const CGFunctionInfo &FuncInfo =
452  CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
453  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
454 
455  llvm::Function *F =
457  FO.FunctionName, &CGM.getModule());
458  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
459  if (CD->isNothrow())
460  F->setDoesNotThrow();
461 
462  // Generate the function.
463  CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
464  FO.S->getLocStart(), CD->getBody()->getLocStart());
465  unsigned Cnt = CD->getContextParamPosition();
466  I = FO.S->captures().begin();
467  for (auto *FD : RD->fields()) {
468  // Do not map arguments if we emit function with non-original types.
469  Address LocalAddr(Address::invalid());
470  if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
471  LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
472  TargetArgs[Cnt]);
473  } else {
474  LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
475  }
476  // If we are capturing a pointer by copy we don't need to do anything, just
477  // use the value that we get from the arguments.
478  if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
479  const VarDecl *CurVD = I->getCapturedVar();
480  // If the variable is a reference we need to materialize it here.
481  if (CurVD->getType()->isReferenceType()) {
482  Address RefAddr = CGF.CreateMemTemp(
483  CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
484  CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
485  /*Volatile=*/false, CurVD->getType());
486  LocalAddr = RefAddr;
487  }
488  if (!FO.RegisterCastedArgsOnly)
489  LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
490  ++Cnt;
491  ++I;
492  continue;
493  }
494 
495  LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
497  if (FD->hasCapturedVLAType()) {
498  if (FO.UIntPtrCastRequired) {
499  ArgLVal = CGF.MakeAddrLValue(castValueFromUintptr(CGF, FD->getType(),
500  Args[Cnt]->getName(),
501  ArgLVal),
503  }
504  auto *ExprArg =
505  CGF.EmitLoadOfLValue(ArgLVal, SourceLocation()).getScalarVal();
506  auto VAT = FD->getCapturedVLAType();
507  VLASizes.insert({Args[Cnt], {VAT->getSizeExpr(), ExprArg}});
508  } else if (I->capturesVariable()) {
509  auto *Var = I->getCapturedVar();
510  QualType VarTy = Var->getType();
511  Address ArgAddr = ArgLVal.getAddress();
512  if (!VarTy->isReferenceType()) {
513  if (ArgLVal.getType()->isLValueReferenceType()) {
514  ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
515  } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
516  assert(ArgLVal.getType()->isPointerType());
517  ArgAddr = CGF.EmitLoadOfPointer(
518  ArgAddr, ArgLVal.getType()->castAs<PointerType>());
519  }
520  }
521  if (!FO.RegisterCastedArgsOnly) {
522  LocalAddrs.insert(
523  {Args[Cnt],
524  {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
525  }
526  } else if (I->capturesVariableByCopy()) {
527  assert(!FD->getType()->isAnyPointerType() &&
528  "Not expecting a captured pointer.");
529  auto *Var = I->getCapturedVar();
530  QualType VarTy = Var->getType();
531  LocalAddrs.insert(
532  {Args[Cnt],
533  {Var,
534  FO.UIntPtrCastRequired
535  ? castValueFromUintptr(CGF, FD->getType(), Args[Cnt]->getName(),
536  ArgLVal, VarTy->isReferenceType())
537  : ArgLVal.getAddress()}});
538  } else {
539  // If 'this' is captured, load it into CXXThisValue.
540  assert(I->capturesThis());
541  CXXThisValue = CGF.EmitLoadOfLValue(ArgLVal, Args[Cnt]->getLocation())
542  .getScalarVal();
543  LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
544  }
545  ++Cnt;
546  ++I;
547  }
548 
549  return F;
550 }
551 
552 llvm::Function *
554  assert(
555  CapturedStmtInfo &&
556  "CapturedStmtInfo should be set when generating the captured function");
557  const CapturedDecl *CD = S.getCapturedDecl();
558  // Build the argument list.
559  bool NeedWrapperFunction =
560  getDebugInfo() &&
561  CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
562  FunctionArgList Args;
563  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
564  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
565  SmallString<256> Buffer;
566  llvm::raw_svector_ostream Out(Buffer);
567  Out << CapturedStmtInfo->getHelperName();
568  if (NeedWrapperFunction)
569  Out << "_debug__";
570  FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
571  Out.str());
572  llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
573  VLASizes, CXXThisValue, FO);
574  for (const auto &LocalAddrPair : LocalAddrs) {
575  if (LocalAddrPair.second.first) {
576  setAddrOfLocalVar(LocalAddrPair.second.first,
577  LocalAddrPair.second.second);
578  }
579  }
580  for (const auto &VLASizePair : VLASizes)
581  VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
582  PGO.assignRegionCounters(GlobalDecl(CD), F);
583  CapturedStmtInfo->EmitBody(*this, CD->getBody());
584  FinishFunction(CD->getBodyRBrace());
585  if (!NeedWrapperFunction)
586  return F;
587 
588  FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
589  /*RegisterCastedArgsOnly=*/true,
590  CapturedStmtInfo->getHelperName());
591  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
592  Args.clear();
593  LocalAddrs.clear();
594  VLASizes.clear();
595  llvm::Function *WrapperF =
596  emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
597  WrapperCGF.CXXThisValue, WrapperFO);
599  for (const auto *Arg : Args) {
601  auto I = LocalAddrs.find(Arg);
602  if (I != LocalAddrs.end()) {
603  LValue LV = WrapperCGF.MakeAddrLValue(
604  I->second.second,
605  I->second.first ? I->second.first->getType() : Arg->getType(),
607  CallArg = WrapperCGF.EmitLoadOfScalar(LV, SourceLocation());
608  } else {
609  auto EI = VLASizes.find(Arg);
610  if (EI != VLASizes.end())
611  CallArg = EI->second.second;
612  else {
613  LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
614  Arg->getType(),
616  CallArg = WrapperCGF.EmitLoadOfScalar(LV, SourceLocation());
617  }
618  }
619  CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
620  }
621  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getLocStart(),
622  F, CallArgs);
623  WrapperCGF.FinishFunction();
624  return WrapperF;
625 }
626 
627 //===----------------------------------------------------------------------===//
628 // OpenMP Directive Emission
629 //===----------------------------------------------------------------------===//
631  Address DestAddr, Address SrcAddr, QualType OriginalType,
632  const llvm::function_ref<void(Address, Address)> &CopyGen) {
633  // Perform element-by-element initialization.
634  QualType ElementTy;
635 
636  // Drill down to the base element type on both arrays.
637  auto ArrayTy = OriginalType->getAsArrayTypeUnsafe();
638  auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
639  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
640 
641  auto SrcBegin = SrcAddr.getPointer();
642  auto DestBegin = DestAddr.getPointer();
643  // Cast from pointer to array type to pointer to single element.
644  auto DestEnd = Builder.CreateGEP(DestBegin, NumElements);
645  // The basic structure here is a while-do loop.
646  auto BodyBB = createBasicBlock("omp.arraycpy.body");
647  auto DoneBB = createBasicBlock("omp.arraycpy.done");
648  auto IsEmpty =
649  Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
650  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
651 
652  // Enter the loop body, making that address the current address.
653  auto EntryBB = Builder.GetInsertBlock();
654  EmitBlock(BodyBB);
655 
656  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
657 
658  llvm::PHINode *SrcElementPHI =
659  Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
660  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
661  Address SrcElementCurrent =
662  Address(SrcElementPHI,
663  SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
664 
665  llvm::PHINode *DestElementPHI =
666  Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
667  DestElementPHI->addIncoming(DestBegin, EntryBB);
668  Address DestElementCurrent =
669  Address(DestElementPHI,
670  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
671 
672  // Emit copy.
673  CopyGen(DestElementCurrent, SrcElementCurrent);
674 
675  // Shift the address forward by one element.
676  auto DestElementNext = Builder.CreateConstGEP1_32(
677  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
678  auto SrcElementNext = Builder.CreateConstGEP1_32(
679  SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
680  // Check whether we've reached the end.
681  auto Done =
682  Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
683  Builder.CreateCondBr(Done, DoneBB, BodyBB);
684  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
685  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
686 
687  // Done.
688  EmitBlock(DoneBB, /*IsFinished=*/true);
689 }
690 
691 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
692  Address SrcAddr, const VarDecl *DestVD,
693  const VarDecl *SrcVD, const Expr *Copy) {
694  if (OriginalType->isArrayType()) {
695  auto *BO = dyn_cast<BinaryOperator>(Copy);
696  if (BO && BO->getOpcode() == BO_Assign) {
697  // Perform simple memcpy for simple copying.
698  EmitAggregateAssign(DestAddr, SrcAddr, OriginalType);
699  } else {
700  // For arrays with complex element types perform element by element
701  // copying.
702  EmitOMPAggregateAssign(
703  DestAddr, SrcAddr, OriginalType,
704  [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
705  // Working with the single array element, so have to remap
706  // destination and source variables to corresponding array
707  // elements.
709  Remap.addPrivate(DestVD, [DestElement]() -> Address {
710  return DestElement;
711  });
712  Remap.addPrivate(
713  SrcVD, [SrcElement]() -> Address { return SrcElement; });
714  (void)Remap.Privatize();
715  EmitIgnoredExpr(Copy);
716  });
717  }
718  } else {
719  // Remap pseudo source variable to private copy.
721  Remap.addPrivate(SrcVD, [SrcAddr]() -> Address { return SrcAddr; });
722  Remap.addPrivate(DestVD, [DestAddr]() -> Address { return DestAddr; });
723  (void)Remap.Privatize();
724  // Emit copying of the whole variable.
725  EmitIgnoredExpr(Copy);
726  }
727 }
728 
730  OMPPrivateScope &PrivateScope) {
731  if (!HaveInsertPoint())
732  return false;
733  bool FirstprivateIsLastprivate = false;
734  llvm::DenseSet<const VarDecl *> Lastprivates;
735  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
736  for (const auto *D : C->varlists())
737  Lastprivates.insert(
738  cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
739  }
740  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
741  CGCapturedStmtInfo CapturesInfo(cast<CapturedStmt>(*D.getAssociatedStmt()));
742  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
743  auto IRef = C->varlist_begin();
744  auto InitsRef = C->inits().begin();
745  for (auto IInit : C->private_copies()) {
746  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
747  bool ThisFirstprivateIsLastprivate =
748  Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
749  auto *CapFD = CapturesInfo.lookup(OrigVD);
750  auto *FD = CapturedStmtInfo->lookup(OrigVD);
751  if (!ThisFirstprivateIsLastprivate && FD && (FD == CapFD) &&
752  !FD->getType()->isReferenceType()) {
753  EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
754  ++IRef;
755  ++InitsRef;
756  continue;
757  }
758  FirstprivateIsLastprivate =
759  FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
760  if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
761  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
762  auto *VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
763  bool IsRegistered;
764  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
765  /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
766  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
767  Address OriginalAddr = EmitLValue(&DRE).getAddress();
768  QualType Type = VD->getType();
769  if (Type->isArrayType()) {
770  // Emit VarDecl with copy init for arrays.
771  // Get the address of the original variable captured in current
772  // captured region.
773  IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
774  auto Emission = EmitAutoVarAlloca(*VD);
775  auto *Init = VD->getInit();
776  if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
777  // Perform simple memcpy.
778  EmitAggregateAssign(Emission.getAllocatedAddress(), OriginalAddr,
779  Type);
780  } else {
781  EmitOMPAggregateAssign(
782  Emission.getAllocatedAddress(), OriginalAddr, Type,
783  [this, VDInit, Init](Address DestElement,
784  Address SrcElement) {
785  // Clean up any temporaries needed by the initialization.
786  RunCleanupsScope InitScope(*this);
787  // Emit initialization for single element.
788  setAddrOfLocalVar(VDInit, SrcElement);
789  EmitAnyExprToMem(Init, DestElement,
790  Init->getType().getQualifiers(),
791  /*IsInitializer*/ false);
792  LocalDeclMap.erase(VDInit);
793  });
794  }
795  EmitAutoVarCleanups(Emission);
796  return Emission.getAllocatedAddress();
797  });
798  } else {
799  IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
800  // Emit private VarDecl with copy init.
801  // Remap temp VDInit variable to the address of the original
802  // variable
803  // (for proper handling of captured global variables).
804  setAddrOfLocalVar(VDInit, OriginalAddr);
805  EmitDecl(*VD);
806  LocalDeclMap.erase(VDInit);
807  return GetAddrOfLocalVar(VD);
808  });
809  }
810  assert(IsRegistered &&
811  "firstprivate var already registered as private");
812  // Silence the warning about unused variable.
813  (void)IsRegistered;
814  }
815  ++IRef;
816  ++InitsRef;
817  }
818  }
819  return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
820 }
821 
823  const OMPExecutableDirective &D,
824  CodeGenFunction::OMPPrivateScope &PrivateScope) {
825  if (!HaveInsertPoint())
826  return;
827  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
828  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
829  auto IRef = C->varlist_begin();
830  for (auto IInit : C->private_copies()) {
831  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
832  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
833  auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
834  bool IsRegistered =
835  PrivateScope.addPrivate(OrigVD, [&]() -> Address {
836  // Emit private VarDecl with copy init.
837  EmitDecl(*VD);
838  return GetAddrOfLocalVar(VD);
839  });
840  assert(IsRegistered && "private var already registered as private");
841  // Silence the warning about unused variable.
842  (void)IsRegistered;
843  }
844  ++IRef;
845  }
846  }
847 }
848 
850  if (!HaveInsertPoint())
851  return false;
852  // threadprivate_var1 = master_threadprivate_var1;
853  // operator=(threadprivate_var2, master_threadprivate_var2);
854  // ...
855  // __kmpc_barrier(&loc, global_tid);
857  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
858  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
859  auto IRef = C->varlist_begin();
860  auto ISrcRef = C->source_exprs().begin();
861  auto IDestRef = C->destination_exprs().begin();
862  for (auto *AssignOp : C->assignment_ops()) {
863  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
864  QualType Type = VD->getType();
865  if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
866  // Get the address of the master variable. If we are emitting code with
867  // TLS support, the address is passed from the master as field in the
868  // captured declaration.
869  Address MasterAddr = Address::invalid();
870  if (getLangOpts().OpenMPUseTLS &&
871  getContext().getTargetInfo().isTLSSupported()) {
872  assert(CapturedStmtInfo->lookup(VD) &&
873  "Copyin threadprivates should have been captured!");
874  DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(),
875  VK_LValue, (*IRef)->getExprLoc());
876  MasterAddr = EmitLValue(&DRE).getAddress();
877  LocalDeclMap.erase(VD);
878  } else {
879  MasterAddr =
880  Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
881  : CGM.GetAddrOfGlobal(VD),
882  getContext().getDeclAlign(VD));
883  }
884  // Get the address of the threadprivate variable.
885  Address PrivateAddr = EmitLValue(*IRef).getAddress();
886  if (CopiedVars.size() == 1) {
887  // At first check if current thread is a master thread. If it is, no
888  // need to copy data.
889  CopyBegin = createBasicBlock("copyin.not.master");
890  CopyEnd = createBasicBlock("copyin.not.master.end");
891  Builder.CreateCondBr(
892  Builder.CreateICmpNE(
893  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
894  Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)),
895  CopyBegin, CopyEnd);
896  EmitBlock(CopyBegin);
897  }
898  auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
899  auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
900  EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
901  }
902  ++IRef;
903  ++ISrcRef;
904  ++IDestRef;
905  }
906  }
907  if (CopyEnd) {
908  // Exit out of copying procedure for non-master thread.
909  EmitBlock(CopyEnd, /*IsFinished=*/true);
910  return true;
911  }
912  return false;
913 }
914 
916  const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
917  if (!HaveInsertPoint())
918  return false;
919  bool HasAtLeastOneLastprivate = false;
922  auto *LoopDirective = cast<OMPLoopDirective>(&D);
923  for (auto *C : LoopDirective->counters()) {
924  SIMDLCVs.insert(
925  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
926  }
927  }
928  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
929  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
930  HasAtLeastOneLastprivate = true;
932  !getLangOpts().OpenMPSimd)
933  break;
934  auto IRef = C->varlist_begin();
935  auto IDestRef = C->destination_exprs().begin();
936  for (auto *IInit : C->private_copies()) {
937  // Keep the address of the original variable for future update at the end
938  // of the loop.
939  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
940  // Taskloops do not require additional initialization, it is done in
941  // runtime support library.
942  if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
943  auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
944  PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> Address {
945  DeclRefExpr DRE(
946  const_cast<VarDecl *>(OrigVD),
947  /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
948  OrigVD) != nullptr,
949  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
950  return EmitLValue(&DRE).getAddress();
951  });
952  // Check if the variable is also a firstprivate: in this case IInit is
953  // not generated. Initialization of this variable will happen in codegen
954  // for 'firstprivate' clause.
955  if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
956  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
957  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
958  // Emit private VarDecl with copy init.
959  EmitDecl(*VD);
960  return GetAddrOfLocalVar(VD);
961  });
962  assert(IsRegistered &&
963  "lastprivate var already registered as private");
964  (void)IsRegistered;
965  }
966  }
967  ++IRef;
968  ++IDestRef;
969  }
970  }
971  return HasAtLeastOneLastprivate;
972 }
973 
975  const OMPExecutableDirective &D, bool NoFinals,
976  llvm::Value *IsLastIterCond) {
977  if (!HaveInsertPoint())
978  return;
979  // Emit following code:
980  // if (<IsLastIterCond>) {
981  // orig_var1 = private_orig_var1;
982  // ...
983  // orig_varn = private_orig_varn;
984  // }
985  llvm::BasicBlock *ThenBB = nullptr;
986  llvm::BasicBlock *DoneBB = nullptr;
987  if (IsLastIterCond) {
988  ThenBB = createBasicBlock(".omp.lastprivate.then");
989  DoneBB = createBasicBlock(".omp.lastprivate.done");
990  Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
991  EmitBlock(ThenBB);
992  }
993  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
994  llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
995  if (auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
996  auto IC = LoopDirective->counters().begin();
997  for (auto F : LoopDirective->finals()) {
998  auto *D =
999  cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1000  if (NoFinals)
1001  AlreadyEmittedVars.insert(D);
1002  else
1003  LoopCountersAndUpdates[D] = F;
1004  ++IC;
1005  }
1006  }
1007  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1008  auto IRef = C->varlist_begin();
1009  auto ISrcRef = C->source_exprs().begin();
1010  auto IDestRef = C->destination_exprs().begin();
1011  for (auto *AssignOp : C->assignment_ops()) {
1012  auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1013  QualType Type = PrivateVD->getType();
1014  auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1015  if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1016  // If lastprivate variable is a loop control variable for loop-based
1017  // directive, update its value before copyin back to original
1018  // variable.
1019  if (auto *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1020  EmitIgnoredExpr(FinalExpr);
1021  auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1022  auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1023  // Get the address of the original variable.
1024  Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1025  // Get the address of the private variable.
1026  Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1027  if (auto RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1028  PrivateAddr =
1029  Address(Builder.CreateLoad(PrivateAddr),
1030  getNaturalTypeAlignment(RefTy->getPointeeType()));
1031  EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1032  }
1033  ++IRef;
1034  ++ISrcRef;
1035  ++IDestRef;
1036  }
1037  if (auto *PostUpdate = C->getPostUpdateExpr())
1038  EmitIgnoredExpr(PostUpdate);
1039  }
1040  if (IsLastIterCond)
1041  EmitBlock(DoneBB, /*IsFinished=*/true);
1042 }
1043 
1045  const OMPExecutableDirective &D,
1046  CodeGenFunction::OMPPrivateScope &PrivateScope) {
1047  if (!HaveInsertPoint())
1048  return;
1051  SmallVector<const Expr *, 4> ReductionOps;
1054  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1055  auto IPriv = C->privates().begin();
1056  auto IRed = C->reduction_ops().begin();
1057  auto ILHS = C->lhs_exprs().begin();
1058  auto IRHS = C->rhs_exprs().begin();
1059  for (const auto *Ref : C->varlists()) {
1060  Shareds.emplace_back(Ref);
1061  Privates.emplace_back(*IPriv);
1062  ReductionOps.emplace_back(*IRed);
1063  LHSs.emplace_back(*ILHS);
1064  RHSs.emplace_back(*IRHS);
1065  std::advance(IPriv, 1);
1066  std::advance(IRed, 1);
1067  std::advance(ILHS, 1);
1068  std::advance(IRHS, 1);
1069  }
1070  }
1071  ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1072  unsigned Count = 0;
1073  auto ILHS = LHSs.begin();
1074  auto IRHS = RHSs.begin();
1075  auto IPriv = Privates.begin();
1076  for (const auto *IRef : Shareds) {
1077  auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1078  // Emit private VarDecl with reduction init.
1079  RedCG.emitSharedLValue(*this, Count);
1080  RedCG.emitAggregateType(*this, Count);
1081  auto Emission = EmitAutoVarAlloca(*PrivateVD);
1082  RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1083  RedCG.getSharedLValue(Count),
1084  [&Emission](CodeGenFunction &CGF) {
1085  CGF.EmitAutoVarInit(Emission);
1086  return true;
1087  });
1088  EmitAutoVarCleanups(Emission);
1089  Address BaseAddr = RedCG.adjustPrivateAddress(
1090  *this, Count, Emission.getAllocatedAddress());
1091  bool IsRegistered = PrivateScope.addPrivate(
1092  RedCG.getBaseDecl(Count), [BaseAddr]() -> Address { return BaseAddr; });
1093  assert(IsRegistered && "private var already registered as private");
1094  // Silence the warning about unused variable.
1095  (void)IsRegistered;
1096 
1097  auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1098  auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1099  QualType Type = PrivateVD->getType();
1100  bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1101  if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1102  // Store the address of the original variable associated with the LHS
1103  // implicit variable.
1104  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() -> Address {
1105  return RedCG.getSharedLValue(Count).getAddress();
1106  });
1107  PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address {
1108  return GetAddrOfLocalVar(PrivateVD);
1109  });
1110  } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1111  isa<ArraySubscriptExpr>(IRef)) {
1112  // Store the address of the original variable associated with the LHS
1113  // implicit variable.
1114  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() -> Address {
1115  return RedCG.getSharedLValue(Count).getAddress();
1116  });
1117  PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() -> Address {
1118  return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1119  ConvertTypeForMem(RHSVD->getType()),
1120  "rhs.begin");
1121  });
1122  } else {
1123  QualType Type = PrivateVD->getType();
1124  bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1125  Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1126  // Store the address of the original variable associated with the LHS
1127  // implicit variable.
1128  if (IsArray) {
1129  OriginalAddr = Builder.CreateElementBitCast(
1130  OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1131  }
1132  PrivateScope.addPrivate(
1133  LHSVD, [OriginalAddr]() -> Address { return OriginalAddr; });
1134  PrivateScope.addPrivate(
1135  RHSVD, [this, PrivateVD, RHSVD, IsArray]() -> Address {
1136  return IsArray
1137  ? Builder.CreateElementBitCast(
1138  GetAddrOfLocalVar(PrivateVD),
1139  ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1140  : GetAddrOfLocalVar(PrivateVD);
1141  });
1142  }
1143  ++ILHS;
1144  ++IRHS;
1145  ++IPriv;
1146  ++Count;
1147  }
1148 }
1149 
1151  const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1152  if (!HaveInsertPoint())
1153  return;
1158  bool HasAtLeastOneReduction = false;
1159  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1160  HasAtLeastOneReduction = true;
1161  Privates.append(C->privates().begin(), C->privates().end());
1162  LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1163  RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1164  ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1165  }
1166  if (HasAtLeastOneReduction) {
1167  bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1169  ReductionKind == OMPD_simd;
1170  bool SimpleReduction = ReductionKind == OMPD_simd;
1171  // Emit nowait reduction if nowait clause is present or directive is a
1172  // parallel directive (it always has implicit barrier).
1173  CGM.getOpenMPRuntime().emitReduction(
1174  *this, D.getLocEnd(), Privates, LHSExprs, RHSExprs, ReductionOps,
1175  {WithNowait, SimpleReduction, ReductionKind});
1176  }
1177 }
1178 
1180  CodeGenFunction &CGF, const OMPExecutableDirective &D,
1181  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1182  if (!CGF.HaveInsertPoint())
1183  return;
1184  llvm::BasicBlock *DoneBB = nullptr;
1185  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1186  if (auto *PostUpdate = C->getPostUpdateExpr()) {
1187  if (!DoneBB) {
1188  if (auto *Cond = CondGen(CGF)) {
1189  // If the first post-update expression is found, emit conditional
1190  // block if it was requested.
1191  auto *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1192  DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1193  CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1194  CGF.EmitBlock(ThenBB);
1195  }
1196  }
1197  CGF.EmitIgnoredExpr(PostUpdate);
1198  }
1199  }
1200  if (DoneBB)
1201  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1202 }
1203 
1204 namespace {
1205 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1206 /// parallel function. This is necessary for combined constructs such as
1207 /// 'distribute parallel for'
1208 typedef llvm::function_ref<void(CodeGenFunction &,
1209  const OMPExecutableDirective &,
1211  CodeGenBoundParametersTy;
1212 } // anonymous namespace
1213 
1215  CodeGenFunction &CGF, const OMPExecutableDirective &S,
1216  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1217  const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1218  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1219  auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1220  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1221  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1222  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1223  auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1224  /*IgnoreResultAssign*/ true);
1225  CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1226  CGF, NumThreads, NumThreadsClause->getLocStart());
1227  }
1228  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1229  CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1230  CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1231  CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
1232  }
1233  const Expr *IfCond = nullptr;
1234  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1235  if (C->getNameModifier() == OMPD_unknown ||
1236  C->getNameModifier() == OMPD_parallel) {
1237  IfCond = C->getCondition();
1238  break;
1239  }
1240  }
1241 
1242  OMPParallelScope Scope(CGF, S);
1244  // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1245  // lower and upper bounds with the pragma 'for' chunking mechanism.
1246  // The following lambda takes care of appending the lower and upper bound
1247  // parameters when necessary
1248  CodeGenBoundParameters(CGF, S, CapturedVars);
1249  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1250  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn,
1251  CapturedVars, IfCond);
1252 }
1253 
1255  const OMPExecutableDirective &,
1257 
1259  // Emit parallel region as a standalone region.
1260  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
1261  OMPPrivateScope PrivateScope(CGF);
1262  bool Copyins = CGF.EmitOMPCopyinClause(S);
1263  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1264  if (Copyins) {
1265  // Emit implicit barrier to synchronize threads and avoid data races on
1266  // propagation master's thread values of threadprivate variables to local
1267  // instances of that variables of all other implicit threads.
1268  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1269  CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
1270  /*ForceSimpleCall=*/true);
1271  }
1272  CGF.EmitOMPPrivateClause(S, PrivateScope);
1273  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1274  (void)PrivateScope.Privatize();
1275  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1276  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1277  };
1278  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1281  *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1282 }
1283 
1285  JumpDest LoopExit) {
1286  RunCleanupsScope BodyScope(*this);
1287  // Update counters values on current iteration.
1288  for (auto I : D.updates()) {
1289  EmitIgnoredExpr(I);
1290  }
1291  // Update the linear variables.
1292  // In distribute directives only loop counters may be marked as linear, no
1293  // need to generate the code for them.
1295  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1296  for (auto *U : C->updates())
1297  EmitIgnoredExpr(U);
1298  }
1299  }
1300 
1301  // On a continue in the body, jump to the end.
1302  auto Continue = getJumpDestInCurrentScope("omp.body.continue");
1303  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1304  // Emit loop body.
1305  EmitStmt(D.getBody());
1306  // The end (updates/cleanups).
1307  EmitBlock(Continue.getBlock());
1308  BreakContinueStack.pop_back();
1309 }
1310 
1312  const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1313  const Expr *IncExpr,
1314  const llvm::function_ref<void(CodeGenFunction &)> &BodyGen,
1315  const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen) {
1316  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1317 
1318  // Start the loop with a block that tests the condition.
1319  auto CondBlock = createBasicBlock("omp.inner.for.cond");
1320  EmitBlock(CondBlock);
1321  const SourceRange &R = S.getSourceRange();
1322  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1323  SourceLocToDebugLoc(R.getEnd()));
1324 
1325  // If there are any cleanups between here and the loop-exit scope,
1326  // create a block to stage a loop exit along.
1327  auto ExitBlock = LoopExit.getBlock();
1328  if (RequiresCleanup)
1329  ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1330 
1331  auto LoopBody = createBasicBlock("omp.inner.for.body");
1332 
1333  // Emit condition.
1334  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1335  if (ExitBlock != LoopExit.getBlock()) {
1336  EmitBlock(ExitBlock);
1337  EmitBranchThroughCleanup(LoopExit);
1338  }
1339 
1340  EmitBlock(LoopBody);
1341  incrementProfileCounter(&S);
1342 
1343  // Create a block for the increment.
1344  auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1345  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1346 
1347  BodyGen(*this);
1348 
1349  // Emit "IV = IV + 1" and a back-edge to the condition block.
1350  EmitBlock(Continue.getBlock());
1351  EmitIgnoredExpr(IncExpr);
1352  PostIncGen(*this);
1353  BreakContinueStack.pop_back();
1354  EmitBranch(CondBlock);
1355  LoopStack.pop();
1356  // Emit the fall-through block.
1357  EmitBlock(LoopExit.getBlock());
1358 }
1359 
1361  if (!HaveInsertPoint())
1362  return false;
1363  // Emit inits for the linear variables.
1364  bool HasLinears = false;
1365  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1366  for (auto *Init : C->inits()) {
1367  HasLinears = true;
1368  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1369  if (auto *Ref = dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1370  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1371  auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1372  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1373  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1374  VD->getInit()->getType(), VK_LValue,
1375  VD->getInit()->getExprLoc());
1376  EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1377  VD->getType()),
1378  /*capturedByInit=*/false);
1379  EmitAutoVarCleanups(Emission);
1380  } else
1381  EmitVarDecl(*VD);
1382  }
1383  // Emit the linear steps for the linear clauses.
1384  // If a step is not constant, it is pre-calculated before the loop.
1385  if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1386  if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1387  EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1388  // Emit calculation of the linear step.
1389  EmitIgnoredExpr(CS);
1390  }
1391  }
1392  return HasLinears;
1393 }
1394 
1396  const OMPLoopDirective &D,
1397  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1398  if (!HaveInsertPoint())
1399  return;
1400  llvm::BasicBlock *DoneBB = nullptr;
1401  // Emit the final values of the linear variables.
1402  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1403  auto IC = C->varlist_begin();
1404  for (auto *F : C->finals()) {
1405  if (!DoneBB) {
1406  if (auto *Cond = CondGen(*this)) {
1407  // If the first post-update expression is found, emit conditional
1408  // block if it was requested.
1409  auto *ThenBB = createBasicBlock(".omp.linear.pu");
1410  DoneBB = createBasicBlock(".omp.linear.pu.done");
1411  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1412  EmitBlock(ThenBB);
1413  }
1414  }
1415  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1416  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1417  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1418  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1419  Address OrigAddr = EmitLValue(&DRE).getAddress();
1420  CodeGenFunction::OMPPrivateScope VarScope(*this);
1421  VarScope.addPrivate(OrigVD, [OrigAddr]() -> Address { return OrigAddr; });
1422  (void)VarScope.Privatize();
1423  EmitIgnoredExpr(F);
1424  ++IC;
1425  }
1426  if (auto *PostUpdate = C->getPostUpdateExpr())
1427  EmitIgnoredExpr(PostUpdate);
1428  }
1429  if (DoneBB)
1430  EmitBlock(DoneBB, /*IsFinished=*/true);
1431 }
1432 
1434  const OMPExecutableDirective &D) {
1435  if (!CGF.HaveInsertPoint())
1436  return;
1437  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1438  unsigned ClauseAlignment = 0;
1439  if (auto AlignmentExpr = Clause->getAlignment()) {
1440  auto AlignmentCI =
1441  cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1442  ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1443  }
1444  for (auto E : Clause->varlists()) {
1445  unsigned Alignment = ClauseAlignment;
1446  if (Alignment == 0) {
1447  // OpenMP [2.8.1, Description]
1448  // If no optional parameter is specified, implementation-defined default
1449  // alignments for SIMD instructions on the target platforms are assumed.
1450  Alignment =
1451  CGF.getContext()
1453  E->getType()->getPointeeType()))
1454  .getQuantity();
1455  }
1456  assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1457  "alignment is not power of 2");
1458  if (Alignment != 0) {
1459  llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1460  CGF.EmitAlignmentAssumption(PtrValue, Alignment);
1461  }
1462  }
1463  }
1464 }
1465 
1467  const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1468  if (!HaveInsertPoint())
1469  return;
1470  auto I = S.private_counters().begin();
1471  for (auto *E : S.counters()) {
1472  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1473  auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1474  (void)LoopScope.addPrivate(VD, [&]() -> Address {
1475  // Emit var without initialization.
1476  if (!LocalDeclMap.count(PrivateVD)) {
1477  auto VarEmission = EmitAutoVarAlloca(*PrivateVD);
1478  EmitAutoVarCleanups(VarEmission);
1479  }
1480  DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1481  /*RefersToEnclosingVariableOrCapture=*/false,
1482  (*I)->getType(), VK_LValue, (*I)->getExprLoc());
1483  return EmitLValue(&DRE).getAddress();
1484  });
1485  if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1486  VD->hasGlobalStorage()) {
1487  (void)LoopScope.addPrivate(PrivateVD, [&]() -> Address {
1488  DeclRefExpr DRE(const_cast<VarDecl *>(VD),
1489  LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1490  E->getType(), VK_LValue, E->getExprLoc());
1491  return EmitLValue(&DRE).getAddress();
1492  });
1493  }
1494  ++I;
1495  }
1496 }
1497 
1498 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1499  const Expr *Cond, llvm::BasicBlock *TrueBlock,
1500  llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1501  if (!CGF.HaveInsertPoint())
1502  return;
1503  {
1504  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1505  CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1506  (void)PreCondScope.Privatize();
1507  // Get initial values of real counters.
1508  for (auto I : S.inits()) {
1509  CGF.EmitIgnoredExpr(I);
1510  }
1511  }
1512  // Check that loop is executed at least one time.
1513  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1514 }
1515 
1517  const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1518  if (!HaveInsertPoint())
1519  return;
1522  auto *LoopDirective = cast<OMPLoopDirective>(&D);
1523  for (auto *C : LoopDirective->counters()) {
1524  SIMDLCVs.insert(
1525  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1526  }
1527  }
1528  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1529  auto CurPrivate = C->privates().begin();
1530  for (auto *E : C->varlists()) {
1531  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1532  auto *PrivateVD =
1533  cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1534  if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1535  bool IsRegistered = PrivateScope.addPrivate(VD, [&]() -> Address {
1536  // Emit private VarDecl with copy init.
1537  EmitVarDecl(*PrivateVD);
1538  return GetAddrOfLocalVar(PrivateVD);
1539  });
1540  assert(IsRegistered && "linear var already registered as private");
1541  // Silence the warning about unused variable.
1542  (void)IsRegistered;
1543  } else
1544  EmitVarDecl(*PrivateVD);
1545  ++CurPrivate;
1546  }
1547  }
1548 }
1549 
1551  const OMPExecutableDirective &D,
1552  bool IsMonotonic) {
1553  if (!CGF.HaveInsertPoint())
1554  return;
1555  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1556  RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1557  /*ignoreResult=*/true);
1558  llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1559  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1560  // In presence of finite 'safelen', it may be unsafe to mark all
1561  // the memory instructions parallel, because loop-carried
1562  // dependences of 'safelen' iterations are possible.
1563  if (!IsMonotonic)
1565  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1566  RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1567  /*ignoreResult=*/true);
1568  llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1569  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1570  // In presence of finite 'safelen', it may be unsafe to mark all
1571  // the memory instructions parallel, because loop-carried
1572  // dependences of 'safelen' iterations are possible.
1573  CGF.LoopStack.setParallel(false);
1574  }
1575 }
1576 
1578  bool IsMonotonic) {
1579  // Walk clauses and process safelen/lastprivate.
1580  LoopStack.setParallel(!IsMonotonic);
1581  LoopStack.setVectorizeEnable(true);
1582  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1583 }
1584 
1586  const OMPLoopDirective &D,
1587  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> &CondGen) {
1588  if (!HaveInsertPoint())
1589  return;
1590  llvm::BasicBlock *DoneBB = nullptr;
1591  auto IC = D.counters().begin();
1592  auto IPC = D.private_counters().begin();
1593  for (auto F : D.finals()) {
1594  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1595  auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1596  auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1597  if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1598  OrigVD->hasGlobalStorage() || CED) {
1599  if (!DoneBB) {
1600  if (auto *Cond = CondGen(*this)) {
1601  // If the first post-update expression is found, emit conditional
1602  // block if it was requested.
1603  auto *ThenBB = createBasicBlock(".omp.final.then");
1604  DoneBB = createBasicBlock(".omp.final.done");
1605  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1606  EmitBlock(ThenBB);
1607  }
1608  }
1609  Address OrigAddr = Address::invalid();
1610  if (CED)
1611  OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1612  else {
1613  DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
1614  /*RefersToEnclosingVariableOrCapture=*/false,
1615  (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1616  OrigAddr = EmitLValue(&DRE).getAddress();
1617  }
1618  OMPPrivateScope VarScope(*this);
1619  VarScope.addPrivate(OrigVD,
1620  [OrigAddr]() -> Address { return OrigAddr; });
1621  (void)VarScope.Privatize();
1622  EmitIgnoredExpr(F);
1623  }
1624  ++IC;
1625  ++IPC;
1626  }
1627  if (DoneBB)
1628  EmitBlock(DoneBB, /*IsFinished=*/true);
1629 }
1630 
1632  const OMPLoopDirective &S,
1634  CGF.EmitOMPLoopBody(S, LoopExit);
1635  CGF.EmitStopPoint(&S);
1636 }
1637 
1638 /// Emit a helper variable and return corresponding lvalue.
1640  const DeclRefExpr *Helper) {
1641  auto VDecl = cast<VarDecl>(Helper->getDecl());
1642  CGF.EmitVarDecl(*VDecl);
1643  return CGF.EmitLValue(Helper);
1644 }
1645 
1647  PrePostActionTy &Action) {
1648  Action.Enter(CGF);
1650  "Expected simd directive");
1651  OMPLoopScope PreInitScope(CGF, S);
1652  // if (PreCond) {
1653  // for (IV in 0..LastIteration) BODY;
1654  // <Final counter/linear vars updates>;
1655  // }
1656  //
1660  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1661  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1662  }
1663 
1664  // Emit: if (PreCond) - begin.
1665  // If the condition constant folds and can be elided, avoid emitting the
1666  // whole loop.
1667  bool CondConstant;
1668  llvm::BasicBlock *ContBlock = nullptr;
1669  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1670  if (!CondConstant)
1671  return;
1672  } else {
1673  auto *ThenBlock = CGF.createBasicBlock("simd.if.then");
1674  ContBlock = CGF.createBasicBlock("simd.if.end");
1675  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1676  CGF.getProfileCount(&S));
1677  CGF.EmitBlock(ThenBlock);
1678  CGF.incrementProfileCounter(&S);
1679  }
1680 
1681  // Emit the loop iteration variable.
1682  const Expr *IVExpr = S.getIterationVariable();
1683  const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1684  CGF.EmitVarDecl(*IVDecl);
1685  CGF.EmitIgnoredExpr(S.getInit());
1686 
1687  // Emit the iterations count variable.
1688  // If it is not a variable, Sema decided to calculate iterations count on
1689  // each iteration (e.g., it is foldable into a constant).
1690  if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1691  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1692  // Emit calculation of the iterations count.
1694  }
1695 
1696  CGF.EmitOMPSimdInit(S);
1697 
1698  emitAlignedClause(CGF, S);
1699  (void)CGF.EmitOMPLinearClauseInit(S);
1700  {
1701  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1702  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1703  CGF.EmitOMPLinearClause(S, LoopScope);
1704  CGF.EmitOMPPrivateClause(S, LoopScope);
1705  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1706  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1707  (void)LoopScope.Privatize();
1708  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1709  S.getInc(),
1710  [&S](CodeGenFunction &CGF) {
1712  CGF.EmitStopPoint(&S);
1713  },
1714  [](CodeGenFunction &) {});
1715  CGF.EmitOMPSimdFinal(
1716  S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1717  // Emit final copy of the lastprivate variables at the end of loops.
1718  if (HasLastprivateClause)
1719  CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1720  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1722  CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1723  }
1725  S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
1726  // Emit: if (PreCond) - end.
1727  if (ContBlock) {
1728  CGF.EmitBranch(ContBlock);
1729  CGF.EmitBlock(ContBlock, true);
1730  }
1731 }
1732 
1734  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1735  emitOMPSimdRegion(CGF, S, Action);
1736  };
1737  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
1738  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1739 }
1740 
1741 void CodeGenFunction::EmitOMPOuterLoop(
1742  bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1744  const CodeGenFunction::OMPLoopArguments &LoopArgs,
1745  const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1746  const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1747  auto &RT = CGM.getOpenMPRuntime();
1748 
1749  const Expr *IVExpr = S.getIterationVariable();
1750  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1751  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1752 
1753  auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1754 
1755  // Start the loop with a block that tests the condition.
1756  auto CondBlock = createBasicBlock("omp.dispatch.cond");
1757  EmitBlock(CondBlock);
1758  const SourceRange &R = S.getSourceRange();
1759  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1760  SourceLocToDebugLoc(R.getEnd()));
1761 
1762  llvm::Value *BoolCondVal = nullptr;
1763  if (!DynamicOrOrdered) {
1764  // UB = min(UB, GlobalUB) or
1765  // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1766  // 'distribute parallel for')
1767  EmitIgnoredExpr(LoopArgs.EUB);
1768  // IV = LB
1769  EmitIgnoredExpr(LoopArgs.Init);
1770  // IV < UB
1771  BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1772  } else {
1773  BoolCondVal =
1774  RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned, LoopArgs.IL,
1775  LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1776  }
1777 
1778  // If there are any cleanups between here and the loop-exit scope,
1779  // create a block to stage a loop exit along.
1780  auto ExitBlock = LoopExit.getBlock();
1781  if (LoopScope.requiresCleanups())
1782  ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1783 
1784  auto LoopBody = createBasicBlock("omp.dispatch.body");
1785  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1786  if (ExitBlock != LoopExit.getBlock()) {
1787  EmitBlock(ExitBlock);
1788  EmitBranchThroughCleanup(LoopExit);
1789  }
1790  EmitBlock(LoopBody);
1791 
1792  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1793  // LB for loop condition and emitted it above).
1794  if (DynamicOrOrdered)
1795  EmitIgnoredExpr(LoopArgs.Init);
1796 
1797  // Create a block for the increment.
1798  auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1799  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1800 
1801  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1802  // with dynamic/guided scheduling and without ordered clause.
1804  LoopStack.setParallel(!IsMonotonic);
1805  else
1806  EmitOMPSimdInit(S, IsMonotonic);
1807 
1808  SourceLocation Loc = S.getLocStart();
1809 
1810  // when 'distribute' is not combined with a 'for':
1811  // while (idx <= UB) { BODY; ++idx; }
1812  // when 'distribute' is combined with a 'for'
1813  // (e.g. 'distribute parallel for')
1814  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1815  EmitOMPInnerLoop(
1816  S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1817  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1818  CodeGenLoop(CGF, S, LoopExit);
1819  },
1820  [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1821  CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1822  });
1823 
1824  EmitBlock(Continue.getBlock());
1825  BreakContinueStack.pop_back();
1826  if (!DynamicOrOrdered) {
1827  // Emit "LB = LB + Stride", "UB = UB + Stride".
1828  EmitIgnoredExpr(LoopArgs.NextLB);
1829  EmitIgnoredExpr(LoopArgs.NextUB);
1830  }
1831 
1832  EmitBranch(CondBlock);
1833  LoopStack.pop();
1834  // Emit the fall-through block.
1835  EmitBlock(LoopExit.getBlock());
1836 
1837  // Tell the runtime we are done.
1838  auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1839  if (!DynamicOrOrdered)
1840  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
1841  S.getDirectiveKind());
1842  };
1843  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1844 }
1845 
1846 void CodeGenFunction::EmitOMPForOuterLoop(
1847  const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1848  const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1849  const OMPLoopArguments &LoopArgs,
1850  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1851  auto &RT = CGM.getOpenMPRuntime();
1852 
1853  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1854  const bool DynamicOrOrdered =
1855  Ordered || RT.isDynamic(ScheduleKind.Schedule);
1856 
1857  assert((Ordered ||
1858  !RT.isStaticNonchunked(ScheduleKind.Schedule,
1859  LoopArgs.Chunk != nullptr)) &&
1860  "static non-chunked schedule does not need outer loop");
1861 
1862  // Emit outer loop.
1863  //
1864  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1865  // When schedule(dynamic,chunk_size) is specified, the iterations are
1866  // distributed to threads in the team in chunks as the threads request them.
1867  // Each thread executes a chunk of iterations, then requests another chunk,
1868  // until no chunks remain to be distributed. Each chunk contains chunk_size
1869  // iterations, except for the last chunk to be distributed, which may have
1870  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1871  //
1872  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1873  // to threads in the team in chunks as the executing threads request them.
1874  // Each thread executes a chunk of iterations, then requests another chunk,
1875  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1876  // each chunk is proportional to the number of unassigned iterations divided
1877  // by the number of threads in the team, decreasing to 1. For a chunk_size
1878  // with value k (greater than 1), the size of each chunk is determined in the
1879  // same way, with the restriction that the chunks do not contain fewer than k
1880  // iterations (except for the last chunk to be assigned, which may have fewer
1881  // than k iterations).
1882  //
1883  // When schedule(auto) is specified, the decision regarding scheduling is
1884  // delegated to the compiler and/or runtime system. The programmer gives the
1885  // implementation the freedom to choose any possible mapping of iterations to
1886  // threads in the team.
1887  //
1888  // When schedule(runtime) is specified, the decision regarding scheduling is
1889  // deferred until run time, and the schedule and chunk size are taken from the
1890  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1891  // implementation defined
1892  //
1893  // while(__kmpc_dispatch_next(&LB, &UB)) {
1894  // idx = LB;
1895  // while (idx <= UB) { BODY; ++idx;
1896  // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1897  // } // inner loop
1898  // }
1899  //
1900  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1901  // When schedule(static, chunk_size) is specified, iterations are divided into
1902  // chunks of size chunk_size, and the chunks are assigned to the threads in
1903  // the team in a round-robin fashion in the order of the thread number.
1904  //
1905  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1906  // while (idx <= UB) { BODY; ++idx; } // inner loop
1907  // LB = LB + ST;
1908  // UB = UB + ST;
1909  // }
1910  //
1911 
1912  const Expr *IVExpr = S.getIterationVariable();
1913  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1914  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1915 
1916  if (DynamicOrOrdered) {
1917  auto DispatchBounds = CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1918  llvm::Value *LBVal = DispatchBounds.first;
1919  llvm::Value *UBVal = DispatchBounds.second;
1920  CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1921  LoopArgs.Chunk};
1922  RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind, IVSize,
1923  IVSigned, Ordered, DipatchRTInputValues);
1924  } else {
1925  CGOpenMPRuntime::StaticRTInput StaticInit(
1926  IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1927  LoopArgs.ST, LoopArgs.Chunk);
1928  RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(),
1929  ScheduleKind, StaticInit);
1930  }
1931 
1932  auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1933  const unsigned IVSize,
1934  const bool IVSigned) {
1935  if (Ordered) {
1936  CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1937  IVSigned);
1938  }
1939  };
1940 
1941  OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1942  LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1943  OuterLoopArgs.IncExpr = S.getInc();
1944  OuterLoopArgs.Init = S.getInit();
1945  OuterLoopArgs.Cond = S.getCond();
1946  OuterLoopArgs.NextLB = S.getNextLowerBound();
1947  OuterLoopArgs.NextUB = S.getNextUpperBound();
1948  EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1949  emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1950 }
1951 
1953  const unsigned IVSize, const bool IVSigned) {}
1954 
1955 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1956  OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
1957  OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
1958  const CodeGenLoopTy &CodeGenLoopContent) {
1959 
1960  auto &RT = CGM.getOpenMPRuntime();
1961 
1962  // Emit outer loop.
1963  // Same behavior as a OMPForOuterLoop, except that schedule cannot be
1964  // dynamic
1965  //
1966 
1967  const Expr *IVExpr = S.getIterationVariable();
1968  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1969  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1970 
1971  CGOpenMPRuntime::StaticRTInput StaticInit(
1972  IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
1973  LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
1974  RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind, StaticInit);
1975 
1976  // for combined 'distribute' and 'for' the increment expression of distribute
1977  // is store in DistInc. For 'distribute' alone, it is in Inc.
1978  Expr *IncExpr;
1980  IncExpr = S.getDistInc();
1981  else
1982  IncExpr = S.getInc();
1983 
1984  // this routine is shared by 'omp distribute parallel for' and
1985  // 'omp distribute': select the right EUB expression depending on the
1986  // directive
1987  OMPLoopArguments OuterLoopArgs;
1988  OuterLoopArgs.LB = LoopArgs.LB;
1989  OuterLoopArgs.UB = LoopArgs.UB;
1990  OuterLoopArgs.ST = LoopArgs.ST;
1991  OuterLoopArgs.IL = LoopArgs.IL;
1992  OuterLoopArgs.Chunk = LoopArgs.Chunk;
1993  OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1995  : S.getEnsureUpperBound();
1996  OuterLoopArgs.IncExpr = IncExpr;
1997  OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
1998  ? S.getCombinedInit()
1999  : S.getInit();
2000  OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2001  ? S.getCombinedCond()
2002  : S.getCond();
2003  OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2005  : S.getNextLowerBound();
2006  OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2008  : S.getNextUpperBound();
2009 
2010  EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2011  LoopScope, OuterLoopArgs, CodeGenLoopContent,
2013 }
2014 
2015 static std::pair<LValue, LValue>
2017  const OMPExecutableDirective &S) {
2018  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2019  LValue LB =
2020  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2021  LValue UB =
2022  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2023 
2024  // When composing 'distribute' with 'for' (e.g. as in 'distribute
2025  // parallel for') we need to use the 'distribute'
2026  // chunk lower and upper bounds rather than the whole loop iteration
2027  // space. These are parameters to the outlined function for 'parallel'
2028  // and we copy the bounds of the previous schedule into the
2029  // the current ones.
2030  LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2031  LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2032  llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(PrevLB, SourceLocation());
2033  PrevLBVal = CGF.EmitScalarConversion(
2034  PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2036  llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(PrevUB, SourceLocation());
2037  PrevUBVal = CGF.EmitScalarConversion(
2038  PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2040 
2041  CGF.EmitStoreOfScalar(PrevLBVal, LB);
2042  CGF.EmitStoreOfScalar(PrevUBVal, UB);
2043 
2044  return {LB, UB};
2045 }
2046 
2047 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2048 /// we need to use the LB and UB expressions generated by the worksharing
2049 /// code generation support, whereas in non combined situations we would
2050 /// just emit 0 and the LastIteration expression
2051 /// This function is necessary due to the difference of the LB and UB
2052 /// types for the RT emission routines for 'for_static_init' and
2053 /// 'for_dispatch_init'
2054 static std::pair<llvm::Value *, llvm::Value *>
2056  const OMPExecutableDirective &S,
2057  Address LB, Address UB) {
2058  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2059  const Expr *IVExpr = LS.getIterationVariable();
2060  // when implementing a dynamic schedule for a 'for' combined with a
2061  // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2062  // is not normalized as each team only executes its own assigned
2063  // distribute chunk
2064  QualType IteratorTy = IVExpr->getType();
2065  llvm::Value *LBVal = CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy,
2066  SourceLocation());
2067  llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy,
2068  SourceLocation());
2069  return {LBVal, UBVal};
2070 }
2071 
2073  CodeGenFunction &CGF, const OMPExecutableDirective &S,
2074  llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2075  const auto &Dir = cast<OMPLoopDirective>(S);
2076  LValue LB =
2077  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2078  auto LBCast = CGF.Builder.CreateIntCast(
2079  CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2080  CapturedVars.push_back(LBCast);
2081  LValue UB =
2082  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2083 
2084  auto UBCast = CGF.Builder.CreateIntCast(
2085  CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2086  CapturedVars.push_back(UBCast);
2087 }
2088 
2089 static void
2091  const OMPLoopDirective &S,
2093  auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2094  PrePostActionTy &) {
2095  bool HasCancel = false;
2097  if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2098  HasCancel = D->hasCancel();
2099  else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2100  HasCancel = D->hasCancel();
2101  else if (const auto *D =
2102  dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2103  HasCancel = D->hasCancel();
2104  }
2106  HasCancel);
2110  };
2111 
2113  CGF, S,
2114  isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2115  CGInlinedWorksharingLoop,
2117 }
2118 
2121  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2123  S.getDistInc());
2124  };
2125  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2126  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2127 }
2128 
2131  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2133  S.getDistInc());
2134  };
2135  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2136  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2137 }
2138 
2140  const OMPDistributeSimdDirective &S) {
2141  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2143  };
2144  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2145  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2146 }
2147 
2149  CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2150  // Emit SPMD target parallel for region as a standalone region.
2151  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2152  emitOMPSimdRegion(CGF, S, Action);
2153  };
2154  llvm::Function *Fn;
2155  llvm::Constant *Addr;
2156  // Emit target region as a standalone region.
2157  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2158  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2159  assert(Fn && Addr && "Target device function emission failed.");
2160 }
2161 
2163  const OMPTargetSimdDirective &S) {
2164  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2165  emitOMPSimdRegion(CGF, S, Action);
2166  };
2167  emitCommonOMPTargetDirective(*this, S, CodeGen);
2168 }
2169 
2172  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2173  CGM.getOpenMPRuntime().emitInlinedDirective(
2174  *this, OMPD_target_teams_distribute_parallel_for,
2175  [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2176  CGF.EmitStmt(
2177  cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2178  });
2179 }
2180 
2183  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2184  CGM.getOpenMPRuntime().emitInlinedDirective(
2185  *this, OMPD_target_teams_distribute_parallel_for_simd,
2186  [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2187  CGF.EmitStmt(
2188  cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2189  });
2190 }
2191 
2192 namespace {
2193  struct ScheduleKindModifiersTy {
2197  ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2200  : Kind(Kind), M1(M1), M2(M2) {}
2201  };
2202 } // namespace
2203 
2205  const OMPLoopDirective &S, Expr *EUB,
2206  const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2207  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2208  // Emit the loop iteration variable.
2209  auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2210  auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
2211  EmitVarDecl(*IVDecl);
2212 
2213  // Emit the iterations count variable.
2214  // If it is not a variable, Sema decided to calculate iterations count on each
2215  // iteration (e.g., it is foldable into a constant).
2216  if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2217  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2218  // Emit calculation of the iterations count.
2219  EmitIgnoredExpr(S.getCalcLastIteration());
2220  }
2221 
2222  auto &RT = CGM.getOpenMPRuntime();
2223 
2224  bool HasLastprivateClause;
2225  // Check pre-condition.
2226  {
2227  OMPLoopScope PreInitScope(*this, S);
2228  // Skip the entire loop if we don't meet the precondition.
2229  // If the condition constant folds and can be elided, avoid emitting the
2230  // whole loop.
2231  bool CondConstant;
2232  llvm::BasicBlock *ContBlock = nullptr;
2233  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2234  if (!CondConstant)
2235  return false;
2236  } else {
2237  auto *ThenBlock = createBasicBlock("omp.precond.then");
2238  ContBlock = createBasicBlock("omp.precond.end");
2239  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2240  getProfileCount(&S));
2241  EmitBlock(ThenBlock);
2242  incrementProfileCounter(&S);
2243  }
2244 
2245  bool Ordered = false;
2246  if (auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2247  if (OrderedClause->getNumForLoops())
2248  RT.emitDoacrossInit(*this, S);
2249  else
2250  Ordered = true;
2251  }
2252 
2253  llvm::DenseSet<const Expr *> EmittedFinals;
2254  emitAlignedClause(*this, S);
2255  bool HasLinears = EmitOMPLinearClauseInit(S);
2256  // Emit helper vars inits.
2257 
2258  std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2259  LValue LB = Bounds.first;
2260  LValue UB = Bounds.second;
2261  LValue ST =
2262  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2263  LValue IL =
2264  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2265 
2266  // Emit 'then' code.
2267  {
2268  OMPPrivateScope LoopScope(*this);
2269  if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2270  // Emit implicit barrier to synchronize threads and avoid data races on
2271  // initialization of firstprivate variables and post-update of
2272  // lastprivate variables.
2273  CGM.getOpenMPRuntime().emitBarrierCall(
2274  *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2275  /*ForceSimpleCall=*/true);
2276  }
2277  EmitOMPPrivateClause(S, LoopScope);
2278  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2279  EmitOMPReductionClauseInit(S, LoopScope);
2280  EmitOMPPrivateLoopCounters(S, LoopScope);
2281  EmitOMPLinearClause(S, LoopScope);
2282  (void)LoopScope.Privatize();
2283 
2284  // Detect the loop schedule kind and chunk.
2285  llvm::Value *Chunk = nullptr;
2286  OpenMPScheduleTy ScheduleKind;
2287  if (auto *C = S.getSingleClause<OMPScheduleClause>()) {
2288  ScheduleKind.Schedule = C->getScheduleKind();
2289  ScheduleKind.M1 = C->getFirstScheduleModifier();
2290  ScheduleKind.M2 = C->getSecondScheduleModifier();
2291  if (const auto *Ch = C->getChunkSize()) {
2292  Chunk = EmitScalarExpr(Ch);
2293  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
2295  S.getLocStart());
2296  }
2297  }
2298  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2299  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2300  // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2301  // If the static schedule kind is specified or if the ordered clause is
2302  // specified, and if no monotonic modifier is specified, the effect will
2303  // be as if the monotonic modifier was specified.
2304  if (RT.isStaticNonchunked(ScheduleKind.Schedule,
2305  /* Chunked */ Chunk != nullptr) &&
2306  !Ordered) {
2308  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2309  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2310  // When no chunk_size is specified, the iteration space is divided into
2311  // chunks that are approximately equal in size, and at most one chunk is
2312  // distributed to each thread. Note that the size of the chunks is
2313  // unspecified in this case.
2314  CGOpenMPRuntime::StaticRTInput StaticInit(
2315  IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2316  UB.getAddress(), ST.getAddress());
2317  RT.emitForStaticInit(*this, S.getLocStart(), S.getDirectiveKind(),
2318  ScheduleKind, StaticInit);
2319  auto LoopExit =
2320  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2321  // UB = min(UB, GlobalUB);
2322  EmitIgnoredExpr(S.getEnsureUpperBound());
2323  // IV = LB;
2324  EmitIgnoredExpr(S.getInit());
2325  // while (idx <= UB) { BODY; ++idx; }
2326  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
2327  S.getInc(),
2328  [&S, LoopExit](CodeGenFunction &CGF) {
2329  CGF.EmitOMPLoopBody(S, LoopExit);
2330  CGF.EmitStopPoint(&S);
2331  },
2332  [](CodeGenFunction &) {});
2333  EmitBlock(LoopExit.getBlock());
2334  // Tell the runtime we are done.
2335  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2336  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
2337  S.getDirectiveKind());
2338  };
2339  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2340  } else {
2341  const bool IsMonotonic =
2342  Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2343  ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2344  ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2345  ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2346  // Emit the outer loop, which requests its work chunk [LB..UB] from
2347  // runtime and runs the inner loop to process it.
2348  const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2349  ST.getAddress(), IL.getAddress(),
2350  Chunk, EUB);
2351  EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2352  LoopArguments, CGDispatchBounds);
2353  }
2355  EmitOMPSimdFinal(S,
2356  [&](CodeGenFunction &CGF) -> llvm::Value * {
2357  return CGF.Builder.CreateIsNotNull(
2358  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2359  });
2360  }
2361  EmitOMPReductionClauseFinal(
2362  S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2363  ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2364  : /*Parallel only*/ OMPD_parallel);
2365  // Emit post-update of the reduction variables if IsLastIter != 0.
2367  *this, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2368  return CGF.Builder.CreateIsNotNull(
2369  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2370  });
2371  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2372  if (HasLastprivateClause)
2373  EmitOMPLastprivateClauseFinal(
2375  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
2376  }
2377  EmitOMPLinearClauseFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2378  return CGF.Builder.CreateIsNotNull(
2379  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2380  });
2381  // We're now done with the loop, so jump to the continuation block.
2382  if (ContBlock) {
2383  EmitBranch(ContBlock);
2384  EmitBlock(ContBlock, true);
2385  }
2386  }
2387  return HasLastprivateClause;
2388 }
2389 
2390 /// The following two functions generate expressions for the loop lower
2391 /// and upper bounds in case of static and dynamic (dispatch) schedule
2392 /// of the associated 'for' or 'distribute' loop.
2393 static std::pair<LValue, LValue>
2395  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2396  LValue LB =
2397  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2398  LValue UB =
2399  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2400  return {LB, UB};
2401 }
2402 
2403 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2404 /// consider the lower and upper bound expressions generated by the
2405 /// worksharing loop support, but we use 0 and the iteration space size as
2406 /// constants
2407 static std::pair<llvm::Value *, llvm::Value *>
2409  Address LB, Address UB) {
2410  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2411  const Expr *IVExpr = LS.getIterationVariable();
2412  const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2413  llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2414  llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2415  return {LBVal, UBVal};
2416 }
2417 
2419  bool HasLastprivates = false;
2420  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2421  PrePostActionTy &) {
2422  OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2423  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2426  };
2427  {
2428  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2429  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2430  S.hasCancel());
2431  }
2432 
2433  // Emit an implicit barrier at the end.
2434  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
2435  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2436  }
2437 }
2438 
2440  bool HasLastprivates = false;
2441  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2442  PrePostActionTy &) {
2443  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2446  };
2447  {
2448  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2449  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2450  }
2451 
2452  // Emit an implicit barrier at the end.
2453  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
2454  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
2455  }
2456 }
2457 
2459  const Twine &Name,
2460  llvm::Value *Init = nullptr) {
2461  auto LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2462  if (Init)
2463  CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2464  return LVal;
2465 }
2466 
2467 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2468  auto *Stmt = cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
2469  auto *CS = dyn_cast<CompoundStmt>(Stmt);
2470  bool HasLastprivates = false;
2471  auto &&CodeGen = [&S, Stmt, CS, &HasLastprivates](CodeGenFunction &CGF,
2472  PrePostActionTy &) {
2473  auto &C = CGF.CGM.getContext();
2474  auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2475  // Emit helper vars inits.
2476  LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2477  CGF.Builder.getInt32(0));
2478  auto *GlobalUBVal = CS != nullptr ? CGF.Builder.getInt32(CS->size() - 1)
2479  : CGF.Builder.getInt32(0);
2480  LValue UB =
2481  createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2482  LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2483  CGF.Builder.getInt32(1));
2484  LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2485  CGF.Builder.getInt32(0));
2486  // Loop counter.
2487  LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2488  OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2489  CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2490  OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
2491  CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2492  // Generate condition for loop.
2493  BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2495  // Increment for loop counter.
2496  UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2497  S.getLocStart());
2498  auto BodyGen = [Stmt, CS, &S, &IV](CodeGenFunction &CGF) {
2499  // Iterate through all sections and emit a switch construct:
2500  // switch (IV) {
2501  // case 0:
2502  // <SectionStmt[0]>;
2503  // break;
2504  // ...
2505  // case <NumSection> - 1:
2506  // <SectionStmt[<NumSection> - 1]>;
2507  // break;
2508  // }
2509  // .omp.sections.exit:
2510  auto *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2511  auto *SwitchStmt = CGF.Builder.CreateSwitch(
2512  CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB,
2513  CS == nullptr ? 1 : CS->size());
2514  if (CS) {
2515  unsigned CaseNumber = 0;
2516  for (auto *SubStmt : CS->children()) {
2517  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2518  CGF.EmitBlock(CaseBB);
2519  SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2520  CGF.EmitStmt(SubStmt);
2521  CGF.EmitBranch(ExitBB);
2522  ++CaseNumber;
2523  }
2524  } else {
2525  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2526  CGF.EmitBlock(CaseBB);
2527  SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2528  CGF.EmitStmt(Stmt);
2529  CGF.EmitBranch(ExitBB);
2530  }
2531  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2532  };
2533 
2534  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2535  if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2536  // Emit implicit barrier to synchronize threads and avoid data races on
2537  // initialization of firstprivate variables and post-update of lastprivate
2538  // variables.
2539  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2540  CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
2541  /*ForceSimpleCall=*/true);
2542  }
2543  CGF.EmitOMPPrivateClause(S, LoopScope);
2544  HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2545  CGF.EmitOMPReductionClauseInit(S, LoopScope);
2546  (void)LoopScope.Privatize();
2547 
2548  // Emit static non-chunked loop.
2549  OpenMPScheduleTy ScheduleKind;
2550  ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2551  CGOpenMPRuntime::StaticRTInput StaticInit(
2552  /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2553  LB.getAddress(), UB.getAddress(), ST.getAddress());
2554  CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2555  CGF, S.getLocStart(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2556  // UB = min(UB, GlobalUB);
2557  auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart());
2558  auto *MinUBGlobalUB = CGF.Builder.CreateSelect(
2559  CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2560  CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2561  // IV = LB;
2562  CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV);
2563  // while (idx <= UB) { BODY; ++idx; }
2564  CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2565  [](CodeGenFunction &) {});
2566  // Tell the runtime we are done.
2567  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2568  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocEnd(),
2569  S.getDirectiveKind());
2570  };
2571  CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2572  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2573  // Emit post-update of the reduction variables if IsLastIter != 0.
2575  CGF, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
2576  return CGF.Builder.CreateIsNotNull(
2577  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
2578  });
2579 
2580  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2581  if (HasLastprivates)
2583  S, /*NoFinals=*/false,
2584  CGF.Builder.CreateIsNotNull(
2585  CGF.EmitLoadOfScalar(IL, S.getLocStart())));
2586  };
2587 
2588  bool HasCancel = false;
2589  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2590  HasCancel = OSD->hasCancel();
2591  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2592  HasCancel = OPSD->hasCancel();
2593  OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2594  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2595  HasCancel);
2596  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2597  // clause. Otherwise the barrier will be generated by the codegen for the
2598  // directive.
2599  if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2600  // Emit implicit barrier to synchronize threads and avoid data races on
2601  // initialization of firstprivate variables.
2602  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2603  OMPD_unknown);
2604  }
2605 }
2606 
2608  {
2609  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2610  EmitSections(S);
2611  }
2612  // Emit an implicit barrier at the end.
2613  if (!S.getSingleClause<OMPNowaitClause>()) {
2614  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
2615  OMPD_sections);
2616  }
2617 }
2618 
2620  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2621  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2622  };
2623  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2624  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2625  S.hasCancel());
2626 }
2627 
2629  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2632  llvm::SmallVector<const Expr *, 8> AssignmentOps;
2633  // Check if there are any 'copyprivate' clauses associated with this
2634  // 'single' construct.
2635  // Build a list of copyprivate variables along with helper expressions
2636  // (<source>, <destination>, <destination>=<source> expressions)
2637  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2638  CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2639  DestExprs.append(C->destination_exprs().begin(),
2640  C->destination_exprs().end());
2641  SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2642  AssignmentOps.append(C->assignment_ops().begin(),
2643  C->assignment_ops().end());
2644  }
2645  // Emit code for 'single' region along with 'copyprivate' clauses
2646  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2647  Action.Enter(CGF);
2648  OMPPrivateScope SingleScope(CGF);
2649  (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2650  CGF.EmitOMPPrivateClause(S, SingleScope);
2651  (void)SingleScope.Privatize();
2652  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2653  };
2654  {
2655  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2656  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
2657  CopyprivateVars, DestExprs,
2658  SrcExprs, AssignmentOps);
2659  }
2660  // Emit an implicit barrier at the end (to avoid data race on firstprivate
2661  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2662  if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2663  CGM.getOpenMPRuntime().emitBarrierCall(
2664  *this, S.getLocStart(),
2665  S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2666  }
2667 }
2668 
2670  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2671  Action.Enter(CGF);
2672  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2673  };
2674  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2675  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart());
2676 }
2677 
2679  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2680  Action.Enter(CGF);
2681  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2682  };
2683  Expr *Hint = nullptr;
2684  if (auto *HintClause = S.getSingleClause<OMPHintClause>())
2685  Hint = HintClause->getHint();
2686  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
2687  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2689  CodeGen, S.getLocStart(), Hint);
2690 }
2691 
2693  const OMPParallelForDirective &S) {
2694  // Emit directive as a combined directive that consists of two implicit
2695  // directives: 'parallel' with 'for' directive.
2696  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2697  OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2700  };
2701  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2703 }
2704 
2706  const OMPParallelForSimdDirective &S) {
2707  // Emit directive as a combined directive that consists of two implicit
2708  // directives: 'parallel' with 'for' directive.
2709  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2712  };
2713  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2715 }
2716 
2718  const OMPParallelSectionsDirective &S) {
2719  // Emit directive as a combined directive that consists of two implicit
2720  // directives: 'parallel' with 'sections' directive.
2721  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2722  CGF.EmitSections(S);
2723  };
2724  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2726 }
2727 
2729  const RegionCodeGenTy &BodyGen,
2730  const TaskGenTy &TaskGen,
2731  OMPTaskDataTy &Data) {
2732  // Emit outlined function for task construct.
2733  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
2734  auto *I = CS->getCapturedDecl()->param_begin();
2735  auto *PartId = std::next(I);
2736  auto *TaskT = std::next(I, 4);
2737  // Check if the task is final
2738  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2739  // If the condition constant folds and can be elided, try to avoid emitting
2740  // the condition and the dead arm of the if/else.
2741  auto *Cond = Clause->getCondition();
2742  bool CondConstant;
2743  if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2744  Data.Final.setInt(CondConstant);
2745  else
2746  Data.Final.setPointer(EvaluateExprAsBool(Cond));
2747  } else {
2748  // By default the task is not final.
2749  Data.Final.setInt(/*IntVal=*/false);
2750  }
2751  // Check if the task has 'priority' clause.
2752  if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2753  auto *Prio = Clause->getPriority();
2754  Data.Priority.setInt(/*IntVal=*/true);
2755  Data.Priority.setPointer(EmitScalarConversion(
2756  EmitScalarExpr(Prio), Prio->getType(),
2757  getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2758  Prio->getExprLoc()));
2759  }
2760  // The first function argument for tasks is a thread id, the second one is a
2761  // part id (0 for tied tasks, >=0 for untied task).
2762  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2763  // Get list of private variables.
2764  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2765  auto IRef = C->varlist_begin();
2766  for (auto *IInit : C->private_copies()) {
2767  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2768  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2769  Data.PrivateVars.push_back(*IRef);
2770  Data.PrivateCopies.push_back(IInit);
2771  }
2772  ++IRef;
2773  }
2774  }
2775  EmittedAsPrivate.clear();
2776  // Get list of firstprivate variables.
2777  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2778  auto IRef = C->varlist_begin();
2779  auto IElemInitRef = C->inits().begin();
2780  for (auto *IInit : C->private_copies()) {
2781  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2782  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2783  Data.FirstprivateVars.push_back(*IRef);
2784  Data.FirstprivateCopies.push_back(IInit);
2785  Data.FirstprivateInits.push_back(*IElemInitRef);
2786  }
2787  ++IRef;
2788  ++IElemInitRef;
2789  }
2790  }
2791  // Get list of lastprivate variables (for taskloops).
2792  llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2793  for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2794  auto IRef = C->varlist_begin();
2795  auto ID = C->destination_exprs().begin();
2796  for (auto *IInit : C->private_copies()) {
2797  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2798  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2799  Data.LastprivateVars.push_back(*IRef);
2800  Data.LastprivateCopies.push_back(IInit);
2801  }
2802  LastprivateDstsOrigs.insert(
2803  {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2804  cast<DeclRefExpr>(*IRef)});
2805  ++IRef;
2806  ++ID;
2807  }
2808  }
2811  for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2812  auto IPriv = C->privates().begin();
2813  auto IRed = C->reduction_ops().begin();
2814  auto ILHS = C->lhs_exprs().begin();
2815  auto IRHS = C->rhs_exprs().begin();
2816  for (const auto *Ref : C->varlists()) {
2817  Data.ReductionVars.emplace_back(Ref);
2818  Data.ReductionCopies.emplace_back(*IPriv);
2819  Data.ReductionOps.emplace_back(*IRed);
2820  LHSs.emplace_back(*ILHS);
2821  RHSs.emplace_back(*IRHS);
2822  std::advance(IPriv, 1);
2823  std::advance(IRed, 1);
2824  std::advance(ILHS, 1);
2825  std::advance(IRHS, 1);
2826  }
2827  }
2828  Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2829  *this, S.getLocStart(), LHSs, RHSs, Data);
2830  // Build list of dependences.
2831  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2832  for (auto *IRef : C->varlists())
2833  Data.Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
2834  auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs](
2835  CodeGenFunction &CGF, PrePostActionTy &Action) {
2836  // Set proper addresses for generated private copies.
2837  OMPPrivateScope Scope(CGF);
2838  if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2839  !Data.LastprivateVars.empty()) {
2840  enum { PrivatesParam = 2, CopyFnParam = 3 };
2841  auto *CopyFn = CGF.Builder.CreateLoad(
2842  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
2843  auto *PrivatesPtr = CGF.Builder.CreateLoad(
2844  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
2845  // Map privates.
2848  CallArgs.push_back(PrivatesPtr);
2849  for (auto *E : Data.PrivateVars) {
2850  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2851  Address PrivatePtr = CGF.CreateMemTemp(
2852  CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2853  PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2854  CallArgs.push_back(PrivatePtr.getPointer());
2855  }
2856  for (auto *E : Data.FirstprivateVars) {
2857  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2858  Address PrivatePtr =
2859  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2860  ".firstpriv.ptr.addr");
2861  PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2862  CallArgs.push_back(PrivatePtr.getPointer());
2863  }
2864  for (auto *E : Data.LastprivateVars) {
2865  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2866  Address PrivatePtr =
2867  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2868  ".lastpriv.ptr.addr");
2869  PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2870  CallArgs.push_back(PrivatePtr.getPointer());
2871  }
2872  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
2873  CopyFn, CallArgs);
2874  for (auto &&Pair : LastprivateDstsOrigs) {
2875  auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2876  DeclRefExpr DRE(
2877  const_cast<VarDecl *>(OrigVD),
2878  /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup(
2879  OrigVD) != nullptr,
2880  Pair.second->getType(), VK_LValue, Pair.second->getExprLoc());
2881  Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2882  return CGF.EmitLValue(&DRE).getAddress();
2883  });
2884  }
2885  for (auto &&Pair : PrivatePtrs) {
2886  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2887  CGF.getContext().getDeclAlign(Pair.first));
2888  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2889  }
2890  }
2891  if (Data.Reductions) {
2892  OMPLexicalScope LexScope(CGF, S, /*AsInlined=*/true);
2894  Data.ReductionOps);
2895  llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2896  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2897  for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2898  RedCG.emitSharedLValue(CGF, Cnt);
2899  RedCG.emitAggregateType(CGF, Cnt);
2900  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2901  CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2902  Replacement =
2903  Address(CGF.EmitScalarConversion(
2904  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2905  CGF.getContext().getPointerType(
2906  Data.ReductionCopies[Cnt]->getType()),
2907  SourceLocation()),
2908  Replacement.getAlignment());
2909  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2910  Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2911  [Replacement]() { return Replacement; });
2912  // FIXME: This must removed once the runtime library is fixed.
2913  // Emit required threadprivate variables for
2914  // initilizer/combiner/finalizer.
2915  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2916  RedCG, Cnt);
2917  }
2918  }
2919  // Privatize all private variables except for in_reduction items.
2920  (void)Scope.Privatize();
2921  SmallVector<const Expr *, 4> InRedVars;
2922  SmallVector<const Expr *, 4> InRedPrivs;
2924  SmallVector<const Expr *, 4> TaskgroupDescriptors;
2925  for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2926  auto IPriv = C->privates().begin();
2927  auto IRed = C->reduction_ops().begin();
2928  auto ITD = C->taskgroup_descriptors().begin();
2929  for (const auto *Ref : C->varlists()) {
2930  InRedVars.emplace_back(Ref);
2931  InRedPrivs.emplace_back(*IPriv);
2932  InRedOps.emplace_back(*IRed);
2933  TaskgroupDescriptors.emplace_back(*ITD);
2934  std::advance(IPriv, 1);
2935  std::advance(IRed, 1);
2936  std::advance(ITD, 1);
2937  }
2938  }
2939  // Privatize in_reduction items here, because taskgroup descriptors must be
2940  // privatized earlier.
2941  OMPPrivateScope InRedScope(CGF);
2942  if (!InRedVars.empty()) {
2943  ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
2944  for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
2945  RedCG.emitSharedLValue(CGF, Cnt);
2946  RedCG.emitAggregateType(CGF, Cnt);
2947  // The taskgroup descriptor variable is always implicit firstprivate and
2948  // privatized already during procoessing of the firstprivates.
2949  llvm::Value *ReductionsPtr = CGF.EmitLoadOfScalar(
2950  CGF.EmitLValue(TaskgroupDescriptors[Cnt]), SourceLocation());
2951  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2952  CGF, S.getLocStart(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2953  Replacement = Address(
2954  CGF.EmitScalarConversion(
2955  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2956  CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
2957  SourceLocation()),
2958  Replacement.getAlignment());
2959  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2960  InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
2961  [Replacement]() { return Replacement; });
2962  // FIXME: This must removed once the runtime library is fixed.
2963  // Emit required threadprivate variables for
2964  // initilizer/combiner/finalizer.
2965  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getLocStart(),
2966  RedCG, Cnt);
2967  }
2968  }
2969  (void)InRedScope.Privatize();
2970 
2971  Action.Enter(CGF);
2972  BodyGen(CGF);
2973  };
2974  auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
2975  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
2976  Data.NumberOfParts);
2977  OMPLexicalScope Scope(*this, S);
2978  TaskGen(*this, OutlinedFn, Data);
2979 }
2980 
2981 static ImplicitParamDecl *
2983  QualType Ty, CapturedDecl *CD) {
2984  auto *OrigVD = ImplicitParamDecl::Create(
2985  C, CD, SourceLocation(), /*Id=*/nullptr, Ty, ImplicitParamDecl::Other);
2986  auto *OrigRef =
2988  /*RefersToEnclosingVariableOrCapture=*/false,
2989  SourceLocation(), Ty, VK_LValue);
2990  auto *PrivateVD = ImplicitParamDecl::Create(
2991  C, CD, SourceLocation(), /*Id=*/nullptr, Ty, ImplicitParamDecl::Other);
2992  auto *PrivateRef = DeclRefExpr::Create(
2993  C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
2994  /*RefersToEnclosingVariableOrCapture=*/false, SourceLocation(), Ty,
2995  VK_LValue);
2996  QualType ElemType = C.getBaseElementType(Ty);
2997  auto *InitVD =
2998  ImplicitParamDecl::Create(C, CD, SourceLocation(), /*Id=*/nullptr,
2999  ElemType, ImplicitParamDecl::Other);
3000  auto *InitRef =
3002  /*RefersToEnclosingVariableOrCapture=*/false,
3003  SourceLocation(), ElemType, VK_LValue);
3004  PrivateVD->setInitStyle(VarDecl::CInit);
3005  PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3006  InitRef, /*BasePath=*/nullptr,
3007  VK_RValue));
3008  Data.FirstprivateVars.emplace_back(OrigRef);
3009  Data.FirstprivateCopies.emplace_back(PrivateRef);
3010  Data.FirstprivateInits.emplace_back(InitRef);
3011  return OrigVD;
3012 }
3013 
3015  const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3016  OMPTargetDataInfo &InputInfo) {
3017  // Emit outlined function for task construct.
3018  auto CS = S.getCapturedStmt(OMPD_task);
3019  auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
3020  auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3021  auto *I = CS->getCapturedDecl()->param_begin();
3022  auto *PartId = std::next(I);
3023  auto *TaskT = std::next(I, 4);
3024  OMPTaskDataTy Data;
3025  // The task is not final.
3026  Data.Final.setInt(/*IntVal=*/false);
3027  // Get list of firstprivate variables.
3028  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3029  auto IRef = C->varlist_begin();
3030  auto IElemInitRef = C->inits().begin();
3031  for (auto *IInit : C->private_copies()) {
3032  Data.FirstprivateVars.push_back(*IRef);
3033  Data.FirstprivateCopies.push_back(IInit);
3034  Data.FirstprivateInits.push_back(*IElemInitRef);
3035  ++IRef;
3036  ++IElemInitRef;
3037  }
3038  }
3039  OMPPrivateScope TargetScope(*this);
3040  VarDecl *BPVD = nullptr;
3041  VarDecl *PVD = nullptr;
3042  VarDecl *SVD = nullptr;
3043  if (InputInfo.NumberOfTargetItems > 0) {
3044  auto *CD = CapturedDecl::Create(
3045  getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3046  llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3047  QualType BaseAndPointersType = getContext().getConstantArrayType(
3048  getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3049  /*IndexTypeQuals=*/0);
3050  BPVD = createImplicitFirstprivateForType(getContext(), Data,
3051  BaseAndPointersType, CD);
3052  PVD = createImplicitFirstprivateForType(getContext(), Data,
3053  BaseAndPointersType, CD);
3054  QualType SizesType = getContext().getConstantArrayType(
3055  getContext().getSizeType(), ArrSize, ArrayType::Normal,
3056  /*IndexTypeQuals=*/0);
3057  SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD);
3058  TargetScope.addPrivate(
3059  BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3060  TargetScope.addPrivate(PVD,
3061  [&InputInfo]() { return InputInfo.PointersArray; });
3062  TargetScope.addPrivate(SVD,
3063  [&InputInfo]() { return InputInfo.SizesArray; });
3064  }
3065  (void)TargetScope.Privatize();
3066  // Build list of dependences.
3067  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3068  for (auto *IRef : C->varlists())
3069  Data.Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
3070  auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3071  &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3072  // Set proper addresses for generated private copies.
3073  OMPPrivateScope Scope(CGF);
3074  if (!Data.FirstprivateVars.empty()) {
3075  enum { PrivatesParam = 2, CopyFnParam = 3 };
3076  auto *CopyFn = CGF.Builder.CreateLoad(
3077  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
3078  auto *PrivatesPtr = CGF.Builder.CreateLoad(
3079  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
3080  // Map privates.
3083  CallArgs.push_back(PrivatesPtr);
3084  for (auto *E : Data.FirstprivateVars) {
3085  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3086  Address PrivatePtr =
3087  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3088  ".firstpriv.ptr.addr");
3089  PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
3090  CallArgs.push_back(PrivatePtr.getPointer());
3091  }
3092  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
3093  CopyFn, CallArgs);
3094  for (auto &&Pair : PrivatePtrs) {
3095  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3096  CGF.getContext().getDeclAlign(Pair.first));
3097  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3098  }
3099  }
3100  // Privatize all private variables except for in_reduction items.
3101  (void)Scope.Privatize();
3102  InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3103  CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize());
3104  InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3105  CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize());
3106  InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3107  CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize());
3108 
3109  Action.Enter(CGF);
3110  OMPLexicalScope LexScope(CGF, S, /*AsInlined=*/true,
3111  /*EmitPreInitStmt=*/false);
3112  BodyGen(CGF);
3113  };
3114  auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3115  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3116  Data.NumberOfParts);
3117  llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3118  IntegerLiteral IfCond(getContext(), TrueOrFalse,
3119  getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3120  SourceLocation());
3121 
3122  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getLocStart(), S, OutlinedFn,
3123  SharedsTy, CapturedStruct, &IfCond, Data);
3124 }
3125 
3127  // Emit outlined function for task construct.
3128  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
3129  auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
3130  auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3131  const Expr *IfCond = nullptr;
3132  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3133  if (C->getNameModifier() == OMPD_unknown ||
3134  C->getNameModifier() == OMPD_task) {
3135  IfCond = C->getCondition();
3136  break;
3137  }
3138  }
3139 
3140  OMPTaskDataTy Data;
3141  // Check if we should emit tied or untied task.
3142  Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3143  auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3144  CGF.EmitStmt(CS->getCapturedStmt());
3145  };
3146  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3147  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
3148  const OMPTaskDataTy &Data) {
3149  CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getLocStart(), S, OutlinedFn,
3150  SharedsTy, CapturedStruct, IfCond,
3151  Data);
3152  };
3153  EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
3154 }
3155 
3157  const OMPTaskyieldDirective &S) {
3158  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart());
3159 }
3160 
3162  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier);
3163 }
3164 
3166  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart());
3167 }
3168 
3170  const OMPTaskgroupDirective &S) {
3171  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3172  Action.Enter(CGF);
3173  if (const Expr *E = S.getReductionRef()) {
3176  OMPTaskDataTy Data;
3177  for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3178  auto IPriv = C->privates().begin();
3179  auto IRed = C->reduction_ops().begin();
3180  auto ILHS = C->lhs_exprs().begin();
3181  auto IRHS = C->rhs_exprs().begin();
3182  for (const auto *Ref : C->varlists()) {
3183  Data.ReductionVars.emplace_back(Ref);
3184  Data.ReductionCopies.emplace_back(*IPriv);
3185  Data.ReductionOps.emplace_back(*IRed);
3186  LHSs.emplace_back(*ILHS);
3187  RHSs.emplace_back(*IRHS);
3188  std::advance(IPriv, 1);
3189  std::advance(IRed, 1);
3190  std::advance(ILHS, 1);
3191  std::advance(IRHS, 1);
3192  }
3193  }
3194  llvm::Value *ReductionDesc =
3195  CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getLocStart(),
3196  LHSs, RHSs, Data);
3197  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3198  CGF.EmitVarDecl(*VD);
3199  CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3200  /*Volatile=*/false, E->getType());
3201  }
3202  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3203  };
3204  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3205  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart());
3206 }
3207 
3209  CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> {
3210  if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>()) {
3211  return llvm::makeArrayRef(FlushClause->varlist_begin(),
3212  FlushClause->varlist_end());
3213  }
3214  return llvm::None;
3215  }(), S.getLocStart());
3216 }
3217 
3219  const CodeGenLoopTy &CodeGenLoop,
3220  Expr *IncExpr) {
3221  // Emit the loop iteration variable.
3222  auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3223  auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
3224  EmitVarDecl(*IVDecl);
3225 
3226  // Emit the iterations count variable.
3227  // If it is not a variable, Sema decided to calculate iterations count on each
3228  // iteration (e.g., it is foldable into a constant).
3229  if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3230  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3231  // Emit calculation of the iterations count.
3232  EmitIgnoredExpr(S.getCalcLastIteration());
3233  }
3234 
3235  auto &RT = CGM.getOpenMPRuntime();
3236 
3237  bool HasLastprivateClause = false;
3238  // Check pre-condition.
3239  {
3240  OMPLoopScope PreInitScope(*this, S);
3241  // Skip the entire loop if we don't meet the precondition.
3242  // If the condition constant folds and can be elided, avoid emitting the
3243  // whole loop.
3244  bool CondConstant;
3245  llvm::BasicBlock *ContBlock = nullptr;
3246  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3247  if (!CondConstant)
3248  return;
3249  } else {
3250  auto *ThenBlock = createBasicBlock("omp.precond.then");
3251  ContBlock = createBasicBlock("omp.precond.end");
3252  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3253  getProfileCount(&S));
3254  EmitBlock(ThenBlock);
3255  incrementProfileCounter(&S);
3256  }
3257 
3258  emitAlignedClause(*this, S);
3259  // Emit 'then' code.
3260  {
3261  // Emit helper vars inits.
3262 
3263  LValue LB = EmitOMPHelperVar(
3264  *this, cast<DeclRefExpr>(
3267  : S.getLowerBoundVariable())));
3268  LValue UB = EmitOMPHelperVar(
3269  *this, cast<DeclRefExpr>(
3272  : S.getUpperBoundVariable())));
3273  LValue ST =
3274  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3275  LValue IL =
3276  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3277 
3278  OMPPrivateScope LoopScope(*this);
3279  if (EmitOMPFirstprivateClause(S, LoopScope)) {
3280  // Emit implicit barrier to synchronize threads and avoid data races
3281  // on initialization of firstprivate variables and post-update of
3282  // lastprivate variables.
3283  CGM.getOpenMPRuntime().emitBarrierCall(
3284  *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
3285  /*ForceSimpleCall=*/true);
3286  }
3287  EmitOMPPrivateClause(S, LoopScope);
3291  EmitOMPReductionClauseInit(S, LoopScope);
3292  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3293  EmitOMPPrivateLoopCounters(S, LoopScope);
3294  (void)LoopScope.Privatize();
3295 
3296  // Detect the distribute schedule kind and chunk.
3297  llvm::Value *Chunk = nullptr;
3299  if (auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3300  ScheduleKind = C->getDistScheduleKind();
3301  if (const auto *Ch = C->getChunkSize()) {
3302  Chunk = EmitScalarExpr(Ch);
3303  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3305  S.getLocStart());
3306  }
3307  }
3308  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3309  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3310 
3311  // OpenMP [2.10.8, distribute Construct, Description]
3312  // If dist_schedule is specified, kind must be static. If specified,
3313  // iterations are divided into chunks of size chunk_size, chunks are
3314  // assigned to the teams of the league in a round-robin fashion in the
3315  // order of the team number. When no chunk_size is specified, the
3316  // iteration space is divided into chunks that are approximately equal
3317  // in size, and at most one chunk is distributed to each team of the
3318  // league. The size of the chunks is unspecified in this case.
3319  if (RT.isStaticNonchunked(ScheduleKind,
3320  /* Chunked */ Chunk != nullptr)) {
3322  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3323  CGOpenMPRuntime::StaticRTInput StaticInit(
3324  IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3325  LB.getAddress(), UB.getAddress(), ST.getAddress());
3326  RT.emitDistributeStaticInit(*this, S.getLocStart(), ScheduleKind,
3327  StaticInit);
3328  auto LoopExit =
3329  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3330  // UB = min(UB, GlobalUB);
3333  : S.getEnsureUpperBound());
3334  // IV = LB;
3336  ? S.getCombinedInit()
3337  : S.getInit());
3338 
3340  ? S.getCombinedCond()
3341  : S.getCond();
3342 
3343  // for distribute alone, codegen
3344  // while (idx <= UB) { BODY; ++idx; }
3345  // when combined with 'for' (e.g. as in 'distribute parallel for')
3346  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
3347  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3348  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3349  CodeGenLoop(CGF, S, LoopExit);
3350  },
3351  [](CodeGenFunction &) {});
3352  EmitBlock(LoopExit.getBlock());
3353  // Tell the runtime we are done.
3354  RT.emitForStaticFinish(*this, S.getLocStart(), S.getDirectiveKind());
3355  } else {
3356  // Emit the outer loop, which requests its work chunk [LB..UB] from
3357  // runtime and runs the inner loop to process it.
3358  const OMPLoopArguments LoopArguments = {
3359  LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3360  Chunk};
3361  EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3362  CodeGenLoop);
3363  }
3365  EmitOMPSimdFinal(S, [&](CodeGenFunction &CGF) -> llvm::Value * {
3366  return CGF.Builder.CreateIsNotNull(
3367  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
3368  });
3369  }
3370  OpenMPDirectiveKind ReductionKind = OMPD_unknown;
3373  ReductionKind = OMPD_parallel_for_simd;
3374  } else if (isOpenMPParallelDirective(S.getDirectiveKind())) {
3375  ReductionKind = OMPD_parallel_for;
3376  } else if (isOpenMPSimdDirective(S.getDirectiveKind())) {
3377  ReductionKind = OMPD_simd;
3378  } else if (!isOpenMPTeamsDirective(S.getDirectiveKind()) &&
3380  llvm_unreachable(
3381  "No reduction clauses is allowed in distribute directive.");
3382  }
3383  EmitOMPReductionClauseFinal(S, ReductionKind);
3384  // Emit post-update of the reduction variables if IsLastIter != 0.
3386  *this, S, [&](CodeGenFunction &CGF) -> llvm::Value * {
3387  return CGF.Builder.CreateIsNotNull(
3388  CGF.EmitLoadOfScalar(IL, S.getLocStart()));
3389  });
3390  // Emit final copy of the lastprivate variables if IsLastIter != 0.
3391  if (HasLastprivateClause) {
3392  EmitOMPLastprivateClauseFinal(
3393  S, /*NoFinals=*/false,
3394  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
3395  }
3396  }
3397 
3398  // We're now done with the loop, so jump to the continuation block.
3399  if (ContBlock) {
3400  EmitBranch(ContBlock);
3401  EmitBlock(ContBlock, true);
3402  }
3403  }
3404 }
3405 
3407  const OMPDistributeDirective &S) {
3408  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3409 
3411  };
3412  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3413  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3414 }
3415 
3416 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3417  const CapturedStmt *S) {
3418  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3420  CGF.CapturedStmtInfo = &CapStmtInfo;
3421  auto *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3422  Fn->addFnAttr(llvm::Attribute::NoInline);
3423  return Fn;
3424 }
3425 
3427  if (!S.getAssociatedStmt()) {
3428  for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3429  CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3430  return;
3431  }
3432  auto *C = S.getSingleClause<OMPSIMDClause>();
3433  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3434  PrePostActionTy &Action) {
3435  if (C) {
3436  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
3438  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3439  auto *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3440  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getLocStart(),
3441  OutlinedFn, CapturedVars);
3442  } else {
3443  Action.Enter(CGF);
3444  CGF.EmitStmt(
3445  cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3446  }
3447  };
3448  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3449  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart(), !C);
3450 }
3451 
3453  QualType SrcType, QualType DestType,
3454  SourceLocation Loc) {
3455  assert(CGF.hasScalarEvaluationKind(DestType) &&
3456  "DestType must have scalar evaluation kind.");
3457  assert(!Val.isAggregate() && "Must be a scalar or complex.");
3458  return Val.isScalar()
3459  ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType,
3460  Loc)
3461  : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType,
3462  DestType, Loc);
3463 }
3464 
3467  QualType DestType, SourceLocation Loc) {
3468  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3469  "DestType must have complex evaluation kind.");
3470  CodeGenFunction::ComplexPairTy ComplexVal;
3471  if (Val.isScalar()) {
3472  // Convert the input element to the element type of the complex.
3473  auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
3474  auto ScalarVal = CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3475  DestElementType, Loc);
3476  ComplexVal = CodeGenFunction::ComplexPairTy(
3477  ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3478  } else {
3479  assert(Val.isComplex() && "Must be a scalar or complex.");
3480  auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3481  auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
3482  ComplexVal.first = CGF.EmitScalarConversion(
3483  Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3484  ComplexVal.second = CGF.EmitScalarConversion(
3485  Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3486  }
3487  return ComplexVal;
3488 }
3489 
3490 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3491  LValue LVal, RValue RVal) {
3492  if (LVal.isGlobalReg()) {
3493  CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3494  } else {
3495  CGF.EmitAtomicStore(RVal, LVal,
3496  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3497  : llvm::AtomicOrdering::Monotonic,
3498  LVal.isVolatile(), /*IsInit=*/false);
3499  }
3500 }
3501 
3503  QualType RValTy, SourceLocation Loc) {
3504  switch (getEvaluationKind(LVal.getType())) {
3505  case TEK_Scalar:
3506  EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3507  *this, RVal, RValTy, LVal.getType(), Loc)),
3508  LVal);
3509  break;
3510  case TEK_Complex:
3511  EmitStoreOfComplex(
3512  convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3513  /*isInit=*/false);
3514  break;
3515  case TEK_Aggregate:
3516  llvm_unreachable("Must be a scalar or complex.");
3517  }
3518 }
3519 
3520 static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3521  const Expr *X, const Expr *V,
3522  SourceLocation Loc) {
3523  // v = x;
3524  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3525  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3526  LValue XLValue = CGF.EmitLValue(X);
3527  LValue VLValue = CGF.EmitLValue(V);
3528  RValue Res = XLValue.isGlobalReg()
3529  ? CGF.EmitLoadOfLValue(XLValue, Loc)
3530  : CGF.EmitAtomicLoad(
3531  XLValue, Loc,
3532  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3533  : llvm::AtomicOrdering::Monotonic,
3534  XLValue.isVolatile());
3535  // OpenMP, 2.12.6, atomic Construct
3536  // Any atomic construct with a seq_cst clause forces the atomically
3537  // performed operation to include an implicit flush operation without a
3538  // list.
3539  if (IsSeqCst)
3540  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3541  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3542 }
3543 
3544 static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3545  const Expr *X, const Expr *E,
3546  SourceLocation Loc) {
3547  // x = expr;
3548  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3549  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3550  // OpenMP, 2.12.6, atomic Construct
3551  // Any atomic construct with a seq_cst clause forces the atomically
3552  // performed operation to include an implicit flush operation without a
3553  // list.
3554  if (IsSeqCst)
3555  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3556 }
3557 
3558 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3559  RValue Update,
3560  BinaryOperatorKind BO,
3561  llvm::AtomicOrdering AO,
3562  bool IsXLHSInRHSPart) {
3563  auto &Context = CGF.CGM.getContext();
3564  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3565  // expression is simple and atomic is allowed for the given type for the
3566  // target platform.
3567  if (BO == BO_Comma || !Update.isScalar() ||
3568  !Update.getScalarVal()->getType()->isIntegerTy() ||
3569  !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3570  (Update.getScalarVal()->getType() !=
3571  X.getAddress().getElementType())) ||
3572  !X.getAddress().getElementType()->isIntegerTy() ||
3573  !Context.getTargetInfo().hasBuiltinAtomic(
3574  Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3575  return std::make_pair(false, RValue::get(nullptr));
3576 
3577  llvm::AtomicRMWInst::BinOp RMWOp;
3578  switch (BO) {
3579  case BO_Add:
3580  RMWOp = llvm::AtomicRMWInst::Add;
3581  break;
3582  case BO_Sub:
3583  if (!IsXLHSInRHSPart)
3584  return std::make_pair(false, RValue::get(nullptr));
3585  RMWOp = llvm::AtomicRMWInst::Sub;
3586  break;
3587  case BO_And:
3588  RMWOp = llvm::AtomicRMWInst::And;
3589  break;
3590  case BO_Or:
3591  RMWOp = llvm::AtomicRMWInst::Or;
3592  break;
3593  case BO_Xor:
3594  RMWOp = llvm::AtomicRMWInst::Xor;
3595  break;
3596  case BO_LT:
3598  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3599  : llvm::AtomicRMWInst::Max)
3600  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3601  : llvm::AtomicRMWInst::UMax);
3602  break;
3603  case BO_GT:
3605  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3606  : llvm::AtomicRMWInst::Min)
3607  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3608  : llvm::AtomicRMWInst::UMin);
3609  break;
3610  case BO_Assign:
3611  RMWOp = llvm::AtomicRMWInst::Xchg;
3612  break;
3613  case BO_Mul:
3614  case BO_Div:
3615  case BO_Rem:
3616  case BO_Shl:
3617  case BO_Shr:
3618  case BO_LAnd:
3619  case BO_LOr:
3620  return std::make_pair(false, RValue::get(nullptr));
3621  case BO_PtrMemD:
3622  case BO_PtrMemI:
3623  case BO_LE:
3624  case BO_GE:
3625  case BO_EQ:
3626  case BO_NE:
3627  case BO_Cmp:
3628  case BO_AddAssign:
3629  case BO_SubAssign:
3630  case BO_AndAssign:
3631  case BO_OrAssign:
3632  case BO_XorAssign:
3633  case BO_MulAssign:
3634  case BO_DivAssign:
3635  case BO_RemAssign:
3636  case BO_ShlAssign:
3637  case BO_ShrAssign:
3638  case BO_Comma:
3639  llvm_unreachable("Unsupported atomic update operation");
3640  }
3641  auto *UpdateVal = Update.getScalarVal();
3642  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3643  UpdateVal = CGF.Builder.CreateIntCast(
3644  IC, X.getAddress().getElementType(),
3646  }
3647  auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3648  return std::make_pair(true, RValue::get(Res));
3649 }
3650 
3652  LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3653  llvm::AtomicOrdering AO, SourceLocation Loc,
3654  const llvm::function_ref<RValue(RValue)> &CommonGen) {
3655  // Update expressions are allowed to have the following forms:
3656  // x binop= expr; -> xrval + expr;
3657  // x++, ++x -> xrval + 1;
3658  // x--, --x -> xrval - 1;
3659  // x = x binop expr; -> xrval binop expr
3660  // x = expr Op x; - > expr binop xrval;
3661  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3662  if (!Res.first) {
3663  if (X.isGlobalReg()) {
3664  // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3665  // 'xrval'.
3666  EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3667  } else {
3668  // Perform compare-and-swap procedure.
3669  EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3670  }
3671  }
3672  return Res;
3673 }
3674 
3675 static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3676  const Expr *X, const Expr *E,
3677  const Expr *UE, bool IsXLHSInRHSPart,
3678  SourceLocation Loc) {
3679  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3680  "Update expr in 'atomic update' must be a binary operator.");
3681  auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3682  // Update expressions are allowed to have the following forms:
3683  // x binop= expr; -> xrval + expr;
3684  // x++, ++x -> xrval + 1;
3685  // x--, --x -> xrval - 1;
3686  // x = x binop expr; -> xrval binop expr
3687  // x = expr Op x; - > expr binop xrval;
3688  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3689  LValue XLValue = CGF.EmitLValue(X);
3690  RValue ExprRValue = CGF.EmitAnyExpr(E);
3691  auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3692  : llvm::AtomicOrdering::Monotonic;
3693  auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3694  auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3695  auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3696  auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3697  auto Gen =
3698  [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) -> RValue {
3699  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3700  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3701  return CGF.EmitAnyExpr(UE);
3702  };
3704  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3705  // OpenMP, 2.12.6, atomic Construct
3706  // Any atomic construct with a seq_cst clause forces the atomically
3707  // performed operation to include an implicit flush operation without a
3708  // list.
3709  if (IsSeqCst)
3710  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3711 }
3712 
3714  QualType SourceType, QualType ResType,
3715  SourceLocation Loc) {
3716  switch (CGF.getEvaluationKind(ResType)) {
3717  case TEK_Scalar:
3718  return RValue::get(
3719  convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3720  case TEK_Complex: {
3721  auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3722  return RValue::getComplex(Res.first, Res.second);
3723  }
3724  case TEK_Aggregate:
3725  break;
3726  }
3727  llvm_unreachable("Must be a scalar or complex.");
3728 }
3729 
3730 static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3731  bool IsPostfixUpdate, const Expr *V,
3732  const Expr *X, const Expr *E,
3733  const Expr *UE, bool IsXLHSInRHSPart,
3734  SourceLocation Loc) {
3735  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3736  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3737  RValue NewVVal;
3738  LValue VLValue = CGF.EmitLValue(V);
3739  LValue XLValue = CGF.EmitLValue(X);
3740  RValue ExprRValue = CGF.EmitAnyExpr(E);
3741  auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3742  : llvm::AtomicOrdering::Monotonic;
3743  QualType NewVValType;
3744  if (UE) {
3745  // 'x' is updated with some additional value.
3746  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3747  "Update expr in 'atomic capture' must be a binary operator.");
3748  auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3749  // Update expressions are allowed to have the following forms:
3750  // x binop= expr; -> xrval + expr;
3751  // x++, ++x -> xrval + 1;
3752  // x--, --x -> xrval - 1;
3753  // x = x binop expr; -> xrval binop expr
3754  // x = expr Op x; - > expr binop xrval;
3755  auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3756  auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3757  auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3758  NewVValType = XRValExpr->getType();
3759  auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3760  auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3761  IsPostfixUpdate](RValue XRValue) -> RValue {
3762  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3763  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3764  RValue Res = CGF.EmitAnyExpr(UE);
3765  NewVVal = IsPostfixUpdate ? XRValue : Res;
3766  return Res;
3767  };
3768  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3769  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3770  if (Res.first) {
3771  // 'atomicrmw' instruction was generated.
3772  if (IsPostfixUpdate) {
3773  // Use old value from 'atomicrmw'.
3774  NewVVal = Res.second;
3775  } else {
3776  // 'atomicrmw' does not provide new value, so evaluate it using old
3777  // value of 'x'.
3778  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3779  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3780  NewVVal = CGF.EmitAnyExpr(UE);
3781  }
3782  }
3783  } else {
3784  // 'x' is simply rewritten with some 'expr'.
3785  NewVValType = X->getType().getNonReferenceType();
3786  ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3787  X->getType().getNonReferenceType(), Loc);
3788  auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) -> RValue {
3789  NewVVal = XRValue;
3790  return ExprRValue;
3791  };
3792  // Try to perform atomicrmw xchg, otherwise simple exchange.
3793  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3794  XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3795  Loc, Gen);
3796  if (Res.first) {
3797  // 'atomicrmw' instruction was generated.
3798  NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3799  }
3800  }
3801  // Emit post-update store to 'v' of old/new 'x' value.
3802  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3803  // OpenMP, 2.12.6, atomic Construct
3804  // Any atomic construct with a seq_cst clause forces the atomically
3805  // performed operation to include an implicit flush operation without a
3806  // list.
3807  if (IsSeqCst)
3808  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3809 }
3810 
3812  bool IsSeqCst, bool IsPostfixUpdate,
3813  const Expr *X, const Expr *V, const Expr *E,
3814  const Expr *UE, bool IsXLHSInRHSPart,
3815  SourceLocation Loc) {
3816  switch (Kind) {
3817  case OMPC_read:
3818  EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3819  break;
3820  case OMPC_write:
3821  EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3822  break;
3823  case OMPC_unknown:
3824  case OMPC_update:
3825  EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3826  break;
3827  case OMPC_capture:
3828  EmitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3829  IsXLHSInRHSPart, Loc);
3830  break;
3831  case OMPC_if:
3832  case OMPC_final:
3833  case OMPC_num_threads:
3834  case OMPC_private:
3835  case OMPC_firstprivate:
3836  case OMPC_lastprivate:
3837  case OMPC_reduction:
3838  case OMPC_task_reduction:
3839  case OMPC_in_reduction:
3840  case OMPC_safelen:
3841  case OMPC_simdlen:
3842  case OMPC_collapse:
3843  case OMPC_default:
3844  case OMPC_seq_cst:
3845  case OMPC_shared:
3846  case OMPC_linear:
3847  case OMPC_aligned:
3848  case OMPC_copyin:
3849  case OMPC_copyprivate:
3850  case OMPC_flush:
3851  case OMPC_proc_bind:
3852  case OMPC_schedule:
3853  case OMPC_ordered:
3854  case OMPC_nowait:
3855  case OMPC_untied:
3856  case OMPC_threadprivate:
3857  case OMPC_depend:
3858  case OMPC_mergeable:
3859  case OMPC_device:
3860  case OMPC_threads:
3861  case OMPC_simd:
3862  case OMPC_map:
3863  case OMPC_num_teams:
3864  case OMPC_thread_limit:
3865  case OMPC_priority:
3866  case OMPC_grainsize:
3867  case OMPC_nogroup:
3868  case OMPC_num_tasks:
3869  case OMPC_hint:
3870  case OMPC_dist_schedule:
3871  case OMPC_defaultmap:
3872  case OMPC_uniform:
3873  case OMPC_to:
3874  case OMPC_from:
3875  case OMPC_use_device_ptr:
3876  case OMPC_is_device_ptr:
3877  llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3878  }
3879 }
3880 
3882  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
3884  for (auto *C : S.clauses()) {
3885  // Find first clause (skip seq_cst clause, if it is first).
3886  if (C->getClauseKind() != OMPC_seq_cst) {
3887  Kind = C->getClauseKind();
3888  break;
3889  }
3890  }
3891 
3892  const auto *CS =
3893  S.getAssociatedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
3894  if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS)) {
3895  enterFullExpression(EWC);
3896  }
3897  // Processing for statements under 'atomic capture'.
3898  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3899  for (const auto *C : Compound->body()) {
3900  if (const auto *EWC = dyn_cast<ExprWithCleanups>(C)) {
3901  enterFullExpression(EWC);
3902  }
3903  }
3904  }
3905 
3906  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
3907  PrePostActionTy &) {
3908  CGF.EmitStopPoint(CS);
3909  EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
3910  S.getV(), S.getExpr(), S.getUpdateExpr(),
3911  S.isXLHSInRHSPart(), S.getLocStart());
3912  };
3913  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
3914  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
3915 }
3916 
3918  const OMPExecutableDirective &S,
3919  const RegionCodeGenTy &CodeGen) {
3921  CodeGenModule &CGM = CGF.CGM;
3922  const CapturedStmt &CS = *S.getCapturedStmt(OMPD_target);
3923 
3924  llvm::Function *Fn = nullptr;
3925  llvm::Constant *FnID = nullptr;
3926 
3927  const Expr *IfCond = nullptr;
3928  // Check for the at most one if clause associated with the target region.
3929  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3930  if (C->getNameModifier() == OMPD_unknown ||
3931  C->getNameModifier() == OMPD_target) {
3932  IfCond = C->getCondition();
3933  break;
3934  }
3935  }
3936 
3937  // Check if we have any device clause associated with the directive.
3938  const Expr *Device = nullptr;
3939  if (auto *C = S.getSingleClause<OMPDeviceClause>()) {
3940  Device = C->getDevice();
3941  }
3942 
3943  // Check if we have an if clause whose conditional always evaluates to false
3944  // or if we do not have any targets specified. If so the target region is not
3945  // an offload entry point.
3946  bool IsOffloadEntry = true;
3947  if (IfCond) {
3948  bool Val;
3949  if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
3950  IsOffloadEntry = false;
3951  }
3952  if (CGM.getLangOpts().OMPTargetTriples.empty())
3953  IsOffloadEntry = false;
3954 
3955  assert(CGF.CurFuncDecl && "No parent declaration for target region!");
3956  StringRef ParentName;
3957  // In case we have Ctors/Dtors we use the complete type variant to produce
3958  // the mangling of the device outlined kernel.
3959  if (auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
3960  ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
3961  else if (auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
3962  ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
3963  else
3964  ParentName =
3965  CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
3966 
3967  // Emit target region as a standalone region.
3968  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
3969  IsOffloadEntry, CodeGen);
3970  OMPLexicalScope Scope(CGF, S);
3972  CGF.GenerateOpenMPCapturedVars(CS, CapturedVars);
3973  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
3974  CapturedVars);
3975 }
3976 
3978  PrePostActionTy &Action) {
3979  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
3980  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
3981  CGF.EmitOMPPrivateClause(S, PrivateScope);
3982  (void)PrivateScope.Privatize();
3983 
3984  Action.Enter(CGF);
3985  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
3986 }
3987 
3989  StringRef ParentName,
3990  const OMPTargetDirective &S) {
3991  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3992  emitTargetRegion(CGF, S, Action);
3993  };
3994  llvm::Function *Fn;
3995  llvm::Constant *Addr;
3996  // Emit target region as a standalone region.
3997  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
3998  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
3999  assert(Fn && Addr && "Target device function emission failed.");
4000 }
4001 
4003  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4004  emitTargetRegion(CGF, S, Action);
4005  };
4006  emitCommonOMPTargetDirective(*this, S, CodeGen);
4007 }
4008 
4010  const OMPExecutableDirective &S,
4011  OpenMPDirectiveKind InnermostKind,
4012  const RegionCodeGenTy &CodeGen) {
4013  const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4014  auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4015  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4016 
4019  if (NT || TL) {
4020  Expr *NumTeams = (NT) ? NT->getNumTeams() : nullptr;
4021  Expr *ThreadLimit = (TL) ? TL->getThreadLimit() : nullptr;
4022 
4023  CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4024  S.getLocStart());
4025  }
4026 
4027  OMPTeamsScope Scope(CGF, S);
4029  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4030  CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getLocStart(), OutlinedFn,
4031  CapturedVars);
4032 }
4033 
4035  // Emit teams region as a standalone region.
4036  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4037  OMPPrivateScope PrivateScope(CGF);
4038  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4039  CGF.EmitOMPPrivateClause(S, PrivateScope);
4040  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4041  (void)PrivateScope.Privatize();
4042  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
4043  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4044  };
4045  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4047  *this, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
4048 }
4049 
4051  const OMPTargetTeamsDirective &S) {
4052  auto *CS = S.getCapturedStmt(OMPD_teams);
4053  Action.Enter(CGF);
4054  // Emit teams region as a standalone region.
4055  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4056  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4057  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4058  CGF.EmitOMPPrivateClause(S, PrivateScope);
4059  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4060  (void)PrivateScope.Privatize();
4061  Action.Enter(CGF);
4062  CGF.EmitStmt(CS->getCapturedStmt());
4063  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4064  };
4065  emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4067  CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
4068 }
4069 
4071  CodeGenModule &CGM, StringRef ParentName,
4072  const OMPTargetTeamsDirective &S) {
4073  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4074  emitTargetTeamsRegion(CGF, Action, S);
4075  };
4076  llvm::Function *Fn;
4077  llvm::Constant *Addr;
4078  // Emit target region as a standalone region.
4079  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4080  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4081  assert(Fn && Addr && "Target device function emission failed.");
4082 }
4083 
4085  const OMPTargetTeamsDirective &S) {
4086  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4087  emitTargetTeamsRegion(CGF, Action, S);
4088  };
4089  emitCommonOMPTargetDirective(*this, S, CodeGen);
4090 }
4091 
4092 static void
4095  Action.Enter(CGF);
4096  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4098  };
4099 
4100  // Emit teams region as a standalone region.
4101  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4102  PrePostActionTy &) {
4103  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4104  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4105  (void)PrivateScope.Privatize();
4106  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4107  CodeGenDistribute);
4108  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4109  };
4110  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4112  [](CodeGenFunction &) { return nullptr; });
4113 }
4114 
4116  CodeGenModule &CGM, StringRef ParentName,
4118  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4119  emitTargetTeamsDistributeRegion(CGF, Action, S);
4120  };
4121  llvm::Function *Fn;
4122  llvm::Constant *Addr;
4123  // Emit target region as a standalone region.
4124  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4125  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4126  assert(Fn && Addr && "Target device function emission failed.");
4127 }
4128 
4131  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4132  emitTargetTeamsDistributeRegion(CGF, Action, S);
4133  };
4134  emitCommonOMPTargetDirective(*this, S, CodeGen);
4135 }
4136 
4138  CodeGenFunction &CGF, PrePostActionTy &Action,
4140  Action.Enter(CGF);
4141  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4143  };
4144 
4145  // Emit teams region as a standalone region.
4146  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4147  PrePostActionTy &) {
4148  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4149  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4150  (void)PrivateScope.Privatize();
4151  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4152  CodeGenDistribute);
4153  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4154  };
4155  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4157  [](CodeGenFunction &) { return nullptr; });
4158 }
4159 
4161  CodeGenModule &CGM, StringRef ParentName,
4163  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4164  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4165  };
4166  llvm::Function *Fn;
4167  llvm::Constant *Addr;
4168  // Emit target region as a standalone region.
4169  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4170  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4171  assert(Fn && Addr && "Target device function emission failed.");
4172 }
4173 
4176  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4177  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4178  };
4179  emitCommonOMPTargetDirective(*this, S, CodeGen);
4180 }
4181 
4183  const OMPTeamsDistributeDirective &S) {
4184 
4185  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4187  };
4188 
4189  // Emit teams region as a standalone region.
4190  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4191  PrePostActionTy &) {
4192  OMPPrivateScope PrivateScope(CGF);
4193  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4194  (void)PrivateScope.Privatize();
4195  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4196  CodeGenDistribute);
4197  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4198  };
4199  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4201  [](CodeGenFunction &) { return nullptr; });
4202 }
4203 
4206  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4208  };
4209 
4210  // Emit teams region as a standalone region.
4211  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4212  PrePostActionTy &) {
4213  OMPPrivateScope PrivateScope(CGF);
4214  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4215  (void)PrivateScope.Privatize();
4216  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4217  CodeGenDistribute);
4218  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4219  };
4220  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4222  [](CodeGenFunction &) { return nullptr; });
4223 }
4224 
4227  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4229  S.getDistInc());
4230  };
4231 
4232  // Emit teams region as a standalone region.
4233  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4234  PrePostActionTy &) {
4235  OMPPrivateScope PrivateScope(CGF);
4236  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4237  (void)PrivateScope.Privatize();
4238  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4239  CodeGenDistribute);
4240  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4241  };
4242  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4244  [](CodeGenFunction &) { return nullptr; });
4245 }
4246 
4249  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4251  S.getDistInc());
4252  };
4253 
4254  // Emit teams region as a standalone region.
4255  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4256  PrePostActionTy &) {
4257  OMPPrivateScope PrivateScope(CGF);
4258  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4259  (void)PrivateScope.Privatize();
4260  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4261  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4262  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4263  };
4264  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4266  [](CodeGenFunction &) { return nullptr; });
4267 }
4268 
4270  const OMPCancellationPointDirective &S) {
4271  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getLocStart(),
4272  S.getCancelRegion());
4273 }
4274 
4276  const Expr *IfCond = nullptr;
4277  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4278  if (C->getNameModifier() == OMPD_unknown ||
4279  C->getNameModifier() == OMPD_cancel) {
4280  IfCond = C->getCondition();
4281  break;
4282  }
4283  }
4284  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), IfCond,
4285  S.getCancelRegion());
4286 }
4287 
4290  if (Kind == OMPD_parallel || Kind == OMPD_task ||
4291  Kind == OMPD_target_parallel)
4292  return ReturnBlock;
4293  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4294  Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4295  Kind == OMPD_distribute_parallel_for ||
4296  Kind == OMPD_target_parallel_for ||
4297  Kind == OMPD_teams_distribute_parallel_for ||
4298  Kind == OMPD_target_teams_distribute_parallel_for);
4299  return OMPCancelStack.getExitBlock();
4300 }
4301 
4303  const OMPClause &NC, OMPPrivateScope &PrivateScope,
4304  const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4305  const auto &C = cast<OMPUseDevicePtrClause>(NC);
4306  auto OrigVarIt = C.varlist_begin();
4307  auto InitIt = C.inits().begin();
4308  for (auto PvtVarIt : C.private_copies()) {
4309  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4310  auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4311  auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4312 
4313  // In order to identify the right initializer we need to match the
4314  // declaration used by the mapping logic. In some cases we may get
4315  // OMPCapturedExprDecl that refers to the original declaration.
4316  const ValueDecl *MatchingVD = OrigVD;
4317  if (auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4318  // OMPCapturedExprDecl are used to privative fields of the current
4319  // structure.
4320  auto *ME = cast<MemberExpr>(OED->getInit());
4321  assert(isa<CXXThisExpr>(ME->getBase()) &&
4322  "Base should be the current struct!");
4323  MatchingVD = ME->getMemberDecl();
4324  }
4325 
4326  // If we don't have information about the current list item, move on to
4327  // the next one.
4328  auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4329  if (InitAddrIt == CaptureDeviceAddrMap.end())
4330  continue;
4331 
4332  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
4333  // Initialize the temporary initialization variable with the address we
4334  // get from the runtime library. We have to cast the source address
4335  // because it is always a void *. References are materialized in the
4336  // privatization scope, so the initialization here disregards the fact
4337  // the original variable is a reference.
4338  QualType AddrQTy =
4339  getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4340  llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4341  Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4342  setAddrOfLocalVar(InitVD, InitAddr);
4343 
4344  // Emit private declaration, it will be initialized by the value we
4345  // declaration we just added to the local declarations map.
4346  EmitDecl(*PvtVD);
4347 
4348  // The initialization variables reached its purpose in the emission
4349  // ofthe previous declaration, so we don't need it anymore.
4350  LocalDeclMap.erase(InitVD);
4351 
4352  // Return the address of the private variable.
4353  return GetAddrOfLocalVar(PvtVD);
4354  });
4355  assert(IsRegistered && "firstprivate var already registered as private");
4356  // Silence the warning about unused variable.
4357  (void)IsRegistered;
4358 
4359  ++OrigVarIt;
4360  ++InitIt;
4361  }
4362 }
4363 
4364 // Generate the instructions for '#pragma omp target data' directive.
4366  const OMPTargetDataDirective &S) {
4367  CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4368 
4369  // Create a pre/post action to signal the privatization of the device pointer.
4370  // This action can be replaced by the OpenMP runtime code generation to
4371  // deactivate privatization.
4372  bool PrivatizeDevicePointers = false;
4373  class DevicePointerPrivActionTy : public PrePostActionTy {
4374  bool &PrivatizeDevicePointers;
4375 
4376  public:
4377  explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4378  : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4379  void Enter(CodeGenFunction &CGF) override {
4380  PrivatizeDevicePointers = true;
4381  }
4382  };
4383  DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4384 
4385  auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4386  CodeGenFunction &CGF, PrePostActionTy &Action) {
4387  auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4388  CGF.EmitStmt(
4389  cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
4390  };
4391 
4392  // Codegen that selects wheather to generate the privatization code or not.
4393  auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4394  &InnermostCodeGen](CodeGenFunction &CGF,
4395  PrePostActionTy &Action) {
4396  RegionCodeGenTy RCG(InnermostCodeGen);
4397  PrivatizeDevicePointers = false;
4398 
4399  // Call the pre-action to change the status of PrivatizeDevicePointers if
4400  // needed.
4401  Action.Enter(CGF);
4402 
4403  if (PrivatizeDevicePointers) {
4404  OMPPrivateScope PrivateScope(CGF);
4405  // Emit all instances of the use_device_ptr clause.
4406  for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4407  CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4408  Info.CaptureDeviceAddrMap);
4409  (void)PrivateScope.Privatize();
4410  RCG(CGF);
4411  } else
4412  RCG(CGF);
4413  };
4414 
4415  // Forward the provided action to the privatization codegen.
4416  RegionCodeGenTy PrivRCG(PrivCodeGen);
4417  PrivRCG.setAction(Action);
4418 
4419  // Notwithstanding the body of the region is emitted as inlined directive,
4420  // we don't use an inline scope as changes in the references inside the
4421  // region are expected to be visible outside, so we do not privative them.
4422  OMPLexicalScope Scope(CGF, S);
4423  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4424  PrivRCG);
4425  };
4426 
4427  RegionCodeGenTy RCG(CodeGen);
4428 
4429  // If we don't have target devices, don't bother emitting the data mapping
4430  // code.
4431  if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4432  RCG(*this);
4433  return;
4434  }
4435 
4436  // Check if we have any if clause associated with the directive.
4437  const Expr *IfCond = nullptr;
4438  if (auto *C = S.getSingleClause<OMPIfClause>())
4439  IfCond = C->getCondition();
4440 
4441  // Check if we have any device clause associated with the directive.
4442  const Expr *Device = nullptr;
4443  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4444  Device = C->getDevice();
4445 
4446  // Set the action to signal privatization of device pointers.
4447  RCG.setAction(PrivAction);
4448 
4449  // Emit region code.
4450  CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4451  Info);
4452 }
4453 
4455  const OMPTargetEnterDataDirective &S) {
4456  // If we don't have target devices, don't bother emitting the data mapping
4457  // code.
4458  if (CGM.getLangOpts().OMPTargetTriples.empty())
4459  return;
4460 
4461  // Check if we have any if clause associated with the directive.
4462  const Expr *IfCond = nullptr;
4463  if (auto *C = S.getSingleClause<OMPIfClause>())
4464  IfCond = C->getCondition();
4465 
4466  // Check if we have any device clause associated with the directive.
4467  const Expr *Device = nullptr;
4468  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4469  Device = C->getDevice();
4470 
4471  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
4472  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4473 }
4474 
4476  const OMPTargetExitDataDirective &S) {
4477  // If we don't have target devices, don't bother emitting the data mapping
4478  // code.
4479  if (CGM.getLangOpts().OMPTargetTriples.empty())
4480  return;
4481 
4482  // Check if we have any if clause associated with the directive.
4483  const Expr *IfCond = nullptr;
4484  if (auto *C = S.getSingleClause<OMPIfClause>())
4485  IfCond = C->getCondition();
4486 
4487  // Check if we have any device clause associated with the directive.
4488  const Expr *Device = nullptr;
4489  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4490  Device = C->getDevice();
4491 
4492  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
4493  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4494 }
4495 
4497  const OMPTargetParallelDirective &S,
4498  PrePostActionTy &Action) {
4499  // Get the captured statement associated with the 'parallel' region.
4500  auto *CS = S.getCapturedStmt(OMPD_parallel);
4501  Action.Enter(CGF);
4502  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &) {
4503  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4504  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4505  CGF.EmitOMPPrivateClause(S, PrivateScope);
4506  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4507  (void)PrivateScope.Privatize();
4508  // TODO: Add support for clauses.
4509  CGF.EmitStmt(CS->getCapturedStmt());
4510  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4511  };
4512  emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4515  CGF, S, [](CodeGenFunction &) -> llvm::Value * { return nullptr; });
4516 }
4517 
4519  CodeGenModule &CGM, StringRef ParentName,
4520  const OMPTargetParallelDirective &S) {
4521  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4522  emitTargetParallelRegion(CGF, S, Action);
4523  };
4524  llvm::Function *Fn;
4525  llvm::Constant *Addr;
4526  // Emit target region as a standalone region.
4527  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4528  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4529  assert(Fn && Addr && "Target device function emission failed.");
4530 }
4531 
4533  const OMPTargetParallelDirective &S) {
4534  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4535  emitTargetParallelRegion(CGF, S, Action);
4536  };
4537  emitCommonOMPTargetDirective(*this, S, CodeGen);
4538 }
4539 
4542  PrePostActionTy &Action) {
4543  Action.Enter(CGF);
4544  // Emit directive as a combined directive that consists of two implicit
4545  // directives: 'parallel' with 'for' directive.
4546  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4548  CGF, OMPD_target_parallel_for, S.hasCancel());
4551  };
4552  emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4554 }
4555 
4557  CodeGenModule &CGM, StringRef ParentName,
4558  const OMPTargetParallelForDirective &S) {
4559  // Emit SPMD target parallel for region as a standalone region.
4560  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4561  emitTargetParallelForRegion(CGF, S, Action);
4562  };
4563  llvm::Function *Fn;
4564  llvm::Constant *Addr;
4565  // Emit target region as a standalone region.
4566  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4567  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4568  assert(Fn && Addr && "Target device function emission failed.");
4569 }
4570 
4572  const OMPTargetParallelForDirective &S) {
4573  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4574  emitTargetParallelForRegion(CGF, S, Action);
4575  };
4576  emitCommonOMPTargetDirective(*this, S, CodeGen);
4577 }
4578 
4579 static void
4582  PrePostActionTy &Action) {
4583  Action.Enter(CGF);
4584  // Emit directive as a combined directive that consists of two implicit
4585  // directives: 'parallel' with 'for' directive.
4586  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4589  };
4590  emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4592 }
4593 
4595  CodeGenModule &CGM, StringRef ParentName,
4597  // Emit SPMD target parallel for region as a standalone region.
4598  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4599  emitTargetParallelForSimdRegion(CGF, S, Action);
4600  };
4601  llvm::Function *Fn;
4602  llvm::Constant *Addr;
4603  // Emit target region as a standalone region.
4604  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4605  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4606  assert(Fn && Addr && "Target device function emission failed.");
4607 }
4608 
4611  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4612  emitTargetParallelForSimdRegion(CGF, S, Action);
4613  };
4614  emitCommonOMPTargetDirective(*this, S, CodeGen);
4615 }
4616 
4617 /// Emit a helper variable and return corresponding lvalue.
4618 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4619  const ImplicitParamDecl *PVD,
4621  auto *VDecl = cast<VarDecl>(Helper->getDecl());
4622  Privates.addPrivate(
4623  VDecl, [&CGF, PVD]() -> Address { return CGF.GetAddrOfLocalVar(PVD); });
4624 }
4625 
4628  // Emit outlined function for task construct.
4629  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
4630  auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
4631  auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4632  const Expr *IfCond = nullptr;
4633  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4634  if (C->getNameModifier() == OMPD_unknown ||
4635  C->getNameModifier() == OMPD_taskloop) {
4636  IfCond = C->getCondition();
4637  break;
4638  }
4639  }
4640 
4641  OMPTaskDataTy Data;
4642  // Check if taskloop must be emitted without taskgroup.
4644  // TODO: Check if we should emit tied or untied task.
4645  Data.Tied = true;
4646  // Set scheduling for taskloop
4647  if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4648  // grainsize clause
4649  Data.Schedule.setInt(/*IntVal=*/false);
4650  Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4651  } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4652  // num_tasks clause
4653  Data.Schedule.setInt(/*IntVal=*/true);
4654  Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4655  }
4656 
4657  auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4658  // if (PreCond) {
4659  // for (IV in 0..LastIteration) BODY;
4660  // <Final counter/linear vars updates>;
4661  // }
4662  //
4663 
4664  // Emit: if (PreCond) - begin.
4665  // If the condition constant folds and can be elided, avoid emitting the
4666  // whole loop.
4667  bool CondConstant;
4668  llvm::BasicBlock *ContBlock = nullptr;
4669  OMPLoopScope PreInitScope(CGF, S);
4670  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4671  if (!CondConstant)
4672  return;
4673  } else {
4674  auto *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4675  ContBlock = CGF.createBasicBlock("taskloop.if.end");
4676  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4677  CGF.getProfileCount(&S));
4678  CGF.EmitBlock(ThenBlock);
4679  CGF.incrementProfileCounter(&S);
4680  }
4681 
4683  CGF.EmitOMPSimdInit(S);
4684 
4685  OMPPrivateScope LoopScope(CGF);
4686  // Emit helper vars inits.
4687  enum { LowerBound = 5, UpperBound, Stride, LastIter };
4688  auto *I = CS->getCapturedDecl()->param_begin();
4689  auto *LBP = std::next(I, LowerBound);
4690  auto *UBP = std::next(I, UpperBound);
4691  auto *STP = std::next(I, Stride);
4692  auto *LIP = std::next(I, LastIter);
4693  mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4694  LoopScope);
4695  mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4696  LoopScope);
4697  mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4698  mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4699  LoopScope);
4700  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4701  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4702  (void)LoopScope.Privatize();
4703  // Emit the loop iteration variable.
4704  const Expr *IVExpr = S.getIterationVariable();
4705  const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4706  CGF.EmitVarDecl(*IVDecl);
4707  CGF.EmitIgnoredExpr(S.getInit());
4708 
4709  // Emit the iterations count variable.
4710  // If it is not a variable, Sema decided to calculate iterations count on
4711  // each iteration (e.g., it is foldable into a constant).
4712  if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4713  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4714  // Emit calculation of the iterations count.
4715  CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4716  }
4717 
4718  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4719  S.getInc(),
4720  [&S](CodeGenFunction &CGF) {
4721  CGF.EmitOMPLoopBody(S, JumpDest());
4722  CGF.EmitStopPoint(&S);
4723  },
4724  [](CodeGenFunction &) {});
4725  // Emit: if (PreCond) - end.
4726  if (ContBlock) {
4727  CGF.EmitBranch(ContBlock);
4728  CGF.EmitBlock(ContBlock, true);
4729  }
4730  // Emit final copy of the lastprivate variables if IsLastIter != 0.
4731  if (HasLastprivateClause) {
4732  CGF.EmitOMPLastprivateClauseFinal(
4734  CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4735  CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4736  (*LIP)->getType(), S.getLocStart())));
4737  }
4738  };
4739  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4740  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
4741  const OMPTaskDataTy &Data) {
4742  auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &) {
4743  OMPLoopScope PreInitScope(CGF, S);
4744  CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getLocStart(), S,
4745  OutlinedFn, SharedsTy,
4746  CapturedStruct, IfCond, Data);
4747  };
4748  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4749  CodeGen);
4750  };
4751  if (Data.Nogroup)
4752  EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
4753  else {
4754  CGM.getOpenMPRuntime().emitTaskgroupRegion(
4755  *this,
4756  [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
4757  PrePostActionTy &Action) {
4758  Action.Enter(CGF);
4759  CGF.EmitOMPTaskBasedDirective(S, BodyGen, TaskGen, Data);
4760  },
4761  S.getLocStart());
4762  }
4763 }
4764 
4766  EmitOMPTaskLoopBasedDirective(S);
4767 }
4768 
4770  const OMPTaskLoopSimdDirective &S) {
4771  EmitOMPTaskLoopBasedDirective(S);
4772 }
4773 
4774 // Generate the instructions for '#pragma omp target update' directive.
4776  const OMPTargetUpdateDirective &S) {
4777  // If we don't have target devices, don't bother emitting the data mapping
4778  // code.
4779  if (CGM.getLangOpts().OMPTargetTriples.empty())
4780  return;
4781 
4782  // Check if we have any if clause associated with the directive.
4783  const Expr *IfCond = nullptr;
4784  if (auto *C = S.getSingleClause<OMPIfClause>())
4785  IfCond = C->getCondition();
4786 
4787  // Check if we have any device clause associated with the directive.
4788  const Expr *Device = nullptr;
4789  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4790  Device = C->getDevice();
4791 
4792  OMPLexicalScope Scope(*this, S, /*AsInlined=*/true);
4793  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4794 }
4795 
4797  const OMPExecutableDirective &D) {
4798  if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
4799  return;
4800  auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
4802  emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
4803  } else {
4804  if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
4805  for (const auto *E : LD->counters()) {
4806  if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
4807  cast<DeclRefExpr>(E)->getDecl())) {
4808  // Emit only those that were not explicitly referenced in clauses.
4809  if (!CGF.LocalDeclMap.count(VD))
4810  CGF.EmitVarDecl(*VD);
4811  }
4812  }
4813  }
4814  const auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
4815  while (const auto *CCS = dyn_cast<CapturedStmt>(CS->getCapturedStmt()))
4816  CS = CCS;
4817  CGF.EmitStmt(CS->getCapturedStmt());
4818  }
4819  };
4820  OMPSimdLexicalScope Scope(*this, D);
4821  CGM.getOpenMPRuntime().emitInlinedDirective(
4822  *this,
4823  isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
4824  : D.getDirectiveKind(),
4825  CodeGen);
4826 }
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:640
bool isAggregate() const
Definition: CGValue.h:54
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3194
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:822
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1377
static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType, StringRef Name, LValue AddrLV, bool isReferenceType=false)
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1717
static const Decl * getCanonicalDecl(const Decl *D)
An instance of this class is created to represent a function declaration or definition.
Definition: Decl.h:1697
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:790
Other implicit parameter.
Definition: Decl.h:1473
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Complete object ctor.
Definition: ABI.h:26
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2283
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:124
A (possibly-)qualified type.
Definition: Type.h:653
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:2232
bool isArrayType() const
Definition: Type.h:5991
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:239
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:156
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:3946
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> &CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
Definition: Stmt.h:66
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:782
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2071
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:456
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1127
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:86
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3605
bool isNothrow() const
Definition: Decl.cpp:4275
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:240
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:5891
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool isVolatile() const
Definition: CGValue.h:298
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
Definition: Type.h:1351
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:3742
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1836
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:2176
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:671
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition: CGExpr.cpp:2232
Floating point control options.
Definition: LangOptions.h:208
This represents &#39;#pragma omp parallel for&#39; directive.
Definition: StmtOpenMP.h:1498
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
This represents &#39;#pragma omp target teams distribute parallel for&#39; combined directive.
Definition: StmtOpenMP.h:3810
Expr * getCombinedEnsureUpperBound() const
Definition: StmtOpenMP.h:874
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1871
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Represents a point when we exit a loop.
Definition: ProgramPoint.h:664
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
SmallVector< const Expr *, 4 > ReductionCopies
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
This represents &#39;#pragma omp target exit data&#39; directive.
Definition: StmtOpenMP.h:2409
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
CanQualType getCanonicalParamType(QualType T) const
Return the canonical parameter type corresponding to the specific potentially non-canonical one...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
VarDecl - An instance of this class is created to represent a variable declaration or definition...
Definition: Decl.h:806
This represents clause &#39;private&#39; in the &#39;#pragma omp ...&#39; directives.
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:2592
This represents &#39;num_threads&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:382
const Stmt * getBody() const
Definition: StmtOpenMP.h:904
Extra information about a function prototype.
Definition: Type.h:3387
uint64_t getProfileCount(const Stmt *S)
Get the profiler&#39;s count for the given statement.
Expr * getCombinedUpperBoundVariable() const
Definition: StmtOpenMP.h:868
Expr * getCalcLastIteration() const
Definition: StmtOpenMP.h:750
llvm::Value * getPointer() const
Definition: Address.h:38
unsigned getContextParamPosition() const
Definition: Decl.h:3940
This represents implicit clause &#39;flush&#39; for the &#39;#pragma omp flush&#39; directive.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
bool isXLHSInRHSPart() const
Return true if helper update expression has form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; and...
Definition: StmtOpenMP.h:2206
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
This represents &#39;nogroup&#39; clause in the &#39;#pragma omp ...&#39; directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
This represents &#39;safelen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:447
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
IdentifierInfo * getIdentifier() const
getIdentifier - Get the identifier that names this declaration, if there is one.
Definition: Decl.h:265
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
Definition: Expr.h:2865
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
SmallVector< const Expr *, 4 > PrivateVars
RecordDecl - Represents a struct/union/class.
Definition: Decl.h:3488
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
One of these records is kept for each identifier that is lexed.
void EmitOMPSimdDirective(const OMPSimdDirective &S)
This represents &#39;#pragma omp parallel&#39; directive.
Definition: StmtOpenMP.h:255
Address getAddress() const
Definition: CGValue.h:324
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:149
A C++ nested-name-specifier augmented with source location information.
This represents &#39;simd&#39; clause in the &#39;#pragma omp ...&#39; directive.
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
field_range fields() const
Definition: Decl.h:3619
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
Expr * getEnsureUpperBound() const
Definition: StmtOpenMP.h:806
SmallVector< const Expr *, 4 > LastprivateVars
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
Definition: CGExpr.cpp:2210
This represents clause &#39;lastprivate&#39; in the &#39;#pragma omp ...&#39; directives.
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
CharUnits getAlignment() const
Definition: CGValue.h:313
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive...
Definition: StmtOpenMP.h:209
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Expr * getInc() const
Definition: StmtOpenMP.h:766
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, bool IsMonotonic)
ArrayRef< Expr * > updates()
Definition: StmtOpenMP.h:935
bool isReferenceType() const
Definition: Type.h:5954
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr)
Definition: Expr.cpp:391
This represents &#39;#pragma omp target simd&#39; directive.
Definition: StmtOpenMP.h:3330
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself...
OpenMPDirectiveKind getDirectiveKind() const
Definition: StmtOpenMP.h:225
This represents &#39;#pragma omp barrier&#39; directive.
Definition: StmtOpenMP.h:1829
This is a common base class for loop directives (&#39;omp simd&#39;, &#39;omp for&#39;, &#39;omp for simd&#39; etc...
Definition: StmtOpenMP.h:317
Expr * getNumTeams()
Return NumTeams number.
This represents &#39;#pragma omp critical&#39; directive.
Definition: StmtOpenMP.h:1424
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:2708
IdentifierTable & Idents
Definition: ASTContext.h:537
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
Definition: Specifiers.h:107
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
Definition: CGExpr.cpp:173
This represents clause &#39;copyprivate&#39; in the &#39;#pragma omp ...&#39; directives.
OpenMPDistScheduleClauseKind
OpenMP attributes for &#39;dist_schedule&#39; clause.
Definition: OpenMPKinds.h:100
This represents &#39;#pragma omp distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3045
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
Definition: Decl.cpp:4272
This represents &#39;#pragma omp teams distribute parallel for simd&#39; composite directive.
Definition: StmtOpenMP.h:3534
BinaryOperatorKind
static bool hasScalarEvaluationKind(QualType T)
SourceLocation getLocEnd() const
Returns ending location of directive.
Definition: StmtOpenMP.h:170
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
Definition: Stmt.h:2219
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
ArrayRef< Expr * > finals()
Definition: StmtOpenMP.h:941
Expr * getIsLastIterVariable() const
Definition: StmtOpenMP.h:774
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
OpenMPScheduleClauseKind Schedule
Definition: OpenMPKinds.h:125
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value *> &CapturedVars)
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
Definition: TargetInfo.h:457
llvm::function_ref< std::pair< LValue, LValue > CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
Expr * getX()
Get &#39;x&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2190
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
Definition: Type.h:6354
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
Definition: Decl.h:3948
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:67
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
Expr * getIterationVariable() const
Definition: StmtOpenMP.h:742
bool isComplex() const
Definition: CGValue.h:53
A builtin binary operation expression such as "x + y" or "x <= y".
Definition: Expr.h:2985
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
Definition: Type.h:5788
OpenMPScheduleClauseModifier M2
Definition: OpenMPKinds.h:127
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
Definition: CGLoopInfo.h:147
SmallVector< const Expr *, 4 > PrivateCopies
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S)
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
This represents &#39;#pragma omp cancellation point&#39; directive.
Definition: StmtOpenMP.h:2664
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations...
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:39
field_iterator field_begin() const
Definition: Decl.cpp:3960
This represents &#39;final&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:330
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
void EmitOMPParallelDirective(const OMPParallelDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
This represents &#39;#pragma omp teams&#39; directive.
Definition: StmtOpenMP.h:2607
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition: CGExpr.cpp:161
SmallVector< const Expr *, 4 > FirstprivateCopies
Expr * getInit() const
Definition: StmtOpenMP.h:762
This represents clause &#39;reduction&#39; in the &#39;#pragma omp ...&#39; directives.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
This represents &#39;#pragma omp teams distribute simd&#39; combined directive.
Definition: StmtOpenMP.h:3464
bool isSimple() const
Definition: CGValue.h:249
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
An ordinary object is located at an address in memory.
Definition: Specifiers.h:123
SmallVector< const Expr *, 4 > ReductionOps
Controls insertion of cancellation exit blocks in worksharing constructs.
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler&#39;s counter for the given statement by StepV.
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * > CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
This represents the body of a CapturedStmt, and serves as its DeclContext.
Definition: Decl.h:3866
SmallVector< const Expr *, 4 > ReductionVars
static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, bool isConstexprSpecified=false)
Definition: Decl.h:1881
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address >> &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value *>> &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
SourceLocation getLocStart() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:168
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
Definition: Decl.cpp:4210
This represents &#39;#pragma omp target parallel for simd&#39; directive.
Definition: StmtOpenMP.h:3262
ArrayRef< Expr * > private_counters()
Definition: StmtOpenMP.h:923
CompoundStmt - This represents a group of statements like { stmt stmt }.
Definition: Stmt.h:595
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
Definition: CGDecl.cpp:961
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1307
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
This represents &#39;#pragma omp taskgroup&#39; directive.
Definition: StmtOpenMP.h:1917
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
Definition: CGValue.h:39
This represents clause &#39;aligned&#39; in the &#39;#pragma omp ...&#39; directives.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat)
Definition: Expr.cpp:1718
SourceLocation getLocEnd() const LLVM_READONLY
Definition: Stmt.cpp:290
bool addPrivate(const VarDecl *LocalVD, llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
Class intended to support codegen of all kind of the reduction clauses.
bool isGlobalVarCaptured(const VarDecl *VD) const
Checks if the global variable is captured in current function.
Expr * getCombinedLowerBoundVariable() const
Definition: StmtOpenMP.h:862
This represents clause &#39;task_reduction&#39; in the &#39;#pragma omp taskgroup&#39; directives.
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:179
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
This represents &#39;#pragma omp distribute&#39; directive.
Definition: StmtOpenMP.h:2918
This represents implicit clause &#39;depend&#39; for the &#39;#pragma omp task&#39; directive.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
const Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
Definition: StmtOpenMP.h:196
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1561
This represents &#39;proc_bind&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:675
ValueDecl - Represent the declaration of a variable (in which case it is an lvalue) a function (in wh...
Definition: Decl.h:627
Expr - This represents one expression.
Definition: Expr.h:106
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
Emit only debug info necessary for generating line number tables (-gline-tables-only).
void EmitAutoVarInit(const AutoVarEmission &emission)
Definition: CGDecl.cpp:1211
static Address invalid()
Definition: Address.h:35
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
const FunctionProtoType * T
This represents &#39;simdlen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:501
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top, if IgnoreCaptured is true.
Definition: Stmt.cpp:132
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type, where the destination type is an LLVM scalar type.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:66
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
const T * castAs() const
Member-template castAs<specific type>.
Definition: Type.h:6368
void SetInternalFunctionAttributes(const Decl *D, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1111
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
This represents &#39;#pragma omp target teams distribute parallel for simd&#39; combined directive.
Definition: StmtOpenMP.h:3894
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
OpenMPClauseKind
OpenMP clauses.
Definition: OpenMPKinds.h:33
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:44
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause *> Clauses)
Definition: StmtOpenMP.h:130
This represents &#39;#pragma omp target teams distribute simd&#39; combined directive.
Definition: StmtOpenMP.h:3967
void EmitOMPSimdInit(const OMPLoopDirective &D, bool IsMonotonic=false)
Helpers for the OpenMP loop directives.
This represents &#39;ordered&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:925
Expr * getDistInc() const
Definition: StmtOpenMP.h:850
Expr * getNextLowerBound() const
Definition: StmtOpenMP.h:814
QualType getType() const
Definition: Expr.h:128
Expr * getPrevEnsureUpperBound() const
Definition: StmtOpenMP.h:856
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> &CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
This represents &#39;#pragma omp for&#39; directive.
Definition: StmtOpenMP.h:1050
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
SmallVector< const Expr *, 4 > FirstprivateVars
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
Definition: Decl.cpp:4260
void EmitOMPMasterDirective(const OMPMasterDirective &S)
static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the &#39;for&#39; loop has a dispatch schedule (e.g.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
Definition: CharUnits.h:197
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
Definition: CGLoopInfo.h:127
This represents &#39;#pragma omp target teams&#39; directive.
Definition: StmtOpenMP.h:3683
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
SourceLocation getEnd() const
UnaryOperator - This represents the unary-expression&#39;s (except sizeof and alignof), the postinc/postdec operators from postfix-expression, and various extensions.
Definition: Expr.h:1717
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
Definition: ASTContext.h:1333
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
This represents &#39;#pragma omp cancel&#39; directive.
Definition: StmtOpenMP.h:2722
This represents clause &#39;firstprivate&#39; in the &#39;#pragma omp ...&#39; directives.
ValueDecl * getDecl()
Definition: Expr.h:1041
static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const LangOptions & getLangOpts() const
ASTContext & getContext() const
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
Definition: CGAtomic.cpp:1449
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
Definition: Decl.cpp:1977
GlobalDecl - represents a global declaration.
Definition: GlobalDecl.h:35
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
This represents &#39;#pragma omp flush&#39; directive.
Definition: StmtOpenMP.h:1990
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind. ...
Definition: StmtOpenMP.h:162
This represents &#39;#pragma omp parallel for simd&#39; directive.
Definition: StmtOpenMP.h:1578
This represents &#39;seq_cst&#39; clause in the &#39;#pragma omp atomic&#39; directive.
This represents &#39;untied&#39; clause in the &#39;#pragma omp ...&#39; directive.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
Definition: DeclBase.h:984
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
The l-value was considered opaque, so the alignment was determined from a type.
void EmitAlignmentAssumption(llvm::Value *PtrValue, unsigned Alignment, llvm::Value *OffsetValue=nullptr)
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
Expr * getLastIteration() const
Definition: StmtOpenMP.h:746
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
Definition: StmtOpenMP.h:148
bool isPostfixUpdate() const
Return true if &#39;v&#39; expression must be updated to original value of &#39;x&#39;, false if &#39;v&#39; must be updated ...
Definition: StmtOpenMP.h:2209
This represents &#39;#pragma omp target enter data&#39; directive.
Definition: StmtOpenMP.h:2350
void EmitOMPFlushDirective(const OMPFlushDirective &S)
Expr * getStrideVariable() const
Definition: StmtOpenMP.h:798
This represents &#39;num_teams&#39; clause in the &#39;#pragma omp ...&#39; directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
Definition: Expr.h:868
#define false
Definition: stdbool.h:33
Kind
This captures a statement into a function.
Definition: Stmt.h:2058
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
This represents &#39;#pragma omp single&#39; directive.
Definition: StmtOpenMP.h:1322
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
This represents &#39;hint&#39; clause in the &#39;#pragma omp ...&#39; directive.
void EmitOMPForDirective(const OMPForDirective &S)
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
Pseudo declaration for capturing expressions.
Definition: DeclOpenMP.h:187
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Definition: Type.h:1874
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
This is a basic class for representing single OpenMP executable directive.
Definition: StmtOpenMP.h:33
This represents &#39;schedule&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:744
Expr * getExpr()
Get &#39;expr&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2216
OpenMPDirectiveKind
OpenMP directives.
Definition: OpenMPKinds.h:23
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> &CondGen)
Emit final code for linear clauses.
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
DeclarationNameInfo getDirectiveName() const
Return name of the directive.
Definition: StmtOpenMP.h:1482
void EmitStmt(const Stmt *S, ArrayRef< const Attr *> Attrs=None)
EmitStmt - Emit the code for the statement.
Definition: CGStmt.cpp:48
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
Definition: Decl.h:1067
C-style initialization with assignment.
Definition: Decl.h:811
This represents &#39;#pragma omp taskwait&#39; directive.
Definition: StmtOpenMP.h:1873
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
This file defines OpenMP nodes for declarative directives.
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
This is a basic class for representing single OpenMP clause.
Definition: OpenMPClause.h:51
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
CanQualType VoidTy
Definition: ASTContext.h:996
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
bool isAnyPointerType() const
Definition: Type.h:5946
virtual void Enter(CodeGenFunction &CGF)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
An aligned address.
Definition: Address.h:25
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
bool isLValue() const
isLValue - True if this expression is an "l-value" according to the rules of the current language...
Definition: Expr.h:249
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> &CondGen)
This represents &#39;#pragma omp target&#39; directive.
Definition: StmtOpenMP.h:2234
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
Expr * getV()
Get &#39;v&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2211
Complete object dtor.
Definition: ABI.h:36
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
This represents &#39;#pragma omp ordered&#39; directive.
Definition: StmtOpenMP.h:2045
QualType getType() const
Definition: CGValue.h:261
This represents &#39;#pragma omp target update&#39; directive.
Definition: StmtOpenMP.h:2986
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:216
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
Definition: CGCall.h:276
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:59
void setAction(PrePostActionTy &Action) const
CGFunctionInfo - Class to encapsulate the information about a function definition.
This class organizes the cross-function state that is used while generating LLVM code.
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
OpenMPScheduleClauseModifier
OpenMP modifiers for &#39;schedule&#39; clause.
Definition: OpenMPKinds.h:67
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
Definition: Decl.cpp:2455
Dataflow Directional Tag Classes.
std::string getAsString() const
getAsString - Retrieve the human-readable string for this name.
Expr * getPrevUpperBoundVariable() const
Definition: StmtOpenMP.h:844
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
This represents &#39;device&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value *> &)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:93
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
Definition: CGValue.h:480
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
SourceRange getSourceRange(const SourceRange &Range)
Returns the SourceRange of a SourceRange.
Definition: FixIt.h:34
Expr * getCond() const
Definition: StmtOpenMP.h:758
This represents &#39;#pragma omp section&#39; directive.
Definition: StmtOpenMP.h:1260
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Definition: LangOptions.h:147
OpenMPScheduleClauseModifier M1
Definition: OpenMPKinds.h:126
This represents &#39;#pragma omp teams distribute&#39; directive.
Definition: StmtOpenMP.h:3396
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
const Expr * getInit() const
Definition: Decl.h:1212
This represents &#39;#pragma omp simd&#39; directive.
Definition: StmtOpenMP.h:985
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:70
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
DeclarationName - The name of a declaration.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
const Expr * getReductionRef() const
Returns reference to the task_reduction return variable.
Definition: StmtOpenMP.h:1968
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
This represents clause &#39;linear&#39; in the &#39;#pragma omp ...&#39; directives.
SourceLocation getLocStart() const LLVM_READONLY
Definition: Decl.h:731
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPCancelDirective(const OMPCancelDirective &S)
Expr * getUpdateExpr()
Get helper expression of the form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; or &#39;OpaqueValueExp...
Definition: StmtOpenMP.h:2197
llvm::Module & getModule() const
This represents &#39;#pragma omp atomic&#39; directive.
Definition: StmtOpenMP.h:2100
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
Expr * getCombinedInit() const
Definition: StmtOpenMP.h:880
Expr * getCombinedNextLowerBound() const
Definition: StmtOpenMP.h:892
ArrayRef< Expr * > counters()
Definition: StmtOpenMP.h:917
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD)
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
SwitchStmt - This represents a &#39;switch&#39; stmt.
Definition: Stmt.h:1011
Expr * getCombinedNextUpperBound() const
Definition: StmtOpenMP.h:898
void EmitAutoVarCleanups(const AutoVarEmission &emission)
Definition: CGDecl.cpp:1421
API for captured statement code generation.
Complex values, per C99 6.2.5p11.
Definition: Type.h:2223
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, LValue LVal, RValue RVal)
This file defines OpenMP AST classes for executable directives and clauses.
ArrayRef< Expr * > inits()
Definition: StmtOpenMP.h:929
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:52
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
bool isStaticLocal() const
isStaticLocal - Returns true if a variable with function scope is a static local variable.
Definition: Decl.h:1049
OpenMPScheduleClauseKind
OpenMP attributes for &#39;schedule&#39; clause.
Definition: OpenMPKinds.h:59
Base for LValueReferenceType and RValueReferenceType.
Definition: Type.h:2419
StringRef getMangledName(GlobalDecl GD)
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
Definition: Linkage.h:33
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition: CGStmt.cpp:445
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
bool hasAssociatedStmt() const
Returns true if directive has associated statement.
Definition: StmtOpenMP.h:193
Expr * getPrevLowerBoundVariable() const
Definition: StmtOpenMP.h:838
const Expr * Replacement
Definition: AttributeList.h:59
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
Definition: ASTContext.h:2007
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
Definition: CGStmt.cpp:38
CanQualType getCanonicalType(QualType T) const
Return the canonical (structural) type corresponding to the specified potentially non-canonical type ...
Definition: ASTContext.h:2174
bool isLValueReferenceType() const
Definition: Type.h:5958
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
Definition: SemaDecl.cpp:13010
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
Definition: Stmt.cpp:1100
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
TranslationUnitDecl * getTranslationUnitDecl() const
Definition: ASTContext.h:989
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for reduction variables.
This represents &#39;#pragma omp target parallel&#39; directive.
Definition: StmtOpenMP.h:2467
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
This represents &#39;nowait&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:975
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g., it is an signed integer type or a vector.
Definition: Type.cpp:1830
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
Definition: CGStmt.cpp:465
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> &CondGen)
This represents &#39;num_tasks&#39; clause in the &#39;#pragma omp ...&#39; directive.
Privates[]
Gets the list of initial values for linear variables.
Definition: OpenMPClause.h:140
Expr * getCombinedCond() const
Definition: StmtOpenMP.h:886
void EmitOMPUseDevicePtrClause(const OMPClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, Address > &CaptureDeviceAddrMap)
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
Definition: CGExpr.cpp:1170
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Definition: Stmt.cpp:265
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition: CGExpr.cpp:1682
This represents &#39;#pragma omp taskloop simd&#39; directive.
Definition: StmtOpenMP.h:2852
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
Expr * getPreCond() const
Definition: StmtOpenMP.h:754
This represents &#39;dist_schedule&#39; clause in the &#39;#pragma omp ...&#39; directive.
CGCapturedStmtInfo * CapturedStmtInfo
bool isGlobalReg() const
Definition: CGValue.h:253
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
const llvm::function_ref< void(CodeGenFunction &, llvm::Value *, const OMPTaskDataTy &)> TaskGenTy
This represents &#39;#pragma omp sections&#39; directive.
Definition: StmtOpenMP.h:1192
This represents &#39;#pragma omp target data&#39; directive.
Definition: StmtOpenMP.h:2292
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:2771
capture_range captures()
Definition: Stmt.h:2193
void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> &BodyGen, const llvm::function_ref< void(CodeGenFunction &)> &PostIncGen)
Emit inner loop of the worksharing/simd construct.
A reference to a declared variable, function, enum, etc.
Definition: Expr.h:956
static RValue get(llvm::Value *V)
Definition: CGValue.h:86
bool isPointerType() const
Definition: Type.h:5942
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
QualType getType() const
Definition: Decl.h:638
An l-value expression is a reference to an object with independent storage.
Definition: Specifiers.h:111
A trivial tuple used to represent a source range.
LValue - This represents an lvalue references.
Definition: CGValue.h:167
This represents &#39;#pragma omp taskyield&#39; directive.
Definition: StmtOpenMP.h:1785
This represents &#39;#pragma omp distribute parallel for simd&#39; composite directive.
Definition: StmtOpenMP.h:3125
This represents &#39;#pragma omp parallel sections&#39; directive.
Definition: StmtOpenMP.h:1646
SourceLocation getLocStart() const LLVM_READONLY
Definition: Stmt.cpp:277
SourceLocation getBegin() const
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
Definition: Stmt.h:2242
Address CreateMemTemp(QualType T, const Twine &Name="tmp", bool CastToDefaultAddrSpace=true)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignment...
Definition: CGExpr.cpp:127
llvm::Value * getPointer() const
Definition: CGValue.h:320
void EmitOMPTaskDirective(const OMPTaskDirective &S)
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
Definition: DeclBase.cpp:882
bool isScalar() const
Definition: CGValue.h:52
This represents &#39;#pragma omp target parallel for&#39; directive.
Definition: StmtOpenMP.h:2527
SmallVector< const Expr *, 4 > FirstprivateInits
SourceLocation getLocation() const
Definition: DeclBase.h:416
This represents clause &#39;use_device_ptr&#39; in the &#39;#pragma omp ...&#39; directives.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool Privatize()
Privatizes local variables previously registered as private.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1524
This represents &#39;#pragma omp taskloop&#39; directive.
Definition: StmtOpenMP.h:2787